diff --git a/docs/contributing.rst b/docs/contributing.rst index 4e8330b3c..b0a435ad2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -100,6 +100,11 @@ starting point is to send a pull request on GitHub. It can be simple enough to illustrate your idea, and leave documentation/tests for later, after the idea has been validated and proven useful. Alternatively, you can send an email to `scrapy-users`_ to discuss your idea first. +When writing GitHub pull requests, try to keep titles short but descriptive. +E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" +prefer "Fix hanging when exception occurs in start_requests (#411)" +instead of "Fix for #411". +Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits than functional changes. This will make pull diff --git a/docs/faq.rst b/docs/faq.rst index 82e1f3422..415331515 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -73,6 +73,9 @@ Scrapy is supported under Python 2.7 and Python 3.3+. Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. +.. note:: + Python 3 is not yet supported on Windows. + Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 25520b4b9..3364c3b31 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -11,7 +11,7 @@ Installing Scrapy The installation steps assume that you have the following things installed: -* `Python`_ 2.7 +* `Python`_ 2.7 or above 3.3 * `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and installs `setuptools`_ if not installed. Python 2.7.9 and later include @@ -85,6 +85,10 @@ Windows pip install Scrapy +.. note:: + Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support + Python 3 on Windows. + Ubuntu 9.10 or above -------------------- diff --git a/docs/news.rst b/docs/news.rst index cbcd4d613..ac87b449d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -72,7 +72,7 @@ features are still missing (and some may never be ported). Almost all builtin extensions/middlewares are expected to work. However, we are aware of some limitations in Python 3: -- Scrapy has not been tested on Windows with Python 3 +- Scrapy does not work on Windows with Python 3 - Sending emails is not supported - FTP download handler is not supported - Telnet console is not supported diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9a40a2c29..d7999900b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -159,6 +159,7 @@ settings). Global commands: * :command:`startproject` +* :command:`genspider` * :command:`settings` * :command:`runspider` * :command:`shell` @@ -173,7 +174,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`genspider` * :command:`bench` .. command:: startproject @@ -197,14 +197,9 @@ genspider --------- * Syntax: ``scrapy genspider [-t template] `` -* Requires project: *yes* +* Requires project: *no* -Create a new spider in the current project. - -This is just a convenience shortcut command for creating spiders based on -pre-defined templates, but certainly not the only way to create spiders. You -can just create the spider source code files yourself, instead of using this -command. +Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -215,22 +210,16 @@ Usage example:: csvfeed xmlfeed - $ scrapy genspider -d basic - import scrapy + $ scrapy genspider example example.com + Created spider 'example' using template 'basic' - class $classname(scrapy.Spider): - name = "$name" - allowed_domains = ["$domain"] - start_urls = ( - 'http://www.$domain/', - ) + $ scrapy genspider -t crawl scrapyorg scrapy.org + Created spider 'scrapyorg' using template 'crawl' - def parse(self, response): - pass - - $ scrapy genspider -t basic example example.com - Created spider 'example' using template 'basic' in module: - mybot.spiders.example +This is just a convenience shortcut command for creating spiders based on +pre-defined templates, but certainly not the only way to create spiders. You +can just create the spider source code files yourself, instead of using this +command. .. command:: crawl diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7f49aacdb..c845c59b9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -459,9 +459,9 @@ Default:: 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 58bdb9156..d5498bb5c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -25,7 +25,7 @@ def sanitize_module_name(module_name): class Command(ScrapyCommand): - requires_project = True + requires_project = False default_settings = {'LOG_ENABLED': False} def syntax(self): @@ -94,14 +94,19 @@ class Command(ScrapyCommand): 'classname': '%sSpider' % ''.join(s.capitalize() \ for s in module.split('_')) } - spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = abspath(dirname(spiders_module.__file__)) + if self.settings.get('NEWSPIDER_MODULE'): + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = abspath(dirname(spiders_module.__file__)) + else: + spiders_module = None + spiders_dir = "." spider_file = "%s.py" % join(spiders_dir, module) shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r in module:" % (name, \ - template_name)) - print(" %s.%s" % (spiders_module.__name__, module)) + print("Created spider %r using template %r " % (name, \ + template_name), end=('' if spiders_module else '\n')) + if spiders_module: + print("in module:\n %s.%s" % (spiders_module.__name__, module)) def _find_template(self, template): template_file = join(self.templates_dir, '%s.tmpl' % template) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 88c6b9480..404e9160b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile(b'HTTP/1\.. 200') + _responseMatcher = re.compile(b'HTTP/1\.. (?P\d{3})(?P.{,32})') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): @@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocol = protocol return protocol - def processProxyResponse(self, bytes): + def processProxyResponse(self, rcvd_bytes): """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ self._protocol.dataReceived = self._protocolDataReceived - if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): + respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes) + if respm and int(respm.group('status')) == 200: try: # this sets proper Server Name Indication extension # but is only available for Twisted>=14.0 @@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + if respm: + extra = {'status': int(respm.group('status')), + 'reason': respm.group('reason').strip()} + else: + extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % ( - self._host, self._port))) + TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( + self._host, self._port, extra))) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -157,17 +163,15 @@ def tunnel_request_data(host, port, proxy_auth_header=None): >>> from scrapy.utils.python import to_native_str as s >>> s(tunnel_request_data("example.com", 8080)) - 'CONNECT example.com:8080 HTTP/1.1\r\n\r\n' + 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' >>> s(tunnel_request_data("example.com", 8080, b"123")) - 'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n' + 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' >>> s(tunnel_request_data(b"example.com", "8090")) - 'CONNECT example.com:8090 HTTP/1.1\r\n\r\n' + 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ - tunnel_req = ( - b'CONNECT ' + - to_bytes(host, encoding='ascii') + b':' + - to_bytes(str(port)) + - b' HTTP/1.1\r\n') + host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port)) + tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n' + tunnel_req += b'Host: ' + host_value + b'\r\n' if proxy_auth_header: tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n' tunnel_req += b'\r\n' @@ -343,7 +347,7 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url) + respcls = responsetypes.from_args(headers=headers, url=url, body=body) return respcls(url=url, status=status, headers=headers, body=body, flags=flags) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8f064f81e..e563e56aa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -93,9 +93,9 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index a179d16ff..154237d9c 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -3,8 +3,6 @@ import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule -from $project_name.items import ${ProjectName}Item - class $classname(CrawlSpider): name = '$name' @@ -16,7 +14,7 @@ class $classname(CrawlSpider): ) def parse_item(self, response): - i = ${ProjectName}Item() + i = {} #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() #i['name'] = response.xpath('//div[@id="name"]').extract() #i['description'] = response.xpath('//div[@id="description"]').extract() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 69c606538..0544e0ae7 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider -from $project_name.items import ${ProjectName}Item - class $classname(CSVFeedSpider): name = '$name' @@ -16,7 +14,7 @@ class $classname(CSVFeedSpider): # return response def parse_row(self, response, row): - i = ${ProjectName}Item() + i = {} #i['url'] = row['url'] #i['name'] = row['name'] #i['description'] = row['description'] diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 9c0910d23..d8ff61f6e 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider -from $project_name.items import ${ProjectName}Item - class $classname(XMLFeedSpider): name = '$name' @@ -12,7 +10,7 @@ class $classname(XMLFeedSpider): itertag = 'item' # change it accordingly def parse_node(self, response, selector): - i = ${ProjectName}Item() + i = {} #i['url'] = selector.select('url').extract() #i['name'] = selector.select('name').extract() #i['description'] = selector.select('description').extract() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index b2f737193..9fe88d108 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,7 +7,7 @@ except ImportError: from gzip import GzipFile import six - +import re # - Python>=3.5 GzipFile's read() has issues returning leftover # uncompressed data when input is corrupted @@ -50,13 +50,13 @@ def gunzip(data): raise return output +_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I) +_is_octetstream_re = re.compile(br'^(application|binary)/octet-stream\b', re.I) def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'').lower() + ctype = response.headers.get('Content-Type', b'') cenc = response.headers.get('Content-Encoding', b'').lower() - return ( - ctype in (b'application/x-gzip', b'application/gzip') or - (ctype in (b'application/octet-stream', b'binary/octet-stream') and - cenc in (b'gzip', b'x-gzip')) - ) + return (_is_gzipped_re.search(ctype) is not None or + (_is_octetstream_re.search(ctype) is not None and + cenc in (b'gzip', b'x-gzip'))) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c80fc6e70..406eb5843 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions): def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'): + # IDNA encoding can fail for too long labels (>63 characters) + # or missing labels (e.g. http://.example.com) + try: + netloc = parts.netloc.encode('idna') + except UnicodeError: + netloc = parts.netloc + return ( to_native_str(parts.scheme), - to_native_str(parts.netloc.encode('idna')), + to_native_str(netloc), # default encoding for path component SHOULD be UTF-8 quote(to_bytes(parts.path, path_encoding), _safe_chars), diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2a89763a5..ed189c66c 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,6 @@ -pytest==2.7.3 +pytest==2.9.2 pytest-twisted -pytest-cov +pytest-cov==2.2.1 testfixtures jmespath leveldb diff --git a/tests/requirements.txt b/tests/requirements.txt index 8901fe16b..9d0c3c996 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,9 +2,9 @@ mock mitmproxy==0.10.1 netlib==0.10.1 -pytest==2.7.3 +pytest==2.9.2 pytest-twisted -pytest-cov +pytest-cov==2.2.1 jmespath testfixtures # optional for shell wrapper tests diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e47160d7..cf415a388 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -146,6 +146,13 @@ class GenspiderCommandTest(CommandTest): assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) +class GenspiderStandaloneCommandTest(ProjectTest): + + def test_generate_standalone_spider(self): + self.call('genspider', 'example', 'example.com') + assert exists(join(self.temp_path, 'example.py')) + + class MiscCommandsTest(CommandTest): def test_list(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45a806f2e..c9f8bb797 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -27,6 +27,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request +from scrapy.http.response.text import TextResponse from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes @@ -114,6 +115,16 @@ class ContentLengthHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"content-length")[0] +class EmptyContentTypeHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of request body + without content-type header in response. + """ + def render(self, request): + request.setHeader("content-type", "") + return request.content.read() + + class HttpTestCase(unittest.TestCase): scheme = 'http' @@ -136,6 +147,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) + r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' @@ -243,11 +255,6 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) - d = self.download_request(request, Spider('foo')) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'0') - return d - def test_payload(self): body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) @@ -284,6 +291,20 @@ class Http11TestCase(HttpTestCase): d.addCallback(self.assertEquals, b"0123456789") return d + def test_response_class_choosing_request(self): + """Tests choosing of correct response type + in case of Content-Type is empty but body contains text. + """ + body = b'Some plain text\ndata with tabs\t and null bytes\0' + + def _test_type(response): + self.assertEquals(type(response), TextResponse) + + request = Request(self.getURL('nocontenttype'), body=body) + d = self.download_request(request, Spider('foo')) + d.addCallback(_test_type) + return d + @defer.inlineCallbacks def test_download_with_maxsize(self): request = Request(self.getURL('file')) @@ -401,7 +422,13 @@ class UriResource(resource.Resource): return self def render(self, request): - return request.uri + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b'CONNECT': + return request.uri + else: + return b'' class HttpProxyTestCase(unittest.TestCase): diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 8fb1e414d..2b47bf8da 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,7 +1,8 @@ import unittest from os.path import join -from scrapy.utils.gz import gunzip +from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.http import Response, Headers from tests import tests_datadir SAMPLEDIR = join(tests_datadir, 'compressed') @@ -27,3 +28,41 @@ class GunzipTest(unittest.TestCase): with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: text = gunzip(f.read()) assert text.endswith(b'') + + def test_is_x_gzipped_right(self): + hdrs = Headers({"Content-Type": "application/x-gzip"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + + def test_is_gzipped_right(self): + hdrs = Headers({"Content-Type": "application/gzip"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + + def test_is_gzipped_not_quite(self): + hdrs = Headers({"Content-Type": "application/gzippppp"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertFalse(is_gzipped(r1)) + + def test_is_gzipped_case_insensitive(self): + hdrs = Headers({"Content-Type": "Application/X-Gzip"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + + hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + + def test_is_gzipped_empty(self): + r1 = Response("http://www.example.com") + self.assertFalse(is_gzipped(r1)) + + def test_is_gzipped_wrong(self): + hdrs = Headers({"Content-Type": "application/javascript"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertFalse(is_gzipped(r1)) + + def test_is_gzipped_with_charset(self): + hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1fc3a3510..b4819874d 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase): # without encoding, already canonicalized URL is canonicalized identically self.assertEqual(canonicalize_url(canonicalized), canonicalized) + def test_canonicalize_url_idna_exceptions(self): + # missing DNS label + self.assertEqual( + canonicalize_url(u"http://.example.com/résumé?q=résumé"), + "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9") + + # DNS label too long + self.assertEqual( + canonicalize_url( + u"http://www.{label}.com/résumé?q=résumé".format( + label=u"example"*11)), + "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format( + label=u"example"*11)) + class AddHttpIfNoScheme(unittest.TestCase):