From b4f368c37ee0b42812d5412b98976038ea335d0e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 16 May 2012 13:12:25 -0300 Subject: [PATCH] warn if Link objects are instantiated with unicode urls --- scrapy/link.py | 5 +++++ scrapy/tests/test_link.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/scrapy/link.py b/scrapy/link.py index 0909709a6..51ec29f5c 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -11,6 +11,11 @@ class Link(object): __slots__ = ['url', 'text', 'fragment', 'nofollow'] def __init__(self, url, text='', fragment='', nofollow=False): + if isinstance(url, unicode): + import warnings + warnings.warn("Do not instantiate Link objects with unicode urls. " \ + "Assuming utf-8 encoding (which could be wrong)") + url = url.encode('utf-8') self.url = url self.text = text self.fragment = fragment diff --git a/scrapy/tests/test_link.py b/scrapy/tests/test_link.py index b0edd642d..32c35fdde 100644 --- a/scrapy/tests/test_link.py +++ b/scrapy/tests/test_link.py @@ -1,4 +1,5 @@ import unittest +import warnings from scrapy.link import Link @@ -41,3 +42,10 @@ class LinkTest(unittest.TestCase): l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) + + def test_unicode_url(self): + with warnings.catch_warnings(record=True) as w: + l = Link(u"http://www.example.com/\xa3") + assert isinstance(l.url, str) + assert l.url == 'http://www.example.com/\xc2\xa3' + assert len(w) == 1, "warning not issued"