diff --git a/scrapy/link.py b/scrapy/link.py index dc6e64adc..2c8301680 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,7 +4,10 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -from scrapy.utils.python import to_native_str +import warnings +import six + +from scrapy.utils.python import to_bytes class Link(object): @@ -14,9 +17,13 @@ class Link(object): def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): - import warnings - warnings.warn("Link urls must be str objects.") - url = to_native_str(url) + if six.PY2: + warnings.warn("Link urls must be str objects. " + "Assuming utf-8 encoding (which could be wrong)") + url = to_bytes(url, encoding='utf8') + else: + got = url.__class__.__name__ + raise TypeError("Link urls must be str objects, got %s" % got) self.url = url self.text = text self.fragment = fragment diff --git a/tests/test_link.py b/tests/test_link.py index c8487698f..955430b37 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -16,44 +16,42 @@ class LinkTest(unittest.TestCase): self.assertNotEqual(hash(link1), hash(link2)) def test_eq_and_hash(self): - l1 = Link(b"http://www.example.com") - l2 = Link(b"http://www.example.com/other") - l3 = Link(b"http://www.example.com") + l1 = Link("http://www.example.com") + l2 = Link("http://www.example.com/other") + l3 = Link("http://www.example.com") self._assert_same_links(l1, l1) self._assert_different_links(l1, l2) self._assert_same_links(l1, l3) - l4 = Link(b"http://www.example.com", text="test") - l5 = Link(b"http://www.example.com", text="test2") - l6 = Link(b"http://www.example.com", text="test") + l4 = Link("http://www.example.com", text="test") + l5 = Link("http://www.example.com", text="test2") + l6 = Link("http://www.example.com", text="test") self._assert_same_links(l4, l4) self._assert_different_links(l4, l5) self._assert_same_links(l4, l6) - l7 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False) - l8 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False) - l9 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True) - l10 = Link(b"http://www.example.com", text="test", fragment='other', nofollow=False) + l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False) + l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False) + l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) + l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False) self._assert_same_links(l7, l8) self._assert_different_links(l7, l9) self._assert_different_links(l7, l10) def test_repr(self): - l1 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True) + l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_non_str_url(self): - with warnings.catch_warnings(record=True) as w: - if six.PY2: + def test_non_str_url_py2(self): + if six.PY2: + with warnings.catch_warnings(record=True) as w: link = Link(u"http://www.example.com/\xa3") self.assertIsInstance(link.url, str) self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3') - else: - link = Link(b"http://www.example.com/\xc2\xa3") - self.assertIsInstance(link.url, str) - self.assertEqual(link.url, u'http://www.example.com/\xa3') - - assert len(w) == 1, "warning not issued" + assert len(w) == 1, "warning not issued" + else: + with self.assertRaises(TypeError): + Link(b"http://www.example.com/\xc2\xa3")