PY3 port scrapy.link

scrapy/link.py

@@ -5,15 +5,17 @@ For actual link extractors implementation see scrapy.contrib.linkextractor, or
 its documentation in: docs/topics/link-extractors.rst
 """
 
+import six
+
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
     __slots__ = ['url', 'text', 'fragment', 'nofollow']
 
     def __init__(self, url, text='', fragment='', nofollow=False):
-        if isinstance(url, unicode):
+        if isinstance(url, six.text_type):
             import warnings
-            warnings.warn("Do not instantiate Link objects with unicode urls. " \
+            warnings.warn("Do not instantiate Link objects with unicode urls. "
                 "Assuming utf-8 encoding (which could be wrong)")
             url = url.encode('utf-8')
         self.url = url
@@ -24,7 +26,7 @@ class Link(object):
     def __eq__(self, other):
         return self.url == other.url and self.text == other.text and \
             self.fragment == other.fragment and self.nofollow == other.nofollow
-   
+
     def __hash__(self):
         return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
 

tests/py3-ignores.txt

@@ -28,7 +28,6 @@ tests/test_engine.py
 tests/test_http_cookies.py
 tests/test_http_request.py
 tests/test_http_response.py
-tests/test_link.py
 tests/test_logformatter.py
 tests/test_log.py
 tests/test_mail.py

tests/test_link.py

@@ -14,38 +14,38 @@ class LinkTest(unittest.TestCase):
         self.assertNotEqual(hash(link1), hash(link2))
 
     def test_eq_and_hash(self):
-        l1 = Link("http://www.example.com")
-        l2 = Link("http://www.example.com/other")
-        l3 = Link("http://www.example.com")
+        l1 = Link(b"http://www.example.com")
+        l2 = Link(b"http://www.example.com/other")
+        l3 = Link(b"http://www.example.com")
 
         self._assert_same_links(l1, l1)
         self._assert_different_links(l1, l2)
         self._assert_same_links(l1, l3)
 
-        l4 = Link("http://www.example.com", text="test")
-        l5 = Link("http://www.example.com", text="test2")
-        l6 = Link("http://www.example.com", text="test")
+        l4 = Link(b"http://www.example.com", text="test")
+        l5 = Link(b"http://www.example.com", text="test2")
+        l6 = Link(b"http://www.example.com", text="test")
 
         self._assert_same_links(l4, l4)
         self._assert_different_links(l4, l5)
         self._assert_same_links(l4, l6)
 
-        l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
-        l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
-        l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
-        l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False)
+        l7 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
+        l8 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
+        l9 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
+        l10 = Link(b"http://www.example.com", text="test", fragment='other', nofollow=False)
         self._assert_same_links(l7, l8)
         self._assert_different_links(l7, l9)
         self._assert_different_links(l7, l10)
 
     def test_repr(self):
-        l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
+        l1 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
     def test_unicode_url(self):
         with warnings.catch_warnings(record=True) as w:
-            l = Link(u"http://www.example.com/\xa3")
-            assert isinstance(l.url, str)
-            assert l.url == 'http://www.example.com/\xc2\xa3'
+            link = Link(u"http://www.example.com/\xa3")
+            self.assertIsInstance(link.url, bytes)
+            self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
             assert len(w) == 1, "warning not issued"