RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PY3 raise an exception if bytes are passed as url to Link constructor

This commit is contained in:
Mikhail Korobov 2015-08-28 23:04:02 +05:00
parent 9bfe6ece59
commit f7052413e0
2 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scrapy/link.py
View File

@ -4,7 +4,10 @@ This module defines the Link object used in Link extractors.
For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
from scrapy.utils.python import to_native_str
import warnings
import six
from scrapy.utils.python import to_bytes
class Link(object):
@ -14,9 +17,13 @@ class Link(object):
def __init__(self, url, text='', fragment='', nofollow=False):
if not isinstance(url, str):
import warnings
warnings.warn("Link urls must be str objects.")
url = to_native_str(url)
if six.PY2:
warnings.warn("Link urls must be str objects. "
"Assuming utf-8 encoding (which could be wrong)")
url = to_bytes(url, encoding='utf8')
else:
got = url.__class__.__name__
raise TypeError("Link urls must be str objects, got %s" % got)
self.url = url
self.text = text
self.fragment = fragment

38
tests/test_link.py
View File

@ -16,44 +16,42 @@ class LinkTest(unittest.TestCase):
self.assertNotEqual(hash(link1), hash(link2))
def test_eq_and_hash(self):
l1 = Link(b"http://www.example.com")
l2 = Link(b"http://www.example.com/other")
l3 = Link(b"http://www.example.com")
l1 = Link("http://www.example.com")
l2 = Link("http://www.example.com/other")
l3 = Link("http://www.example.com")
self._assert_same_links(l1, l1)
self._assert_different_links(l1, l2)
self._assert_same_links(l1, l3)
l4 = Link(b"http://www.example.com", text="test")
l5 = Link(b"http://www.example.com", text="test2")
l6 = Link(b"http://www.example.com", text="test")
l4 = Link("http://www.example.com", text="test")
l5 = Link("http://www.example.com", text="test2")
l6 = Link("http://www.example.com", text="test")
self._assert_same_links(l4, l4)
self._assert_different_links(l4, l5)
self._assert_same_links(l4, l6)
l7 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
l8 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
l9 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
l10 = Link(b"http://www.example.com", text="test", fragment='other', nofollow=False)
l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False)
self._assert_same_links(l7, l8)
self._assert_different_links(l7, l9)
self._assert_different_links(l7, l10)
def test_repr(self):
l1 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
l2 = eval(repr(l1))
self._assert_same_links(l1, l2)
def test_non_str_url(self):
with warnings.catch_warnings(record=True) as w:
if six.PY2:
def test_non_str_url_py2(self):
if six.PY2:
with warnings.catch_warnings(record=True) as w:
link = Link(u"http://www.example.com/\xa3")
self.assertIsInstance(link.url, str)
self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
else:
link = Link(b"http://www.example.com/\xc2\xa3")
self.assertIsInstance(link.url, str)
self.assertEqual(link.url, u'http://www.example.com/\xa3')
assert len(w) == 1, "warning not issued"
assert len(w) == 1, "warning not issued"
else:
with self.assertRaises(TypeError):
Link(b"http://www.example.com/\xc2\xa3")