mirror of https://github.com/scrapy/scrapy.git
222 lines
15 KiB
Python
222 lines
15 KiB
Python
import unittest
|
|
from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, \
|
|
url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, \
|
|
urljoin_rfc
|
|
|
|
|
|
class UrlUtilsTest(unittest.TestCase):
|
|
|
|
def test_url_is_from_any_domain(self):
|
|
url = 'http://www.wheele-bin-art.co.uk/get/product/123'
|
|
self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk']))
|
|
self.assertFalse(url_is_from_any_domain(url, ['art.co.uk']))
|
|
|
|
url = 'http://wheele-bin-art.co.uk/get/product/123'
|
|
self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk']))
|
|
self.assertFalse(url_is_from_any_domain(url, ['art.co.uk']))
|
|
|
|
url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
|
|
self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
|
|
self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com']))
|
|
|
|
def test_urljoin_rfc(self):
|
|
self.assertEqual(urljoin_rfc('http://example.com/some/path', 'newpath/test'),
|
|
'http://example.com/some/newpath/test')
|
|
self.assertEqual(urljoin_rfc('http://example.com/some/path/a.jpg', '../key/other'),
|
|
'http://example.com/some/key/other')
|
|
u = urljoin_rfc(u'http://example.com/lolo/\xa3/lele', u'lala/\xa3')
|
|
self.assertEqual(u, 'http://example.com/lolo/\xc2\xa3/lala/\xc2\xa3')
|
|
assert isinstance(u, str)
|
|
u = urljoin_rfc(u'http://example.com/lolo/\xa3/lele', 'lala/\xa3', encoding='latin-1')
|
|
self.assertEqual(u, 'http://example.com/lolo/\xa3/lala/\xa3')
|
|
assert isinstance(u, str)
|
|
u = urljoin_rfc('http://example.com/lolo/\xa3/lele', 'lala/\xa3')
|
|
self.assertEqual(u, 'http://example.com/lolo/\xa3/lala/\xa3')
|
|
assert isinstance(u, str)
|
|
|
|
def test_safe_url_string(self):
|
|
# Motoko Kusanagi (Cyborg from Ghost in the Shell)
|
|
motoko = u'\u8349\u8599 \u7d20\u5b50'
|
|
self.assertEqual(safe_url_string(motoko), # note the %20 for space
|
|
'%E8%8D%89%E8%96%99%20%E7%B4%A0%E5%AD%90')
|
|
self.assertEqual(safe_url_string(motoko),
|
|
safe_url_string(safe_url_string(motoko)))
|
|
self.assertEqual(safe_url_string(u'\xa9'), # copyright symbol
|
|
'%C2%A9')
|
|
self.assertEqual(safe_url_string(u'\xa9', 'iso-8859-1'),
|
|
'%A9')
|
|
self.assertEqual(safe_url_string("http://www.scrapy.org/"),
|
|
'http://www.scrapy.org/')
|
|
|
|
alessi = u'/ecommerce/oggetto/Te \xf2/tea-strainer/1273'
|
|
|
|
self.assertEqual(safe_url_string(alessi),
|
|
'/ecommerce/oggetto/Te%20%C3%B2/tea-strainer/1273')
|
|
|
|
self.assertEqual(safe_url_string("http://www.example.com/test?p(29)url(http://www.another.net/page)"),
|
|
"http://www.example.com/test?p(29)url(http://www.another.net/page)")
|
|
self.assertEqual(safe_url_string("http://www.example.com/Brochures_&_Paint_Cards&PageSize=200"),
|
|
"http://www.example.com/Brochures_&_Paint_Cards&PageSize=200")
|
|
|
|
def test_safe_download_url(self):
|
|
self.assertEqual(safe_download_url('http://www.scrapy.org/../'),
|
|
'http://www.scrapy.org/')
|
|
self.assertEqual(safe_download_url('http://www.scrapy.org/../../images/../image'),
|
|
'http://www.scrapy.org/image')
|
|
self.assertEqual(safe_download_url('http://www.scrapy.org/dir/'),
|
|
'http://www.scrapy.org/dir/')
|
|
|
|
def test_url_query_parameter(self):
|
|
self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "id"),
|
|
'200')
|
|
self.assertEqual(url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault"),
|
|
'mydefault')
|
|
self.assertEqual(url_query_parameter("product.html?id=", "id"),
|
|
None)
|
|
self.assertEqual(url_query_parameter("product.html?id=", "id", keep_blank_values=1),
|
|
'')
|
|
|
|
def test_url_query_parameter_2(self):
|
|
"""
|
|
This problem was seen several times in the feeds. Sometime affiliate URLs contains
|
|
nested encoded affiliate URL with direct URL as parameters. For example:
|
|
aff_url1 = 'http://www.tkqlhce.com/click-2590032-10294381?url=http%3A%2F%2Fwww.argos.co.uk%2Fwebapp%2Fwcs%2Fstores%2Fservlet%2FArgosCreateReferral%3FstoreId%3D10001%26langId%3D-1%26referrer%3DCOJUN%26params%3Dadref%253DGarden+and+DIY-%3EGarden+furniture-%3EChildren%26%2339%3Bs+garden+furniture%26referredURL%3Dhttp%3A%2F%2Fwww.argos.co.uk%2Fwebapp%2Fwcs%2Fstores%2Fservlet%2FProductDisplay%253FstoreId%253D10001%2526catalogId%253D1500001501%2526productId%253D1500357023%2526langId%253D-1'
|
|
the typical code to extract needed URL from it is:
|
|
aff_url2 = url_query_parameter(aff_url1, 'url')
|
|
after this aff2_url is:
|
|
'http://www.argos.co.uk/webapp/wcs/stores/servlet/ArgosCreateReferral?storeId=10001&langId=-1&referrer=COJUN¶ms=adref%3DGarden and DIY->Garden furniture->Children's gardenfurniture&referredURL=http://www.argos.co.uk/webapp/wcs/stores/servlet/ProductDisplay%3FstoreId%3D10001%26catalogId%3D1500001501%26productId%3D1500357023%26langId%3D-1'
|
|
the direct URL extraction is
|
|
url = url_query_parameter(aff_url2, 'referredURL')
|
|
but this will not work, because aff_url2 contains ' (comma sign encoded in the feed)
|
|
and the URL extraction will fail, current workaround was made in the spider,
|
|
just a replace for ' to %27
|
|
"""
|
|
return # FIXME: this test should pass but currently doesnt
|
|
# correct case
|
|
aff_url1 = "http://www.anrdoezrs.net/click-2590032-10294381?url=http%3A%2F%2Fwww.argos.co.uk%2Fwebapp%2Fwcs%2Fstores%2Fservlet%2FArgosCreateReferral%3FstoreId%3D10001%26langId%3D-1%26referrer%3DCOJUN%26params%3Dadref%253DGarden+and+DIY-%3EGarden+furniture-%3EGarden+table+and+chair+sets%26referredURL%3Dhttp%3A%2F%2Fwww.argos.co.uk%2Fwebapp%2Fwcs%2Fstores%2Fservlet%2FProductDisplay%253FstoreId%253D10001%2526catalogId%253D1500001501%2526productId%253D1500357199%2526langId%253D-1"
|
|
aff_url2 = url_query_parameter(aff_url1, 'url')
|
|
self.assertEqual(aff_url2, "http://www.argos.co.uk/webapp/wcs/stores/servlet/ArgosCreateReferral?storeId=10001&langId=-1&referrer=COJUN¶ms=adref%3DGarden and DIY->Garden furniture->Garden table and chair sets&referredURL=http://www.argos.co.uk/webapp/wcs/stores/servlet/ProductDisplay%3FstoreId%3D10001%26catalogId%3D1500001501%26productId%3D1500357199%26langId%3D-1")
|
|
prod_url = url_query_parameter(aff_url2, 'referredURL')
|
|
self.assertEqual(prod_url, "http://www.argos.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=10001&catalogId=1500001501&productId=1500357199&langId=-1")
|
|
# weird case
|
|
aff_url1 = "http://www.tkqlhce.com/click-2590032-10294381?url=http%3A%2F%2Fwww.argos.co.uk%2Fwebapp%2Fwcs%2Fstores%2Fservlet%2FArgosCreateReferral%3FstoreId%3D10001%26langId%3D-1%26referrer%3DCOJUN%26params%3Dadref%253DGarden+and+DIY-%3EGarden+furniture-%3EChildren%26%2339%3Bs+garden+furniture%26referredURL%3Dhttp%3A%2F%2Fwww.argos.co.uk%2Fwebapp%2Fwcs%2Fstores%2Fservlet%2FProductDisplay%253FstoreId%253D10001%2526catalogId%253D1500001501%2526productId%253D1500357023%2526langId%253D-1"
|
|
aff_url2 = url_query_parameter(aff_url1, 'url')
|
|
self.assertEqual(aff_url2, "http://www.argos.co.uk/webapp/wcs/stores/servlet/ArgosCreateReferral?storeId=10001&langId=-1&referrer=COJUN¶ms=adref%3DGarden and DIY->Garden furniture->Children's garden furniture&referredURL=http://www.argos.co.uk/webapp/wcs/stores/servlet/ProductDisplay%3FstoreId%3D10001%26catalogId%3D1500001501%26productId%3D1500357023%26langId%3D-1")
|
|
prod_url = url_query_parameter(aff_url2, 'referredURL')
|
|
# fails, prod_url is None now
|
|
self.assertEqual(prod_url, "http://www.argos.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=10001&catalogId=1500001501&productId=1500357023&langId=-1")
|
|
|
|
def test_add_or_replace_parameter(self):
|
|
url = 'http://domain/test'
|
|
self.assertEqual(add_or_replace_parameter(url, 'arg', 'v'),
|
|
'http://domain/test?arg=v')
|
|
url = 'http://domain/test?arg1=v1&arg2=v2&arg3=v3'
|
|
self.assertEqual(add_or_replace_parameter(url, 'arg4', 'v4'),
|
|
'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4')
|
|
self.assertEqual(add_or_replace_parameter(url, 'arg3', 'nv3'),
|
|
'http://domain/test?arg1=v1&arg2=v2&arg3=nv3')
|
|
url = 'http://domain/test?arg1=v1'
|
|
self.assertEqual(add_or_replace_parameter(url, 'arg2', 'v2', sep=';'),
|
|
'http://domain/test?arg1=v1;arg2=v2')
|
|
self.assertEqual(add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20'),
|
|
'http://domain/moreInfo.asp?prodID=20')
|
|
url = 'http://rmc-offers.co.uk/productlist.asp?BCat=2%2C60&CatID=60'
|
|
self.assertEqual(add_or_replace_parameter(url, 'BCat', 'newvalue', url_is_quoted=True),
|
|
'http://rmc-offers.co.uk/productlist.asp?BCat=newvalue&CatID=60')
|
|
url = 'http://rmc-offers.co.uk/productlist.asp?BCat=2,60&CatID=60'
|
|
self.assertEqual(add_or_replace_parameter(url, 'BCat', 'newvalue'),
|
|
'http://rmc-offers.co.uk/productlist.asp?BCat=newvalue&CatID=60')
|
|
|
|
def test_url_query_cleaner(self):
|
|
self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", 'id'),
|
|
'product.html?id=200')
|
|
self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']),
|
|
'product.html?id=200&name=wired')
|
|
self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo']),
|
|
'product.html?id=200&foo=bar')
|
|
|
|
def test_canonicalize_url(self):
|
|
# simplest case
|
|
self.assertEqual(canonicalize_url("http://www.example.com"),
|
|
"http://www.example.com")
|
|
|
|
# always return a str
|
|
assert isinstance(canonicalize_url(u"http://www.example.com"), str)
|
|
|
|
# typical usage
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?a=1&b=2&c=3"),
|
|
"http://www.example.com/do?a=1&b=2&c=3")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?c=1&b=2&a=3"),
|
|
"http://www.example.com/do?a=3&b=2&c=1")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?&a=1"),
|
|
"http://www.example.com/do?a=1")
|
|
|
|
# sorting by argument values
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?c=3&b=5&b=2&a=50"),
|
|
"http://www.example.com/do?a=50&b=2&b=5&c=3")
|
|
|
|
# using keep_blank_values
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=False),
|
|
"http://www.example.com/do?a=2")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"),
|
|
"http://www.example.com/do?a=2&b=")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=False),
|
|
"http://www.example.com/do?a=2")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"),
|
|
"http://www.example.com/do?a=2&b=&c=")
|
|
|
|
self.assertEqual(canonicalize_url(u'http://www.example.com/do?1750,4'),
|
|
'http://www.example.com/do?1750%2C4=')
|
|
|
|
# spaces
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"),
|
|
"http://www.example.com/do?a=1&q=a+space")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?q=a+space&a=1"),
|
|
"http://www.example.com/do?a=1&q=a+space")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?q=a%20space&a=1"),
|
|
"http://www.example.com/do?a=1&q=a+space")
|
|
|
|
# normalize percent-encoding case (in paths)
|
|
self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"),
|
|
"http://www.example.com/a%A3do"),
|
|
# normalize percent-encoding case (in query arguments)
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?k=b%a3"),
|
|
"http://www.example.com/do?k=b%A3")
|
|
|
|
# non-ASCII percent-encoding in paths
|
|
self.assertEqual(canonicalize_url("http://www.example.com/a do?a=1"),
|
|
"http://www.example.com/a%20do?a=1"),
|
|
self.assertEqual(canonicalize_url("http://www.example.com/a %20do?a=1"),
|
|
"http://www.example.com/a%20%20do?a=1"),
|
|
self.assertEqual(canonicalize_url("http://www.example.com/a do\xc2\xa3.html?a=1"),
|
|
"http://www.example.com/a%20do%C2%A3.html?a=1")
|
|
# non-ASCII percent-encoding in query arguments
|
|
self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=\xa3500&a=5&z=3"),
|
|
u"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"),
|
|
"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
|
|
self.assertEqual(canonicalize_url("http://www.example.com/do?price(\xc2\xa3)=500&a=1"),
|
|
"http://www.example.com/do?a=1&price%28%C2%A3%29=500")
|
|
|
|
# urls containing auth and ports
|
|
self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com:81/do?now=1"),
|
|
u"http://user:pass@www.example.com:81/do?now=1")
|
|
|
|
# remove fragments
|
|
self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag"),
|
|
u"http://user:pass@www.example.com/do?a=1")
|
|
self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", keep_fragments=True),
|
|
u"http://user:pass@www.example.com/do?a=1#frag")
|
|
|
|
# urllib.quote uses a mapping cache of encoded characters. when parsing
|
|
# an already percent-encoded url, it will fail if that url was not
|
|
# percent-encoded as utf-8, that's why canonicalize_url must always
|
|
# convert the urls to string. the following test asserts that
|
|
# functionality.
|
|
self.assertEqual(canonicalize_url(u'http://www.example.com/caf%E9-con-leche.htm'),
|
|
'http://www.example.com/caf%E9-con-leche.htm')
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|