mirror of https://github.com/scrapy/scrapy.git
PY3 port LinkExtractor
* tests for other link extractors are moved to test_linkextractors_deprecated.py * in Python 3 Link is converted to use native strings for urls * minor cleanups
This commit is contained in:
Mikhail Korobov
parent
f46a450080
commit
f2edbd05de
|
|
@ -4,8 +4,8 @@ This module defines the Link object used in Link extractors.
|
|||
For actual link extractors implementation see scrapy.linkextractors, or
|
||||
its documentation in: docs/topics/link-extractors.rst
|
||||
"""
|
||||
from scrapy.utils.python import to_native_str
|
||||
|
||||
import six
|
||||
|
||||
class Link(object):
|
||||
"""Link objects represent an extracted link by the LinkExtractor."""
|
||||
|
|
@ -13,11 +13,10 @@ class Link(object):
|
|||
__slots__ = ['url', 'text', 'fragment', 'nofollow']
|
||||
|
||||
def __init__(self, url, text='', fragment='', nofollow=False):
|
||||
if isinstance(url, six.text_type):
|
||||
if not isinstance(url, str):
|
||||
import warnings
|
||||
warnings.warn("Do not instantiate Link objects with unicode urls. "
|
||||
"Assuming utf-8 encoding (which could be wrong)")
|
||||
url = url.encode('utf-8')
|
||||
warnings.warn("Link urls must be str objects.")
|
||||
url = to_native_str(url)
|
||||
self.url = url
|
||||
self.text = text
|
||||
self.fragment = fragment
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ IGNORED_EXTENSIONS = [
|
|||
|
||||
_re_type = type(re.compile("", 0))
|
||||
_matches = lambda url, regexs: any((r.search(url) for r in regexs))
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
|
||||
|
||||
|
||||
class FilteringLinkExtractor(object):
|
||||
|
|
@ -51,8 +51,10 @@ class FilteringLinkExtractor(object):
|
|||
|
||||
self.link_extractor = link_extractor
|
||||
|
||||
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
|
||||
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
|
||||
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
|
||||
for x in arg_to_iter(allow)]
|
||||
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
|
||||
for x in arg_to_iter(deny)]
|
||||
|
||||
self.allow_domains = set(arg_to_iter(allow_domains))
|
||||
self.deny_domains = set(arg_to_iter(deny_domains))
|
||||
|
|
@ -64,7 +66,7 @@ class FilteringLinkExtractor(object):
|
|||
self.canonicalize = canonicalize
|
||||
if deny_extensions is None:
|
||||
deny_extensions = IGNORED_EXTENSIONS
|
||||
self.deny_extensions = set(['.' + e for e in arg_to_iter(deny_extensions)])
|
||||
self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
|
||||
|
||||
def _link_allowed(self, link):
|
||||
if not _is_valid_url(link.url):
|
||||
|
|
@ -104,5 +106,6 @@ class FilteringLinkExtractor(object):
|
|||
def _extract_links(self, *args, **kwargs):
|
||||
return self.link_extractor._extract_links(*args, **kwargs)
|
||||
|
||||
|
||||
# Top-level imports
|
||||
from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
|
||||
|
|
|
|||
|
|
@ -1,16 +1,14 @@
|
|||
"""
|
||||
Link extractor based on lxml.html
|
||||
"""
|
||||
|
||||
import re
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse, urljoin
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
from scrapy.utils.python import unique as unique_list, to_native_str
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
from scrapy.utils.response import get_base_url
|
||||
|
||||
|
|
@ -20,8 +18,9 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
|
|||
|
||||
_collect_string_content = etree.XPath("string()")
|
||||
|
||||
|
||||
def _nons(tag):
|
||||
if isinstance(tag, basestring):
|
||||
if isinstance(tag, six.string_types):
|
||||
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
|
||||
return tag.split('}')[-1]
|
||||
return tag
|
||||
|
|
@ -57,16 +56,13 @@ class LxmlParserLinkExtractor(object):
|
|||
url = self.process_attr(attr_val)
|
||||
if url is None:
|
||||
continue
|
||||
if isinstance(url, unicode):
|
||||
url = url.encode(response_encoding)
|
||||
url = to_native_str(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
nofollow=True if el.get('rel') == 'nofollow' else False)
|
||||
links.append(link)
|
||||
|
||||
return unique_list(links, key=lambda link: link.url) \
|
||||
if self.unique else links
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
||||
def extract_links(self, response):
|
||||
base_url = get_base_url(response)
|
||||
|
|
@ -77,7 +73,11 @@ class LxmlParserLinkExtractor(object):
|
|||
|
||||
The subclass should override it if neccessary
|
||||
"""
|
||||
links = unique_list(links, key=lambda link: link.url) if self.unique else links
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
||||
def _deduplicate_if_needed(self, links):
|
||||
if self.unique:
|
||||
return unique_list(links, key=lambda link: link.url)
|
||||
return links
|
||||
|
||||
|
||||
|
|
@ -110,4 +110,3 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
links = self._extract_links(doc, response.url, response.encoding, base_url)
|
||||
all_links.extend(self._process_links(links))
|
||||
return unique_list(all_links)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ tests/test_command_fetch.py
|
|||
tests/test_command_shell.py
|
||||
tests/test_commands.py
|
||||
tests/test_exporters.py
|
||||
tests/test_linkextractors.py
|
||||
tests/test_linkextractors_deprecated.py
|
||||
tests/test_crawl.py
|
||||
tests/test_crawler.py
|
||||
tests/test_downloader_handlers.py
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import unittest
|
||||
import warnings
|
||||
import six
|
||||
|
||||
from scrapy.link import Link
|
||||
|
||||
|
||||
class LinkTest(unittest.TestCase):
|
||||
|
||||
def _assert_same_links(self, link1, link2):
|
||||
|
|
@ -43,9 +45,15 @@ class LinkTest(unittest.TestCase):
|
|||
l2 = eval(repr(l1))
|
||||
self._assert_same_links(l1, l2)
|
||||
|
||||
def test_unicode_url(self):
|
||||
def test_non_str_url(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
link = Link(u"http://www.example.com/\xa3")
|
||||
self.assertIsInstance(link.url, bytes)
|
||||
self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
|
||||
assert len(w) == 1, "warning not issued"
|
||||
if six.PY2:
|
||||
link = Link(u"http://www.example.com/\xa3")
|
||||
self.assertIsInstance(link.url, str)
|
||||
self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
|
||||
else:
|
||||
link = Link(b"http://www.example.com/\xc2\xa3")
|
||||
self.assertIsInstance(link.url, str)
|
||||
self.assertEqual(link.url, u'http://www.example.com/\xa3')
|
||||
|
||||
assert len(w) == 1, "warning not issued"
|
||||
|
|
|
|||
|
|
@ -3,190 +3,360 @@ import unittest
|
|||
|
||||
import pytest
|
||||
|
||||
from scrapy.linkextractors.regex import RegexLinkExtractor
|
||||
from scrapy.http import HtmlResponse, XmlResponse
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
|
||||
from tests import get_testdata
|
||||
|
||||
|
||||
class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
|
||||
def test_basic(self):
|
||||
html = """<html><head><title>Page title<title>
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
<p><a href="/about.html">About us</a></p>
|
||||
<img src="/logo.png" alt="Company logo (not a link)" />
|
||||
<p><a href="../othercat.html">Other category</a></p>
|
||||
<p><a href="/">>></a></p>
|
||||
<p><a href="/" /></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
# a hack to skip base class tests in pytest
|
||||
class Base:
|
||||
class LinkExtractorTestCase(unittest.TestCase):
|
||||
extractor_cls = None
|
||||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
|
||||
Link(url='http://example.org/about.html', text='About us'),
|
||||
Link(url='http://example.org/othercat.html', text='Other category'),
|
||||
Link(url='http://example.org/', text='>>'),
|
||||
Link(url='http://example.org/', text='')])
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_base_url(self):
|
||||
html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
def test_urls_type(self):
|
||||
''' Test that the resulting urls are str objects '''
|
||||
lx = self.extractor_cls()
|
||||
self.assertTrue(all(isinstance(link.url, str)
|
||||
for link in lx.extract_links(self.response)))
|
||||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
def test_extraction(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
|
||||
# base url is an absolute path and relative to host
|
||||
html = """<html><head><title>Page title<title><base href="/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
|
||||
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='https://example.org/item/12.html', text='Item 12')])
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
])
|
||||
|
||||
# base url has no scheme
|
||||
html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
|
||||
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
|
||||
lx = self.extractor_cls(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
def test_link_text_wrong_encoding(self):
|
||||
html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
|
||||
response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
|
||||
])
|
||||
lx = self.extractor_cls(allow=('sample', ), unique=False)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
])
|
||||
|
||||
def test_extraction_encoding(self):
|
||||
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
|
||||
response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
|
||||
response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
|
||||
body = get_testdata('link_extractor', 'linkextractor_latin1.html')
|
||||
response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
|
||||
lx = self.extractor_cls(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response_utf8), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_noenc), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
lx = self.extractor_cls(allow_domains=('google.com', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_latin1), [
|
||||
Link(url='http://example.com/sample_%F1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
|
||||
])
|
||||
def test_extraction_using_single_values(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
lx = self.extractor_cls(allow='sample')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
lx = self.extractor_cls(allow='sample', deny='3')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(allow_domains='google.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
class BaseLinkExtractorTestCase(unittest.TestCase):
|
||||
extractor_cls = None
|
||||
lx = self.extractor_cls(deny_domains='example.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
def setUp(self):
|
||||
if self.extractor_cls is None:
|
||||
raise unittest.SkipTest()
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
def test_nofollow(self):
|
||||
'''Test the extractor's behaviour for links with rel="nofollow"'''
|
||||
|
||||
def test_urls_type(self):
|
||||
''' Test that the resulting urls are str objects '''
|
||||
lx = self.extractor_cls()
|
||||
self.assertTrue(all(isinstance(link.url, str)
|
||||
for link in lx.extract_links(self.response)))
|
||||
html = b"""<html><head><title>Page title<title>
|
||||
<body>
|
||||
<div class='links'>
|
||||
<p><a href="/about.html">About us</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/follow.html">Follow this link</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
|
||||
</div>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
|
||||
def test_extraction(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/about.html', text=u'About us'),
|
||||
Link(url='http://example.org/follow.html', text=u'Follow this link'),
|
||||
Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
|
||||
Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
])
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
|
||||
lx = self.extractor_cls(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
lx = self.extractor_cls(allow=(r'stuff1', ))
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), False)
|
||||
|
||||
lx = self.extractor_cls(allow=('sample', ), unique=False)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
])
|
||||
lx = self.extractor_cls(deny=(r'uglystuff', ))
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), False)
|
||||
|
||||
lx = self.extractor_cls(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
|
||||
self.assertEqual(lx.matches(url1), False)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
|
||||
self.assertEqual(lx.matches(url1), False)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
lx = self.extractor_cls(allow_domains=('google.com', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
|
||||
allow_domains=('blah1.com',),
|
||||
deny_domains=('blah2.com',))
|
||||
self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
|
||||
self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
|
||||
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
|
||||
self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
|
||||
|
||||
def test_extraction_using_single_values(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
def test_restrict_xpaths(self):
|
||||
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(allow='sample')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
def test_restrict_xpaths_encoding(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
html = b"""<html><head><title>Page title<title>
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
<div class='links'>
|
||||
<p><a href="/about.html">About us\xa3</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/nofollow.html">This shouldn't be followed</a></p>
|
||||
</div>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
|
||||
|
||||
lx = self.extractor_cls(allow='sample', deny='3')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
|
||||
|
||||
lx = self.extractor_cls(allow_domains='google.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
def test_restrict_xpaths_with_html_entities(self):
|
||||
html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
|
||||
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
|
||||
self.assertEqual(links,
|
||||
[Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
|
||||
|
||||
lx = self.extractor_cls(deny_domains='example.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
def test_restrict_xpaths_concat_in_handle_data(self):
|
||||
"""html entities cause SGMLParser to call handle_data hook twice"""
|
||||
body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>"""
|
||||
response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
|
||||
lx = self.extractor_cls(restrict_xpaths="//div")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
|
||||
fragment='', nofollow=False)])
|
||||
|
||||
def test_nofollow(self):
|
||||
'''Test the extractor's behaviour for links with rel="nofollow"'''
|
||||
def test_restrict_css(self):
|
||||
lx = self.extractor_cls(restrict_css=('#subwrapper a',))
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2')
|
||||
])
|
||||
|
||||
html = """<html><head><title>Page title<title>
|
||||
<body>
|
||||
def test_restrict_css_and_restrict_xpaths_together(self):
|
||||
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
|
||||
restrict_css=('#subwrapper + a', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
def test_area_tag_with_unicode_present(self):
|
||||
body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
|
||||
response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
|
||||
lx = self.extractor_cls()
|
||||
lx.extract_links(response)
|
||||
lx.extract_links(response)
|
||||
lx.extract_links(response)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/foo', text=u'',
|
||||
fragment='', nofollow=False)])
|
||||
|
||||
def test_encoded_url(self):
|
||||
body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
|
||||
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_encoded_url_in_restricted_xpath(self):
|
||||
body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
|
||||
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls(restrict_xpaths="//div")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_ignored_extensions(self):
|
||||
# jpg is ignored by default
|
||||
html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">"""
|
||||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
])
|
||||
|
||||
# override denied extensions
|
||||
lx = self.extractor_cls(deny_extensions=['html'])
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/photo.jpg'),
|
||||
])
|
||||
|
||||
def test_process_value(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
html = b"""
|
||||
<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
|
||||
<a href="/about.html">About us</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
|
||||
|
||||
def process_value(value):
|
||||
m = re.search("javascript:goToPage\('(.*?)'", value)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
lx = self.extractor_cls(process_value=process_value)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/other/page.html', text='Link text')])
|
||||
|
||||
def test_base_url_with_restrict_xpaths(self):
|
||||
html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
lx = self.extractor_cls(restrict_xpaths="//p")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
def test_attrs(self):
|
||||
lx = self.extractor_cls(attrs="href")
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u''),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=None)
|
||||
self.assertEqual(lx.extract_links(self.response), [])
|
||||
|
||||
def test_tags(self):
|
||||
html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
|
||||
lx = self.extractor_cls(tags=None)
|
||||
self.assertEqual(lx.extract_links(response), [])
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags="area")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags="a")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u''),
|
||||
])
|
||||
|
||||
def test_tags_attrs(self):
|
||||
html = b"""
|
||||
<html><body>
|
||||
<div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>
|
||||
<div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>
|
||||
</body></html>
|
||||
"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
|
||||
lx = self.extractor_cls(tags='div', attrs='data-url')
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
|
||||
])
|
||||
|
||||
def test_xhtml(self):
|
||||
xhtml = b"""
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head>
|
||||
<title>XHTML document title</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class='links'>
|
||||
<p><a href="/about.html">About us</a></p>
|
||||
</div>
|
||||
|
|
@ -199,306 +369,49 @@ class BaseLinkExtractorTestCase(unittest.TestCase):
|
|||
<div>
|
||||
<p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
|
||||
</div>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/about.html', text=u'About us'),
|
||||
Link(url='http://example.org/follow.html', text=u'Follow this link'),
|
||||
Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
|
||||
Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
|
||||
])
|
||||
response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
|
||||
)
|
||||
|
||||
lx = self.extractor_cls(allow=(r'stuff1', ))
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), False)
|
||||
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls(deny=(r'uglystuff', ))
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), False)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
|
||||
)
|
||||
|
||||
lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
|
||||
self.assertEqual(lx.matches(url1), False)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
|
||||
self.assertEqual(lx.matches(url1), False)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
|
||||
allow_domains=('blah1.com',),
|
||||
deny_domains=('blah2.com',))
|
||||
self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
|
||||
self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
|
||||
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
|
||||
self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
|
||||
|
||||
def test_restrict_xpaths(self):
|
||||
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
def test_restrict_xpaths_encoding(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
html = """<html><head><title>Page title<title>
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
<div class='links'>
|
||||
<p><a href="/about.html">About us\xa3</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/nofollow.html">This shouldn't be followed</a></p>
|
||||
</div>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
|
||||
|
||||
lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
|
||||
|
||||
def test_restrict_xpaths_with_html_entities(self):
|
||||
html = '<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
|
||||
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
|
||||
self.assertEqual(links,
|
||||
[Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
|
||||
|
||||
def test_restrict_xpaths_concat_in_handle_data(self):
|
||||
"""html entities cause SGMLParser to call handle_data hook twice"""
|
||||
body = """<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>"""
|
||||
response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
|
||||
lx = self.extractor_cls(restrict_xpaths="//div")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
|
||||
fragment='', nofollow=False)])
|
||||
|
||||
def test_restrict_css(self):
|
||||
lx = self.extractor_cls(restrict_css=('#subwrapper a',))
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2')
|
||||
])
|
||||
|
||||
def test_restrict_css_and_restrict_xpaths_together(self):
|
||||
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
|
||||
restrict_css=('#subwrapper + a', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
def test_area_tag_with_unicode_present(self):
|
||||
body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
|
||||
response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
|
||||
lx = self.extractor_cls()
|
||||
lx.extract_links(response)
|
||||
lx.extract_links(response)
|
||||
lx.extract_links(response)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/foo', text=u'',
|
||||
fragment='', nofollow=False)])
|
||||
|
||||
def test_encoded_url(self):
|
||||
body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
|
||||
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_encoded_url_in_restricted_xpath(self):
|
||||
body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
|
||||
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls(restrict_xpaths="//div")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_ignored_extensions(self):
|
||||
# jpg is ignored by default
|
||||
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
|
||||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
])
|
||||
|
||||
# override denied extensions
|
||||
lx = self.extractor_cls(deny_extensions=['html'])
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/photo.jpg'),
|
||||
])
|
||||
|
||||
def test_process_value(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
html = """
|
||||
<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
|
||||
<a href="/about.html">About us</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
|
||||
|
||||
def process_value(value):
|
||||
m = re.search("javascript:goToPage\('(.*?)'", value)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
lx = self.extractor_cls(process_value=process_value)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/other/page.html', text='Link text')])
|
||||
|
||||
def test_base_url_with_restrict_xpaths(self):
|
||||
html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
lx = self.extractor_cls(restrict_xpaths="//p")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
def test_attrs(self):
|
||||
lx = self.extractor_cls(attrs="href")
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u''),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=None)
|
||||
self.assertEqual(lx.extract_links(self.response), [])
|
||||
|
||||
def test_tags(self):
|
||||
html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
|
||||
lx = self.extractor_cls(tags=None)
|
||||
self.assertEqual(lx.extract_links(response), [])
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags="area")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags="a")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u''),
|
||||
])
|
||||
|
||||
def test_tags_attrs(self):
|
||||
html = """
|
||||
<html><body>
|
||||
<div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>
|
||||
<div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>
|
||||
</body></html>
|
||||
"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
|
||||
lx = self.extractor_cls(tags='div', attrs='data-url')
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
|
||||
])
|
||||
|
||||
def test_xhtml(self):
|
||||
xhtml = """
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head>
|
||||
<title>XHTML document title</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class='links'>
|
||||
<p><a href="/about.html">About us</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/follow.html">Follow this link</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>
|
||||
</div>
|
||||
<div>
|
||||
<p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
|
||||
)
|
||||
|
||||
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
|
||||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
|
||||
)
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
def test_link_wrong_href(self):
|
||||
html = b"""
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
|
||||
class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
|
||||
extractor_cls = LxmlLinkExtractor
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
html = b"""
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
|
|
@ -514,94 +427,3 @@ class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
|
|||
def test_restrict_xpaths_with_html_entities(self):
|
||||
super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
|
||||
|
||||
|
||||
class HtmlParserLinkExtractorTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response),
|
||||
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class SgmlLinkExtractorTestCase(BaseLinkExtractorTestCase):
|
||||
extractor_cls = SgmlLinkExtractor
|
||||
|
||||
def test_deny_extensions(self):
|
||||
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
|
||||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = SgmlLinkExtractor(deny_extensions="jpg")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
])
|
||||
|
||||
def test_attrs_sgml(self):
|
||||
html = """<html><area href="sample1.html"></area>
|
||||
<a ref="sample2.html">sample text 2</a></html>"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
lx = SgmlLinkExtractor(attrs="href")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
])
|
||||
|
||||
def test_link_nofollow(self):
|
||||
html = """
|
||||
<a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
|
||||
<a href="about.html">About us</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/page.html", body=html)
|
||||
lx = SgmlLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
|
||||
Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class RegexLinkExtractorTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response),
|
||||
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,190 @@
|
|||
import unittest
|
||||
from scrapy.linkextractors.regex import RegexLinkExtractor
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
|
||||
from tests import get_testdata
|
||||
|
||||
from tests.test_linkextractors import Base
|
||||
|
||||
|
||||
class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
|
||||
# XXX: should we move some of these tests to base link extractor tests?
|
||||
|
||||
def test_basic(self):
|
||||
html = """<html><head><title>Page title<title>
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
<p><a href="/about.html">About us</a></p>
|
||||
<img src="/logo.png" alt="Company logo (not a link)" />
|
||||
<p><a href="../othercat.html">Other category</a></p>
|
||||
<p><a href="/">>></a></p>
|
||||
<p><a href="/" /></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
|
||||
Link(url='http://example.org/about.html', text='About us'),
|
||||
Link(url='http://example.org/othercat.html', text='Other category'),
|
||||
Link(url='http://example.org/', text='>>'),
|
||||
Link(url='http://example.org/', text='')])
|
||||
|
||||
def test_base_url(self):
|
||||
html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
# base url is an absolute path and relative to host
|
||||
html = """<html><head><title>Page title<title><base href="/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
|
||||
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='https://example.org/item/12.html', text='Item 12')])
|
||||
|
||||
# base url has no scheme
|
||||
html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
|
||||
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
|
||||
|
||||
def test_link_text_wrong_encoding(self):
|
||||
html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
|
||||
response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
|
||||
])
|
||||
|
||||
def test_extraction_encoding(self):
|
||||
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
|
||||
response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
|
||||
response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
|
||||
body = get_testdata('link_extractor', 'linkextractor_latin1.html')
|
||||
response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response_utf8), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_noenc), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_latin1), [
|
||||
Link(url='http://example.com/sample_%F1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
|
||||
])
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
|
||||
class HtmlParserLinkExtractorTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response),
|
||||
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
|
||||
extractor_cls = SgmlLinkExtractor
|
||||
|
||||
def test_deny_extensions(self):
|
||||
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
|
||||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = SgmlLinkExtractor(deny_extensions="jpg")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
])
|
||||
|
||||
def test_attrs_sgml(self):
|
||||
html = """<html><area href="sample1.html"></area>
|
||||
<a ref="sample2.html">sample text 2</a></html>"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
lx = SgmlLinkExtractor(attrs="href")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
])
|
||||
|
||||
def test_link_nofollow(self):
|
||||
html = """
|
||||
<a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
|
||||
<a href="about.html">About us</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/page.html", body=html)
|
||||
lx = SgmlLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
|
||||
Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class RegexLinkExtractorTestCase(unittest.TestCase):
|
||||
# XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
|
||||
# not to depend on SgmlLinkExractor. Its speed is also much worse
|
||||
# than it should be.
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response),
|
||||
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
Loading…
Reference in New Issue