diff --git a/scrapy/link.py b/scrapy/link.py
index 8bdcce761..2c8301680 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,20 +4,26 @@ This module defines the Link object used in Link extractors.
For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
-
+import warnings
import six
+from scrapy.utils.python import to_bytes
+
+
class Link(object):
"""Link objects represent an extracted link by the LinkExtractor."""
__slots__ = ['url', 'text', 'fragment', 'nofollow']
def __init__(self, url, text='', fragment='', nofollow=False):
- if isinstance(url, six.text_type):
- import warnings
- warnings.warn("Do not instantiate Link objects with unicode urls. "
- "Assuming utf-8 encoding (which could be wrong)")
- url = url.encode('utf-8')
+ if not isinstance(url, str):
+ if six.PY2:
+ warnings.warn("Link urls must be str objects. "
+ "Assuming utf-8 encoding (which could be wrong)")
+ url = to_bytes(url, encoding='utf8')
+ else:
+ got = url.__class__.__name__
+ raise TypeError("Link urls must be str objects, got %s" % got)
self.url = url
self.text = text
self.fragment = fragment
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index bb799e572..64efa0c55 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -39,7 +39,7 @@ IGNORED_EXTENSIONS = [
_re_type = type(re.compile("", 0))
_matches = lambda url, regexs: any((r.search(url) for r in regexs))
-_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])
+_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
class FilteringLinkExtractor(object):
@@ -51,8 +51,10 @@ class FilteringLinkExtractor(object):
self.link_extractor = link_extractor
- self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
- self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
+ self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
+ for x in arg_to_iter(allow)]
+ self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
+ for x in arg_to_iter(deny)]
self.allow_domains = set(arg_to_iter(allow_domains))
self.deny_domains = set(arg_to_iter(deny_domains))
@@ -64,7 +66,7 @@ class FilteringLinkExtractor(object):
self.canonicalize = canonicalize
if deny_extensions is None:
deny_extensions = IGNORED_EXTENSIONS
- self.deny_extensions = set(['.' + e for e in arg_to_iter(deny_extensions)])
+ self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
def _link_allowed(self, link):
if not _is_valid_url(link.url):
@@ -104,5 +106,6 @@ class FilteringLinkExtractor(object):
def _extract_links(self, *args, **kwargs):
return self.link_extractor._extract_links(*args, **kwargs)
+
# Top-level imports
from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 7064e886d..71d57b392 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,16 +1,14 @@
"""
Link extractor based on lxml.html
"""
-
-import re
+import six
from six.moves.urllib.parse import urlparse, urljoin
import lxml.etree as etree
-from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list
+from scrapy.utils.python import unique as unique_list, to_native_str
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.response import get_base_url
@@ -20,8 +18,9 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
_collect_string_content = etree.XPath("string()")
+
def _nons(tag):
- if isinstance(tag, basestring):
+ if isinstance(tag, six.string_types):
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
return tag.split('}')[-1]
return tag
@@ -57,16 +56,13 @@ class LxmlParserLinkExtractor(object):
url = self.process_attr(attr_val)
if url is None:
continue
- if isinstance(url, unicode):
- url = url.encode(response_encoding)
+ url = to_native_str(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',
nofollow=rel_has_nofollow(el.get('rel')))
links.append(link)
-
- return unique_list(links, key=lambda link: link.url) \
- if self.unique else links
+ return self._deduplicate_if_needed(links)
def extract_links(self, response):
base_url = get_base_url(response)
@@ -77,7 +73,11 @@ class LxmlParserLinkExtractor(object):
The subclass should override it if neccessary
"""
- links = unique_list(links, key=lambda link: link.url) if self.unique else links
+ return self._deduplicate_if_needed(links)
+
+ def _deduplicate_if_needed(self, links):
+ if self.unique:
+ return unique_list(links, key=lambda link: link.url)
return links
@@ -110,4 +110,3 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
links = self._extract_links(doc, response.url, response.encoding, base_url)
all_links.extend(self._process_links(links))
return unique_list(all_links)
-
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 795b60eb4..e2f039146 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -35,14 +35,18 @@ class DepthMiddleware(object):
if self.prio:
request.priority -= depth * self.prio
if self.maxdepth and depth > self.maxdepth:
- logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
- {'maxdepth': self.maxdepth, 'requrl': request.url},
- extra={'spider': spider})
+ logger.debug(
+ "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+ {'maxdepth': self.maxdepth, 'requrl': request.url},
+ extra={'spider': spider}
+ )
return False
elif self.stats:
if self.verbose_stats:
- self.stats.inc_value('request_depth_count/%s' % depth, spider=spider)
- self.stats.max_value('request_depth_max', depth, spider=spider)
+ self.stats.inc_value('request_depth_count/%s' % depth,
+ spider=spider)
+ self.stats.max_value('request_depth_max', depth,
+ spider=spider)
return True
# base case (depth=0)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index a90f9f1e0..ea1c9270f 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
logger = logging.getLogger(__name__)
+
class OffsiteMiddleware(object):
def __init__(self, stats):
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 77551753e..031f649d6 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,14 +6,17 @@ See documentation in docs/topics/spiders.rst
"""
import copy
+import six
from scrapy.http import Request, HtmlResponse
from scrapy.utils.spider import iterate_spider_output
from scrapy.spiders import Spider
+
def identity(x):
return x
+
class Rule(object):
def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
@@ -27,6 +30,7 @@ class Rule(object):
else:
self.follow = follow
+
class CrawlSpider(Spider):
rules = ()
@@ -49,7 +53,8 @@ class CrawlSpider(Spider):
return
seen = set()
for n, rule in enumerate(self._rules):
- links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
+ links = [lnk for lnk in rule.link_extractor.extract_links(response)
+ if lnk not in seen]
if links and rule.process_links:
links = rule.process_links(links)
for link in links:
@@ -77,7 +82,7 @@ class CrawlSpider(Spider):
def get_method(method):
if callable(method):
return method
- elif isinstance(method, basestring):
+ elif isinstance(method, six.string_types):
return getattr(self, method, None)
self._rules = [copy.copy(r) for r in self.rules]
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index 7717c8819..2efb1a869 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,6 +1,7 @@
from scrapy.spiders import Spider
from scrapy.utils.spider import iterate_spider_output
+
class InitSpider(Spider):
"""Base Spider with initialization facilities"""
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 5aa0b944d..eede467a8 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,5 +1,6 @@
import re
import logging
+import six
from scrapy.spiders import Spider
from scrapy.http import Request, XmlResponse
@@ -20,13 +21,14 @@ class SitemapSpider(Spider):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
- if isinstance(c, basestring):
+ if isinstance(c, six.string_types):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
def start_requests(self):
- return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls)
+ for url in self.sitemap_urls:
+ yield Request(url, self._parse_sitemap)
def _parse_sitemap(self, response):
if response.url.endswith('/robots.txt'):
@@ -52,8 +54,8 @@ class SitemapSpider(Spider):
break
def _get_sitemap_body(self, response):
- """Return the sitemap body contained in the given response, or None if the
- response is not a sitemap.
+ """Return the sitemap body contained in the given response,
+ or None if the response is not a sitemap.
"""
if isinstance(response, XmlResponse):
return response.body
@@ -64,11 +66,13 @@ class SitemapSpider(Spider):
elif response.url.endswith('.xml.gz'):
return gunzip(response.body)
+
def regex(x):
- if isinstance(x, basestring):
+ if isinstance(x, six.string_types):
return re.compile(x)
return x
+
def iterloc(it, alt=False):
for d in it:
yield d['loc']
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 741948359..7fa4bba57 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -7,6 +7,7 @@ except ImportError:
from gzip import GzipFile
+
def gunzip(data):
"""Gunzip the given data and return as much data as possible.
@@ -31,7 +32,8 @@ def gunzip(data):
raise
return output
+
def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
- ctype = response.headers.get('Content-Type', '')
- return ctype in ('application/x-gzip', 'application/gzip')
+ ctype = response.headers.get('Content-Type', b'')
+ return ctype in (b'application/x-gzip', b'application/gzip')
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index f380e6679..759eeffff 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -3,9 +3,8 @@ tests/test_command_fetch.py
tests/test_command_shell.py
tests/test_commands.py
tests/test_exporters.py
-tests/test_linkextractors.py
+tests/test_linkextractors_deprecated.py
tests/test_crawl.py
-tests/test_crawler.py
tests/test_downloader_handlers.py
tests/test_downloadermiddleware_ajaxcrawlable.py
tests/test_downloadermiddleware_defaultheaders.py
@@ -24,11 +23,7 @@ tests/test_mail.py
tests/test_pipeline_files.py
tests/test_pipeline_images.py
tests/test_proxy_connect.py
-tests/test_spidermiddleware_depth.py
tests/test_spidermiddleware_httperror.py
-tests/test_spidermiddleware_offsite.py
-tests/test_spidermiddleware_referer.py
-tests/test_spider.py
tests/test_utils_iterators.py
tests/test_utils_template.py
tests/test_webclient.py
diff --git a/tests/test_link.py b/tests/test_link.py
index 0b79e47cd..955430b37 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,8 +1,10 @@
import unittest
import warnings
+import six
from scrapy.link import Link
+
class LinkTest(unittest.TestCase):
def _assert_same_links(self, link1, link2):
@@ -14,38 +16,42 @@ class LinkTest(unittest.TestCase):
self.assertNotEqual(hash(link1), hash(link2))
def test_eq_and_hash(self):
- l1 = Link(b"http://www.example.com")
- l2 = Link(b"http://www.example.com/other")
- l3 = Link(b"http://www.example.com")
+ l1 = Link("http://www.example.com")
+ l2 = Link("http://www.example.com/other")
+ l3 = Link("http://www.example.com")
self._assert_same_links(l1, l1)
self._assert_different_links(l1, l2)
self._assert_same_links(l1, l3)
- l4 = Link(b"http://www.example.com", text="test")
- l5 = Link(b"http://www.example.com", text="test2")
- l6 = Link(b"http://www.example.com", text="test")
+ l4 = Link("http://www.example.com", text="test")
+ l5 = Link("http://www.example.com", text="test2")
+ l6 = Link("http://www.example.com", text="test")
self._assert_same_links(l4, l4)
self._assert_different_links(l4, l5)
self._assert_same_links(l4, l6)
- l7 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
- l8 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False)
- l9 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
- l10 = Link(b"http://www.example.com", text="test", fragment='other', nofollow=False)
+ l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
+ l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False)
+ l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
+ l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False)
self._assert_same_links(l7, l8)
self._assert_different_links(l7, l9)
self._assert_different_links(l7, l10)
def test_repr(self):
- l1 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True)
+ l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True)
l2 = eval(repr(l1))
self._assert_same_links(l1, l2)
- def test_unicode_url(self):
- with warnings.catch_warnings(record=True) as w:
- link = Link(u"http://www.example.com/\xa3")
- self.assertIsInstance(link.url, bytes)
- self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
+ def test_non_str_url_py2(self):
+ if six.PY2:
+ with warnings.catch_warnings(record=True) as w:
+ link = Link(u"http://www.example.com/\xa3")
+ self.assertIsInstance(link.url, str)
+ self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
assert len(w) == 1, "warning not issued"
+ else:
+ with self.assertRaises(TypeError):
+ Link(b"http://www.example.com/\xc2\xa3")
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d32ff2d55..129336d14 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,200 +1,361 @@
import re
import unittest
-from scrapy.linkextractors.regex import RegexLinkExtractor
+
+import pytest
+
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.link import Link
-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
from tests import get_testdata
-class LinkExtractorTestCase(unittest.TestCase):
- def test_basic(self):
- html = """Page title<title>
- <body><p><a href="item/12.html">Item 12</a></p>
- <p><a href="/about.html">About us</a></p>
- <img src="/logo.png" alt="Company logo (not a link)" />
- <p><a href="../othercat.html">Other category</a></p>
- <p><a href="/">>></a></p>
- <p><a href="/" /></p>
- </body></html>"""
- response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+# a hack to skip base class tests in pytest
+class Base:
+ class LinkExtractorTestCase(unittest.TestCase):
+ extractor_cls = None
- lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
- Link(url='http://example.org/about.html', text='About us'),
- Link(url='http://example.org/othercat.html', text='Other category'),
- Link(url='http://example.org/', text='>>'),
- Link(url='http://example.org/', text='')])
+ def setUp(self):
+ body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+ self.response = HtmlResponse(url='http://example.com/index', body=body)
- def test_base_url(self):
- html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
- <body><p><a href="item/12.html">Item 12</a></p>
- </body></html>"""
- response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+ def test_urls_type(self):
+ ''' Test that the resulting urls are str objects '''
+ lx = self.extractor_cls()
+ self.assertTrue(all(isinstance(link.url, str)
+ for link in lx.extract_links(self.response)))
- lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+ def test_extract_all_links(self):
+ lx = self.extractor_cls()
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://www.google.com/something', text=u''),
+ Link(url='http://example.com/innertag.html', text=u'inner tag'),
+ ])
- # base url is an absolute path and relative to host
- html = """<html><head><title>Page title<title><base href="/" />
- <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
- response = HtmlResponse("https://example.org/somepage/index.html", body=html)
- self.assertEqual(lx.extract_links(response),
- [Link(url='https://example.org/item/12.html', text='Item 12')])
+ def test_extract_filter_allow(self):
+ lx = self.extractor_cls(allow=('sample', ))
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ ])
- # base url has no scheme
- html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
- <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
- response = HtmlResponse("https://example.org/somepage/index.html", body=html)
- self.assertEqual(lx.extract_links(response),
- [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
+ def test_extract_filter_allow_with_duplicates(self):
+ lx = self.extractor_cls(allow=('sample', ), unique=False)
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+ ])
- def test_link_text_wrong_encoding(self):
- html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
- response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
- lx = BaseSgmlLinkExtractor()
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
- ])
+ def test_extract_filter_allow_and_deny(self):
+ lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ ])
- def test_extraction_encoding(self):
- body = get_testdata('link_extractor', 'linkextractor_noenc.html')
- response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
- response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
- body = get_testdata('link_extractor', 'linkextractor_latin1.html')
- response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
+ def test_extract_filter_allowed_domains(self):
+ lx = self.extractor_cls(allow_domains=('google.com', ))
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://www.google.com/something', text=u''),
+ ])
- lx = BaseSgmlLinkExtractor()
- self.assertEqual(lx.extract_links(response_utf8), [
- Link(url='http://example.com/sample_%C3%B1.html', text=''),
- Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
- ])
+ def test_extraction_using_single_values(self):
+ '''Test the extractor's behaviour among different situations'''
- self.assertEqual(lx.extract_links(response_noenc), [
- Link(url='http://example.com/sample_%C3%B1.html', text=''),
- Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
- ])
+ lx = self.extractor_cls(allow='sample')
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ ])
- self.assertEqual(lx.extract_links(response_latin1), [
- Link(url='http://example.com/sample_%F1.html', text=''),
- Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
- ])
+ lx = self.extractor_cls(allow='sample', deny='3')
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ ])
- def test_matches(self):
- url1 = 'http://lotsofstuff.com/stuff1/index'
- url2 = 'http://evenmorestuff.com/uglystuff/index'
+ lx = self.extractor_cls(allow_domains='google.com')
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://www.google.com/something', text=u''),
+ ])
- lx = BaseSgmlLinkExtractor()
- self.assertEqual(lx.matches(url1), True)
- self.assertEqual(lx.matches(url2), True)
+ lx = self.extractor_cls(deny_domains='example.com')
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://www.google.com/something', text=u''),
+ ])
- def test_link_nofollow(self):
- html = """
- <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
- <a href="about.html">About us</a>
- <a href="http://google.com/something" rel="external nofollow">Something</a>
- """
- response = HtmlResponse("http://example.org/page.html", body=html)
- lx = SgmlLinkExtractor()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
- Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
- Link(url='http://google.com/something', text=u'Something', nofollow=True),
- ])
+ def test_nofollow(self):
+ '''Test the extractor's behaviour for links with rel="nofollow"'''
+ html = b"""<html><head><title>Page title<title>
+ <body>
+ <div class='links'>
+ <p><a href="/about.html">About us</a></p>
+ </div>
+ <div>
+ <p><a href="/follow.html">Follow this link</a></p>
+ </div>
+ <div>
+ <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>
+ </div>
+ <div>
+ <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
+ </div>
+ <div>
+ <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>
+ </div>
+ </body></html>"""
+ response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-class SgmlLinkExtractorTestCase(unittest.TestCase):
- extractor_cls = SgmlLinkExtractor
+ lx = self.extractor_cls()
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.org/about.html', text=u'About us'),
+ Link(url='http://example.org/follow.html', text=u'Follow this link'),
+ Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
+ Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
+ Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+ ])
- def setUp(self):
- body = get_testdata('link_extractor', 'sgml_linkextractor.html')
- self.response = HtmlResponse(url='http://example.com/index', body=body)
+ def test_matches(self):
+ url1 = 'http://lotsofstuff.com/stuff1/index'
+ url2 = 'http://evenmorestuff.com/uglystuff/index'
- def test_urls_type(self):
- '''Test that the resulting urls are regular strings and not a unicode objects'''
- lx = self.extractor_cls()
- self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response)))
+ lx = self.extractor_cls(allow=(r'stuff1', ))
+ self.assertEqual(lx.matches(url1), True)
+ self.assertEqual(lx.matches(url2), False)
- def test_extraction(self):
- '''Test the extractor's behaviour among different situations'''
+ lx = self.extractor_cls(deny=(r'uglystuff', ))
+ self.assertEqual(lx.matches(url1), True)
+ self.assertEqual(lx.matches(url2), False)
- lx = self.extractor_cls()
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- Link(url='http://www.google.com/something', text=u''),
- Link(url='http://example.com/innertag.html', text=u'inner tag'),
- ])
+ lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
+ self.assertEqual(lx.matches(url1), False)
+ self.assertEqual(lx.matches(url2), True)
- lx = self.extractor_cls(allow=('sample', ))
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- ])
+ lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
+ self.assertEqual(lx.matches(url1), False)
+ self.assertEqual(lx.matches(url2), True)
- lx = self.extractor_cls(allow=('sample', ), unique=False)
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
- ])
+ lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
+ allow_domains=('blah1.com',),
+ deny_domains=('blah2.com',))
+ self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
+ self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
+ self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
+ self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
- lx = self.extractor_cls(allow=('sample', ))
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- ])
+ def test_restrict_xpaths(self):
+ lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ ])
- lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- ])
+ def test_restrict_xpaths_encoding(self):
+ """Test restrict_xpaths with encodings"""
+ html = b"""<html><head><title>Page title<title>
+ <body><p><a href="item/12.html">Item 12</a></p>
+ <div class='links'>
+ <p><a href="/about.html">About us\xa3</a></p>
+ </div>
+ <div>
+ <p><a href="/nofollow.html">This shouldn't be followed</a></p>
+ </div>
+ </body></html>"""
+ response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
- lx = self.extractor_cls(allow_domains=('google.com', ))
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://www.google.com/something', text=u''),
- ])
+ lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://example.org/about.html', text=u'About us\xa3')])
- def test_extraction_using_single_values(self):
- '''Test the extractor's behaviour among different situations'''
+ def test_restrict_xpaths_with_html_entities(self):
+ html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'
+ response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
+ links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
+ self.assertEqual(links,
+ [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
- lx = self.extractor_cls(allow='sample')
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- ])
+ def test_restrict_xpaths_concat_in_handle_data(self):
+ """html entities cause SGMLParser to call handle_data hook twice"""
+ body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>"""
+ response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
+ lx = self.extractor_cls(restrict_xpaths="//div")
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
+ fragment='', nofollow=False)])
- lx = self.extractor_cls(allow='sample', deny='3')
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- ])
+ def test_restrict_css(self):
+ lx = self.extractor_cls(restrict_css=('#subwrapper a',))
+ self.assertEqual(lx.extract_links(self.response), [
+ Link(url='http://example.com/sample2.html', text=u'sample 2')
+ ])
- lx = self.extractor_cls(allow_domains='google.com')
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://www.google.com/something', text=u''),
- ])
+ def test_restrict_css_and_restrict_xpaths_together(self):
+ lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
+ restrict_css=('#subwrapper + a', ))
+ self.assertEqual([link for link in lx.extract_links(self.response)], [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ ])
- lx = self.extractor_cls(deny_domains='example.com')
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://www.google.com/something', text=u''),
- ])
+ def test_area_tag_with_unicode_present(self):
+ body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
+ response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
+ lx = self.extractor_cls()
+ lx.extract_links(response)
+ lx.extract_links(response)
+ lx.extract_links(response)
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://example.org/foo', text=u'',
+ fragment='', nofollow=False)])
- def test_nofollow(self):
- '''Test the extractor's behaviour for links with rel="nofollow"'''
+ def test_encoded_url(self):
+ body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
+ response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+ lx = self.extractor_cls()
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+ ])
- html = """<html><head><title>Page title<title>
- <body>
+ def test_encoded_url_in_restricted_xpath(self):
+ body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""
+ response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+ lx = self.extractor_cls(restrict_xpaths="//div")
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+ ])
+
+ def test_ignored_extensions(self):
+ # jpg is ignored by default
+ html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">"""
+ response = HtmlResponse("http://example.org/", body=html)
+ lx = self.extractor_cls()
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.org/page.html', text=u'asd'),
+ ])
+
+ # override denied extensions
+ lx = self.extractor_cls(deny_extensions=['html'])
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.org/photo.jpg'),
+ ])
+
+ def test_process_value(self):
+ """Test restrict_xpaths with encodings"""
+ html = b"""
+ <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
+ <a href="/about.html">About us</a>
+ """
+ response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
+
+ def process_value(value):
+ m = re.search("javascript:goToPage\('(.*?)'", value)
+ if m:
+ return m.group(1)
+
+ lx = self.extractor_cls(process_value=process_value)
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://example.org/other/page.html', text='Link text')])
+
+ def test_base_url_with_restrict_xpaths(self):
+ html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
+ <body><p><a href="item/12.html">Item 12</a></p>
+ </body></html>"""
+ response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+ lx = self.extractor_cls(restrict_xpaths="//p")
+ self.assertEqual(lx.extract_links(response),
+ [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+
+ def test_attrs(self):
+ lx = self.extractor_cls(attrs="href")
+ self.assertEqual(lx.extract_links(self.response), [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://www.google.com/something', text=u''),
+ Link(url='http://example.com/innertag.html', text=u'inner tag'),
+ ])
+
+ lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
+ self.assertEqual(lx.extract_links(self.response), [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample2.jpg', text=u''),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://www.google.com/something', text=u''),
+ Link(url='http://example.com/innertag.html', text=u'inner tag'),
+ ])
+
+ lx = self.extractor_cls(attrs=None)
+ self.assertEqual(lx.extract_links(self.response), [])
+
+ def test_tags(self):
+ html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
+ response = HtmlResponse("http://example.com/index.html", body=html)
+
+ lx = self.extractor_cls(tags=None)
+ self.assertEqual(lx.extract_links(response), [])
+
+ lx = self.extractor_cls()
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ ])
+
+ lx = self.extractor_cls(tags="area")
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.com/sample1.html', text=u''),
+ ])
+
+ lx = self.extractor_cls(tags="a")
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ ])
+
+ lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample2.jpg', text=u''),
+ ])
+
+ def test_tags_attrs(self):
+ html = b"""
+ <html><body>
+ <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>
+ <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>
+ </body></html>
+ """
+ response = HtmlResponse("http://example.com/index.html", body=html)
+
+ lx = self.extractor_cls(tags='div', attrs='data-url')
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
+ Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+ ])
+
+ lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
+ self.assertEqual(lx.extract_links(response), [
+ Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
+ Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+ ])
+
+ def test_xhtml(self):
+ xhtml = b"""
+ <?xml version="1.0"?>
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <title>XHTML document title
+
+
@@ -210,317 +371,51 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
- """
- response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+
+