diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index ceb37c5f1..1615d44d7 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,6 +1,8 @@
"""
Link extractor based on lxml.html
"""
+import operator
+from functools import partial
from urllib.parse import urljoin
import lxml.etree as etree
@@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
from scrapy.link import Link
+from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
-from scrapy.linkextractors import FilteringLinkExtractor
# from lxml/src/lxml/html/__init__.py
@@ -27,19 +29,24 @@ def _nons(tag):
return tag
+def _identity(x):
+ return x
+
+
+def _canonicalize_link_url(link):
+ return canonicalize_url(link.url, keep_fragments=True)
+
+
class LxmlParserLinkExtractor:
- def __init__(self, tag="a", attr="href", process=None, unique=False,
- strip=True, canonicalized=False):
- self.scan_tag = tag if callable(tag) else lambda t: t == tag
- self.scan_attr = attr if callable(attr) else lambda a: a == attr
- self.process_attr = process if callable(process) else lambda v: v
+ def __init__(
+ self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
+ ):
+ self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
+ self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
+ self.process_attr = process if callable(process) else _identity
self.unique = unique
self.strip = strip
- if canonicalized:
- self.link_key = lambda link: link.url
- else:
- self.link_key = lambda link: canonicalize_url(link.url,
- keep_fragments=True)
+ self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
def _iter_links(self, document):
for el in document.iter(etree.Element):
@@ -93,25 +100,44 @@ class LxmlParserLinkExtractor:
class LxmlLinkExtractor(FilteringLinkExtractor):
- def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
- tags=('a', 'area'), attrs=('href',), canonicalize=False,
- unique=True, process_value=None, deny_extensions=None, restrict_css=(),
- strip=True, restrict_text=None):
+ def __init__(
+ self,
+ allow=(),
+ deny=(),
+ allow_domains=(),
+ deny_domains=(),
+ restrict_xpaths=(),
+ tags=('a', 'area'),
+ attrs=('href',),
+ canonicalize=False,
+ unique=True,
+ process_value=None,
+ deny_extensions=None,
+ restrict_css=(),
+ strip=True,
+ restrict_text=None,
+ ):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
lx = LxmlParserLinkExtractor(
- tag=lambda x: x in tags,
- attr=lambda x: x in attrs,
+ tag=partial(operator.contains, tags),
+ attr=partial(operator.contains, attrs),
unique=unique,
process=process_value,
strip=strip,
canonicalized=canonicalize
)
-
- super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
- allow_domains=allow_domains, deny_domains=deny_domains,
- restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
- canonicalize=canonicalize, deny_extensions=deny_extensions,
- restrict_text=restrict_text)
+ super(LxmlLinkExtractor, self).__init__(
+ link_extractor=lx,
+ allow=allow,
+ deny=deny,
+ allow_domains=allow_domains,
+ deny_domains=deny_domains,
+ restrict_xpaths=restrict_xpaths,
+ restrict_css=restrict_css,
+ canonicalize=canonicalize,
+ deny_extensions=deny_extensions,
+ restrict_text=restrict_text,
+ )
def extract_links(self, response):
"""Returns a list of :class:`~scrapy.link.Link` objects from the
@@ -124,9 +150,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
"""
base_url = get_base_url(response)
if self.restrict_xpaths:
- docs = [subdoc
- for x in self.restrict_xpaths
- for subdoc in response.xpath(x)]
+ docs = [
+ subdoc
+ for x in self.restrict_xpaths
+ for subdoc in response.xpath(x)
+ ]
else:
docs = [response.selector]
all_links = []
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 68e8514ba..46d8c13af 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,3 +1,4 @@
+import pickle
import re
import unittest
from warnings import catch_warnings
@@ -462,6 +463,10 @@ class Base:
Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
])
+ def test_pickle_extractor(self):
+ lx = self.extractor_cls()
+ self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
+
class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
extractor_cls = LxmlLinkExtractor