diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index b6de74f33..f747fa99b 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -81,8 +81,8 @@ class LxmlParserLinkExtractor(object): class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, - deny_extensions=None): + restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, + unique=True, process_value=None, deny_extensions=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs @@ -90,8 +90,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor): unique=unique, process=process_value) super(LxmlLinkExtractor, self).__init__(lx, allow, deny, - allow_domains, deny_domains, restrict_xpaths, canonicalize, - deny_extensions) + allow_domains, deny_domains, restrict_xpaths, restrict_css, + canonicalize, deny_extensions) def extract_links(self, response): html = Selector(response) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 3eb5fd91f..3a8fdbb69 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -98,8 +98,8 @@ class BaseSgmlLinkExtractor(SGMLParser): class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, - deny_extensions=None): + restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, + process_value=None, deny_extensions=None): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " @@ -116,8 +116,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): unique=unique, process_value=process_value) super(SgmlLinkExtractor, self).__init__(lx, allow, deny, - allow_domains, deny_domains, restrict_xpaths, canonicalize, - deny_extensions) + allow_domains, deny_domains, restrict_xpaths, restrict_css, + canonicalize, deny_extensions) # FIXME: was added to fix a RegexLinkExtractor testcase self.base_url = None diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index 5badea5e5..52271959f 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -5,6 +5,7 @@ scrapy.contrib.linkextractor). import re from six.moves.urllib.parse import urlparse +from scrapy.selector.csstranslator import ScrapyHTMLTranslator from scrapy.utils.url import url_is_from_any_domain from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension from scrapy.utils.misc import arg_to_iter @@ -38,8 +39,10 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil class FilteringLinkExtractor(object): + _csstranslator = ScrapyHTMLTranslator() + def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions): + restrict_xpaths, restrict_css, canonicalize, deny_extensions): self.link_extractor = link_extractor @@ -50,6 +53,9 @@ class FilteringLinkExtractor(object): self.deny_domains = set(arg_to_iter(deny_domains)) self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, + arg_to_iter(restrict_css))) + self.canonicalize = canonicalize if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS