add CSS support for link extractors

scrapy/contrib/linkextractors/lxmlhtml.py

@@ -81,8 +81,8 @@ class LxmlParserLinkExtractor(object):
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
-                 deny_extensions=None):
+                 restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True,
+                 unique=True, process_value=None, deny_extensions=None):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
@@ -90,8 +90,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
             unique=unique, process=process_value)
 
         super(LxmlLinkExtractor, self).__init__(lx, allow, deny,
-            allow_domains, deny_domains, restrict_xpaths, canonicalize,
-            deny_extensions)
+            allow_domains, deny_domains, restrict_xpaths, restrict_css,
+            canonicalize, deny_extensions)
 
     def extract_links(self, response):
         html = Selector(response)

scrapy/contrib/linkextractors/sgml.py

@@ -98,8 +98,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
 class SgmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
-                 deny_extensions=None):
+                 restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
+                 process_value=None, deny_extensions=None):
 
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
@@ -116,8 +116,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
                 unique=unique, process_value=process_value)
 
         super(SgmlLinkExtractor, self).__init__(lx, allow, deny,
-            allow_domains, deny_domains, restrict_xpaths, canonicalize,
-            deny_extensions)
+            allow_domains, deny_domains, restrict_xpaths, restrict_css,
+            canonicalize, deny_extensions)
 
         # FIXME: was added to fix a RegexLinkExtractor testcase
         self.base_url = None

scrapy/linkextractor.py

@@ -5,6 +5,7 @@ scrapy.contrib.linkextractor).
 import re
 from six.moves.urllib.parse import urlparse
 
+from scrapy.selector.csstranslator import ScrapyHTMLTranslator
 from scrapy.utils.url import url_is_from_any_domain
 from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
 from scrapy.utils.misc import arg_to_iter
@@ -38,8 +39,10 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil
 
 class FilteringLinkExtractor(object):
 
+    _csstranslator = ScrapyHTMLTranslator()
+
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
-                 restrict_xpaths, canonicalize, deny_extensions):
+                 restrict_xpaths, restrict_css, canonicalize, deny_extensions):
 
         self.link_extractor = link_extractor
 
@@ -50,6 +53,9 @@ class FilteringLinkExtractor(object):
         self.deny_domains = set(arg_to_iter(deny_domains))
 
         self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
+        self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,
+                                          arg_to_iter(restrict_css)))
+
         self.canonicalize = canonicalize
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS