From d4cb03eded95dbd27af8dd90495fa4c277ef0b48 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Thu, 11 Dec 2014 16:45:20 -0200 Subject: [PATCH 1/4] add CSS support for link extractors --- scrapy/contrib/linkextractors/lxmlhtml.py | 8 ++++---- scrapy/contrib/linkextractors/sgml.py | 8 ++++---- scrapy/linkextractor.py | 8 +++++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index b6de74f33..f747fa99b 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -81,8 +81,8 @@ class LxmlParserLinkExtractor(object): class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, - deny_extensions=None): + restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, + unique=True, process_value=None, deny_extensions=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs @@ -90,8 +90,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor): unique=unique, process=process_value) super(LxmlLinkExtractor, self).__init__(lx, allow, deny, - allow_domains, deny_domains, restrict_xpaths, canonicalize, - deny_extensions) + allow_domains, deny_domains, restrict_xpaths, restrict_css, + canonicalize, deny_extensions) def extract_links(self, response): html = Selector(response) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 3eb5fd91f..3a8fdbb69 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -98,8 +98,8 @@ class BaseSgmlLinkExtractor(SGMLParser): class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, - deny_extensions=None): + restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, + process_value=None, deny_extensions=None): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " @@ -116,8 +116,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): unique=unique, process_value=process_value) super(SgmlLinkExtractor, self).__init__(lx, allow, deny, - allow_domains, deny_domains, restrict_xpaths, canonicalize, - deny_extensions) + allow_domains, deny_domains, restrict_xpaths, restrict_css, + canonicalize, deny_extensions) # FIXME: was added to fix a RegexLinkExtractor testcase self.base_url = None diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index 5badea5e5..52271959f 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -5,6 +5,7 @@ scrapy.contrib.linkextractor). import re from six.moves.urllib.parse import urlparse +from scrapy.selector.csstranslator import ScrapyHTMLTranslator from scrapy.utils.url import url_is_from_any_domain from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension from scrapy.utils.misc import arg_to_iter @@ -38,8 +39,10 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil class FilteringLinkExtractor(object): + _csstranslator = ScrapyHTMLTranslator() + def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions): + restrict_xpaths, restrict_css, canonicalize, deny_extensions): self.link_extractor = link_extractor @@ -50,6 +53,9 @@ class FilteringLinkExtractor(object): self.deny_domains = set(arg_to_iter(deny_domains)) self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, + arg_to_iter(restrict_css))) + self.canonicalize = canonicalize if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS From 403fc686b861b193a13a9abe038eda92aebf5ce8 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Thu, 11 Dec 2014 18:20:30 -0200 Subject: [PATCH 2/4] tests for CSS support in link extractors --- tests/test_contrib_linkextractors.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_contrib_linkextractors.py b/tests/test_contrib_linkextractors.py index 3902d4c50..a624f9e66 100644 --- a/tests/test_contrib_linkextractors.py +++ b/tests/test_contrib_linkextractors.py @@ -284,6 +284,21 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', fragment='', nofollow=False)]) + def test_restrict_css(self): + lx = self.extractor_cls(restrict_css=('#subwrapper a',)) + self.assertEqual(lx.extract_links(self.response), [ + Link(url='http://example.com/sample2.html', text=u'sample 2') + ]) + + def test_restrict_css_and_restrict_xpaths_together(self): + lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), + restrict_css=('#subwrapper + a', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) + def test_area_tag_with_unicode_present(self): body = """\xbe\xa9""" response = HtmlResponse("http://example.org", body=body, encoding='utf-8') From b0730a1d16b608a34d12218ed1fd36c6720eee61 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Thu, 11 Dec 2014 18:22:08 -0200 Subject: [PATCH 3/4] documentation for CSS support in link extractors --- docs/topics/link-extractors.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 43740adcc..9758c2f35 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -51,7 +51,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None) +.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None) LxmlLinkExtractor is the recommended link extractor with handy filtering options. It is implemented using lxml's robust HTMLParser. @@ -88,6 +88,11 @@ LxmlLinkExtractor links. See examples below. :type restrict_xpaths: str or list + :param restrict_css: a CSS selector (or list of selectors) which defines + regions inside the response where links should be extracted from. + Has the same behaviour as ``restrict_xpaths``. + :type restrict_css: str or list + :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. :type tags: str or list From 22247cf79152dbeb635b77f445ebd359c6d910db Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Mon, 15 Dec 2014 09:18:15 -0200 Subject: [PATCH 4/4] move restrict_css argument to end of argument list in link extractors for backwards compatibility, use keyword arguments in link extractor super().__init__() calls --- scrapy/contrib/linkextractors/lxmlhtml.py | 11 ++++++----- scrapy/contrib/linkextractors/sgml.py | 11 ++++++----- scrapy/linkextractor.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index f747fa99b..1ff8e4d36 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -81,17 +81,18 @@ class LxmlParserLinkExtractor(object): class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, - unique=True, process_value=None, deny_extensions=None): + tags=('a', 'area'), attrs=('href',), canonicalize=True, + unique=True, process_value=None, deny_extensions=None, restrict_css=()): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, unique=unique, process=process_value) - super(LxmlLinkExtractor, self).__init__(lx, allow, deny, - allow_domains, deny_domains, restrict_xpaths, restrict_css, - canonicalize, deny_extensions) + super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions) def extract_links(self, response): html = Selector(response) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 3a8fdbb69..335773db1 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -98,8 +98,8 @@ class BaseSgmlLinkExtractor(SGMLParser): class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, - process_value=None, deny_extensions=None): + tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, + process_value=None, deny_extensions=None, restrict_css=()): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " @@ -115,9 +115,10 @@ class SgmlLinkExtractor(FilteringLinkExtractor): lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, unique=unique, process_value=process_value) - super(SgmlLinkExtractor, self).__init__(lx, allow, deny, - allow_domains, deny_domains, restrict_xpaths, restrict_css, - canonicalize, deny_extensions) + super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions) # FIXME: was added to fix a RegexLinkExtractor testcase self.base_url = None diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index 52271959f..227d79b46 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -42,7 +42,7 @@ class FilteringLinkExtractor(object): _csstranslator = ScrapyHTMLTranslator() def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, restrict_css, canonicalize, deny_extensions): + restrict_xpaths, canonicalize, deny_extensions, restrict_css): self.link_extractor = link_extractor