From 1d0e2d12027079fa3b147e0a51c3afe6f207a41f Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 27 Mar 2009 06:05:54 +0000 Subject: [PATCH] linkextractors: add arg_to_iter support to RegexLinkExtractor --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401021 --- scrapy/trunk/docs/ref/link-extractors.rst | 28 +++++++++++------------ scrapy/trunk/scrapy/link/extractors.py | 11 +++++---- scrapy/trunk/scrapy/tests/test_link.py | 22 ++++++++++++++++++ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/scrapy/trunk/docs/ref/link-extractors.rst b/scrapy/trunk/docs/ref/link-extractors.rst index 8f4a593d4..dec5bef44 100644 --- a/scrapy/trunk/docs/ref/link-extractors.rst +++ b/scrapy/trunk/docs/ref/link-extractors.rst @@ -17,7 +17,7 @@ a tag name and returns True if links should be extracted from it, or False if they shouldn't. Defaults to 'a'. ``attr`` is either a string (with the name of an tag attribute), or a function -that receives a an attribute name and returns True if links should be extracted from it, or False if the shouldn't. +that receives a an attribute name and returns True if links should be extracted from it, or False if the shouldn't. ``unique`` is a boolean that specifies if a duplicate filtering should be applied to links extracted. @@ -32,28 +32,26 @@ that you can specify, including regular expressions that match (or don't match) the extracted links. These parameters are configured when instantiating the RegexLinkExtractor object. -``allow`` is a list of regular expressions that the (absolute) urls must match -in order to be extracted. deny: A list of regular expressions that makes any -url matching them be ignored. allow_domains: A list of domains from which to -extract urls. deny_domains: A list of domains to not extract urls from. -restrict_xpaths: Only extract links from the areas inside the provided xpaths -(in a list). tags: List of tags to extract links from. Defaults to ('a', -'area'). attrs: List of attributes to extract links from. Defaults to ('href', -) canonicalize: Canonicalize each extracted url (using -scrapy.utils.url.canonicalize_url). Defaults to True. +``allow`` is a single value or a list of regular expressions that the (absolute) urls must match +in order to be extracted. -``allow_domains`` is a list of string containing domains which will be +``deny`` is a single value or list of regular expressions that makes any url +matching them be ignored. + +``allow_domains`` is single value or a list of string containing domains which will be considered for extracting the links -``deny_domains`` is a list of strings containing domains which which won't be +``deny_domains`` is single value or a list of strings containing domains which which won't be considered for extracting the links -``restrict_xpaths`` is a list of string with XPath's. If specified, links will +``restrict_xpaths`` is single value or a list of string with XPath's. If specified, links will only be looked inside the sections of the pages specified by those XPaths. -``tags`` is an iterable with the name of the tags where links should be extracted from +``tags`` is an iterable with the name of the tags where links should be extracted from. Defaults to ('a', 'area'). -``attrs`` is an interable with the name of the attributes where links should be extracted from +``attrs`` is an interable with the name of the attributes where links should be extracted from. Defaults to ('href',) + +``canonicalize`` canonicalize each extracted url (using scrapy.utils.url.canonicalize_url). Defaults to True. ``unique`` is a boolean that specifies if a duplicate filtering should be applied to links extracted. diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index a0daa46a9..88a3d8855 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -10,6 +10,7 @@ import re from scrapy.link import LinkExtractor from scrapy.utils.url import canonicalize_url, url_is_from_any_domain from scrapy.xpath import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter _re_type = type(re.compile("", 0)) @@ -42,11 +43,11 @@ class RegexLinkExtractor(LinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True): - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in allow] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in deny] - self.allow_domains = set(allow_domains) - self.deny_domains = set(deny_domains) - self.restrict_xpaths = restrict_xpaths + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] + self.allow_domains = set(arg_to_iter(allow_domains)) + self.deny_domains = set(arg_to_iter(deny_domains)) + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) self.canonicalize = canonicalize tag_func = lambda x: x in tags attr_func = lambda x: x in attrs diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index e2a1ead6c..53528e0ba 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -115,6 +115,28 @@ class RegexLinkExtractorTestCase(unittest.TestCase): # [ Link(url='http://example.com/sample1.html', text=u''), # Link(url='http://example.com/sample2.html', text=u'sample 2') ]) + def test_extraction_using_single_values(self): + '''Test the extractor's behaviour among different situations''' + + lx = RegexLinkExtractor(allow='sample') + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text') ]) + + lx = RegexLinkExtractor(allow='sample', deny='3') + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2') ]) + + lx = RegexLinkExtractor(allow_domains='google.com') + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://www.google.com/something', text=u'') ]) + + lx = RegexLinkExtractor(deny_domains='example.com') + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://www.google.com/something', text=u'') ]) + def test_matches(self): url1 = 'http://lotsofstuff.com/stuff1/index' url2 = 'http://evenmorestuff.com/uglystuff/index'