linkextractors: add arg_to_iter support to RegexLinkExtractor

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401021

scrapy/trunk/docs/ref/link-extractors.rst

@@ -17,7 +17,7 @@ a tag name and returns True if links should be extracted from it, or False if
 they shouldn't. Defaults to 'a'.
 
 ``attr`` is either a string (with the name of an tag attribute), or a function
-that receives a an attribute name and returns True if links should be extracted from it, or False if the shouldn't. 
+that receives a an attribute name and returns True if links should be extracted from it, or False if the shouldn't.
 
 ``unique`` is a boolean that specifies if a duplicate filtering should be
 applied to links extracted.
@@ -32,28 +32,26 @@ that you can specify, including regular expressions that match (or don't match)
 the extracted links.  These parameters are configured when instantiating the
 RegexLinkExtractor object.
 
-``allow`` is a list of regular expressions that the (absolute) urls must match
-in order to be extracted.  deny: A list of regular expressions that makes any
-url matching them be ignored.  allow_domains: A list of domains from which to
-extract urls.  deny_domains: A list of domains to not extract urls from.
-restrict_xpaths: Only extract links from the areas inside the provided xpaths
-(in a list).  tags: List of tags to extract links from. Defaults to ('a',
-'area').  attrs: List of attributes to extract links from. Defaults to ('href',
-) canonicalize: Canonicalize each extracted url (using
-scrapy.utils.url.canonicalize_url). Defaults to True.
+``allow`` is a single value or a list of regular expressions that the (absolute) urls must match
+in order to be extracted.
 
-``allow_domains`` is a list of string containing domains which will be
+``deny`` is a single value or list of regular expressions that makes any url
+matching them be ignored.
+
+``allow_domains`` is single value or a list of string containing domains which will be
 considered for extracting the links
 
-``deny_domains`` is a list of strings containing domains which which won't be
+``deny_domains`` is single value or a list of strings containing domains which which won't be
 considered for extracting the links
 
-``restrict_xpaths`` is a list of string with XPath's. If specified, links will
+``restrict_xpaths`` is single value or a list of string with XPath's. If specified, links will
 only be looked inside the sections of the pages specified by those XPaths.
 
-``tags`` is an iterable with the name of the tags where links should be extracted from
+``tags`` is an iterable with the name of the tags where links should be extracted from. Defaults to ('a', 'area').
 
-``attrs`` is an interable with the name of the attributes where links should be extracted from
+``attrs`` is an interable with the name of the attributes where links should be extracted from. Defaults to ('href',)
+
+``canonicalize`` canonicalize each extracted url (using scrapy.utils.url.canonicalize_url). Defaults to True.
 
 ``unique`` is a boolean that specifies if a duplicate filtering should be
 applied to links extracted.

scrapy/trunk/scrapy/link/extractors.py

@@ -10,6 +10,7 @@ import re
 from scrapy.link import LinkExtractor
 from scrapy.utils.url import canonicalize_url, url_is_from_any_domain
 from scrapy.xpath import HtmlXPathSelector
+from scrapy.utils.misc import arg_to_iter
 
 _re_type = type(re.compile("", 0))
 
@@ -42,11 +43,11 @@ class RegexLinkExtractor(LinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), 
                  tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True):
-        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in allow]
-        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in deny]
-        self.allow_domains = set(allow_domains)
-        self.deny_domains = set(deny_domains)
-        self.restrict_xpaths = restrict_xpaths
+        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
+        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
+        self.allow_domains = set(arg_to_iter(allow_domains))
+        self.deny_domains = set(arg_to_iter(deny_domains))
+        self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
         self.canonicalize = canonicalize
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs

scrapy/trunk/scrapy/tests/test_link.py

@@ -115,6 +115,28 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
 #            [ Link(url='http://example.com/sample1.html', text=u''),
 #              Link(url='http://example.com/sample2.html', text=u'sample 2') ])
 
+    def test_extraction_using_single_values(self):
+        '''Test the extractor's behaviour among different situations'''
+
+        lx = RegexLinkExtractor(allow='sample')
+        self.assertEqual([link for link in lx.extract_links(self.response)],
+            [ Link(url='http://example.com/sample1.html', text=u''),
+              Link(url='http://example.com/sample2.html', text=u'sample 2'),
+              Link(url='http://example.com/sample3.html', text=u'sample 3 text') ])
+
+        lx = RegexLinkExtractor(allow='sample', deny='3')
+        self.assertEqual([link for link in lx.extract_links(self.response)],
+            [ Link(url='http://example.com/sample1.html', text=u''),
+              Link(url='http://example.com/sample2.html', text=u'sample 2') ])
+
+        lx = RegexLinkExtractor(allow_domains='google.com')
+        self.assertEqual([link for link in lx.extract_links(self.response)],
+            [ Link(url='http://www.google.com/something', text=u'') ])
+
+        lx = RegexLinkExtractor(deny_domains='example.com')
+        self.assertEqual([link for link in lx.extract_links(self.response)],
+            [ Link(url='http://www.google.com/something', text=u'') ])
+
     def test_matches(self):
         url1 = 'http://lotsofstuff.com/stuff1/index'
         url2 = 'http://evenmorestuff.com/uglystuff/index'