added process_value argument to Link extractors constructor

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401103

scrapy/trunk/docs/ref/link-extractors.rst

@@ -10,7 +10,7 @@ Available Link Extractors
 LinkExtractor
 =============
 
-.. class:: LinkExtractor(tag="a", href="href", unique=False)
+.. class:: LinkExtractor(tag="a", href="href", unique=False, process_value=None)
 
     This is the most basic Link Extractor which extracts links from a response with
     by looking at the given attributes inside the given tags.
@@ -34,10 +34,32 @@ LinkExtractor
         be applied to links extracted.
     :type unique: boolean
 
+    :param process_value: a function which receives each value extracted from
+        the tag and attributes scanned and can modify the value and return a
+        new one, or return ``None`` to ignore the link altogether. If not
+        given, ``process_value`` defaults to ``lambda x: x``.
+
+        .. highlight:: html
+
+        For example, to extract links from this code::
+
+            <a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
+        
+        .. highlight:: python
+
+        You can use the following function in ``process_value``::
+        
+            def process_value(value):
+                m = re.search("javascript:goToPage\('(.*?)'", value)
+                if m:
+                    return m.group(1) 
+
+    :type process_value: callable
+
 RegexLinkExtractor
 ==================
 
-.. class:: RegexLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True)
+.. class:: RegexLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None)
 
     The RegexLinkExtractor extends the base :class:`LinkExtractor` by providing
     additional filters that you can specify to extract links, including regular
@@ -86,3 +108,7 @@ RegexLinkExtractor
         links.
     :type unique: boolean
 
+    :param process_value: see ``process_value`` argument of
+        :class:`LinkExtractor` class constructor
+    :type process_value: boolean
+

scrapy/trunk/scrapy/link/__init__.py

@@ -9,10 +9,11 @@ from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin
 
 class LinkExtractor(FixedSGMLParser):
 
-    def __init__(self, tag="a", attr="href", unique=False):
+    def __init__(self, tag="a", attr="href", unique=False, process_value=None):
         FixedSGMLParser.__init__(self)
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
+        self.process_value = (lambda v: v) if process_value is None else process_value
         self.current_link = None
         self.unique = unique
 
@@ -48,9 +49,11 @@ class LinkExtractor(FixedSGMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
-                    link = Link(url=value)
-                    self.links.append(link)
-                    self.current_link = link
+                    url = self.process_value(value)
+                    if url is not None:
+                        link = Link(url=url)
+                        self.links.append(link)
+                        self.current_link = link
 
     def unknown_endtag(self, tag):
         self.current_link = None

scrapy/trunk/scrapy/link/extractors.py

@@ -20,7 +20,7 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil
 class RegexLinkExtractor(LinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), 
-                 tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True):
+                 tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None):
         self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
         self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
         self.allow_domains = set(arg_to_iter(allow_domains))
@@ -29,7 +29,8 @@ class RegexLinkExtractor(LinkExtractor):
         self.canonicalize = canonicalize
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
-        LinkExtractor.__init__(self, tag=tag_func, attr=attr_func, unique=unique)
+        LinkExtractor.__init__(self, tag=tag_func, attr=attr_func, 
+            unique=unique, process_value=process_value)
 
     def extract_links(self, response):
         if self.restrict_xpaths:

scrapy/trunk/scrapy/tests/test_link.py

@@ -1,4 +1,5 @@
 import os
+import re
 import unittest
 
 from scrapy.http import HtmlResponse
@@ -182,6 +183,22 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
         self.assertEqual(lx.extract_links(response),
                          [Link(url='http://example.org/about.html', text=u'About us\xa3')])
 
+    def test_process_value(self):
+        """Test restrict_xpaths with encodings"""
+        html = """
+        <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a>
+        <a href="/about.html">About us</a>
+        """
+        response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
+
+        def process_value(value):
+            m = re.search("javascript:goToPage\('(.*?)'", value)
+            if m:
+                return m.group(1)
+
+        lx = RegexLinkExtractor(process_value=process_value)
+        self.assertEqual(lx.extract_links(response),
+                         [Link(url='http://example.org/other/page.html', text='Link text')])
 
 class HTMLImageLinkExtractorTestCase(unittest.TestCase):
     def setUp(self):