diff --git a/scrapy/trunk/docs/ref/link-extractors.rst b/scrapy/trunk/docs/ref/link-extractors.rst index 71afef4e7..061a5bb7b 100644 --- a/scrapy/trunk/docs/ref/link-extractors.rst +++ b/scrapy/trunk/docs/ref/link-extractors.rst @@ -10,7 +10,7 @@ Available Link Extractors LinkExtractor ============= -.. class:: LinkExtractor(tag="a", href="href", unique=False) +.. class:: LinkExtractor(tag="a", href="href", unique=False, process_value=None) This is the most basic Link Extractor which extracts links from a response with by looking at the given attributes inside the given tags. @@ -34,10 +34,32 @@ LinkExtractor be applied to links extracted. :type unique: boolean + :param process_value: a function which receives each value extracted from + the tag and attributes scanned and can modify the value and return a + new one, or return ``None`` to ignore the link altogether. If not + given, ``process_value`` defaults to ``lambda x: x``. + + .. highlight:: html + + For example, to extract links from this code:: + + Link text + + .. highlight:: python + + You can use the following function in ``process_value``:: + + def process_value(value): + m = re.search("javascript:goToPage\('(.*?)'", value) + if m: + return m.group(1) + + :type process_value: callable + RegexLinkExtractor ================== -.. class:: RegexLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True) +.. class:: RegexLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None) The RegexLinkExtractor extends the base :class:`LinkExtractor` by providing additional filters that you can specify to extract links, including regular @@ -86,3 +108,7 @@ RegexLinkExtractor links. :type unique: boolean + :param process_value: see ``process_value`` argument of + :class:`LinkExtractor` class constructor + :type process_value: boolean + diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index 91bdf2ad3..b6bfc2cc4 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -9,10 +9,11 @@ from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin class LinkExtractor(FixedSGMLParser): - def __init__(self, tag="a", attr="href", unique=False): + def __init__(self, tag="a", attr="href", unique=False, process_value=None): FixedSGMLParser.__init__(self) self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.process_value = (lambda v: v) if process_value is None else process_value self.current_link = None self.unique = unique @@ -48,9 +49,11 @@ class LinkExtractor(FixedSGMLParser): if self.scan_tag(tag): for attr, value in attrs: if self.scan_attr(attr): - link = Link(url=value) - self.links.append(link) - self.current_link = link + url = self.process_value(value) + if url is not None: + link = Link(url=url) + self.links.append(link) + self.current_link = link def unknown_endtag(self, tag): self.current_link = None diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 1b67da7c3..7e0b3fefc 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -20,7 +20,7 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil class RegexLinkExtractor(LinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True): + tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None): self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] self.allow_domains = set(arg_to_iter(allow_domains)) @@ -29,7 +29,8 @@ class RegexLinkExtractor(LinkExtractor): self.canonicalize = canonicalize tag_func = lambda x: x in tags attr_func = lambda x: x in attrs - LinkExtractor.__init__(self, tag=tag_func, attr=attr_func, unique=unique) + LinkExtractor.__init__(self, tag=tag_func, attr=attr_func, + unique=unique, process_value=process_value) def extract_links(self, response): if self.restrict_xpaths: diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index 9eab45d7b..d28c59795 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -1,4 +1,5 @@ import os +import re import unittest from scrapy.http import HtmlResponse @@ -182,6 +183,22 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + def test_process_value(self): + """Test restrict_xpaths with encodings""" + html = """ + Link text + About us + """ + response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') + + def process_value(value): + m = re.search("javascript:goToPage\('(.*?)'", value) + if m: + return m.group(1) + + lx = RegexLinkExtractor(process_value=process_value) + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/other/page.html', text='Link text')]) class HTMLImageLinkExtractorTestCase(unittest.TestCase): def setUp(self):