From cd85c12c3383ea185d24c9023c33076081cce7a9 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 18 May 2011 12:32:34 -0300 Subject: [PATCH] Some Link extractor improvements: * added support for ignoring common file extensions that are not followed if they occur in links * fixed link extractor documentation issues * slighly improved performance of applying filters * added link to link extractors doc from documentation index --- docs/index.rst | 3 ++ docs/topics/link-extractors.rst | 15 +++++-- scrapy/contrib/linkextractors/sgml.py | 43 +++++++++++++-------- scrapy/linkextractor.py | 21 ++++++++++ scrapy/tests/test_contrib_linkextractors.py | 11 ++++-- scrapy/utils/url.py | 18 ++++++--- 6 files changed, 80 insertions(+), 31 deletions(-) create mode 100644 scrapy/linkextractor.py diff --git a/docs/index.rst b/docs/index.rst index 0e80cc12c..65621eaa6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Basic concepts topics/shell topics/item-pipeline topics/feed-exports + topics/link-extractors :doc:`topics/commands` Learn about the command-line tool used to manage your Scrapy project. @@ -89,6 +90,8 @@ Basic concepts :doc:`topics/feed-exports` Output your scraped data using different formats and storages. +:doc:`topics/link-extractors` + Convenient classes to extract links to follow from pages. Built-in services ================= diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 41160a57d..ac53eaa59 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -42,7 +42,7 @@ All available link extractors classes bundled with Scrapy are provided in the SgmlLinkExtractor ----------------- -.. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None) +.. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None) The SgmlLinkExtractor extends the base :class:`BaseSgmlLinkExtractor` by providing additional filters that you can specify to extract links, @@ -59,15 +59,21 @@ SgmlLinkExtractor that the (absolute) urls must match in order to be excluded (ie. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. - :type allow: a regular expression (or list of) + :type deny: a regular expression (or list of) :param allow_domains: a single value or a list of string containing domains which will be considered for extracting the links - :type allow: str or list + :type allow_domains: str or list :param deny_domains: a single value or a list of strings containing domains which won't be considered for extracting the links - :type allow: str or list + :type deny_domains: str or list + + :param deny_extensions: a list of extensions that should be ignored when + extracting links. If not given, it will default to the + ``IGNORED_EXTENSIONS`` list defined in the `scrapy.linkextractor`_ + module. + :type deny_extensions: list :param restrict_xpaths: is a XPath (or list of XPath's) which defines regions inside the response where links should be extracted from. @@ -145,3 +151,4 @@ BaseSgmlLinkExtractor :type process_value: callable +.. _scrapy.linkextractor: http://dev.scrapy.org/browser/scrapy/linkextractor.py diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index fad6a8218..4412ef75e 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -3,14 +3,16 @@ SGMLParser-based Link extractors """ import re +from urlparse import urlparse from w3lib.url import safe_url_string, urljoin_rfc from scrapy.selector import HtmlXPathSelector from scrapy.link import Link +from scrapy.linkextractor import IGNORED_EXTENSIONS from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode -from scrapy.utils.url import canonicalize_url, url_is_from_any_domain +from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension from scrapy.utils.response import get_base_url class BaseSgmlLinkExtractor(FixedSGMLParser): @@ -91,13 +93,17 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil class SgmlLinkExtractor(BaseSgmlLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None): + tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None, + deny_extensions=None): self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] self.allow_domains = set(arg_to_iter(allow_domains)) self.deny_domains = set(arg_to_iter(deny_domains)) self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) self.canonicalize = canonicalize + if deny_extensions is None: + deny_extensions = IGNORED_EXTENSIONS + self.deny_extensions = set(['.' + e for e in deny_extensions]) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs BaseSgmlLinkExtractor.__init__(self, tag=tag_func, attr=attr_func, @@ -118,24 +124,27 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): return links def _process_links(self, links): - links = [link for link in links if _is_valid_url(link.url)] - - if self.allow_res: - links = [link for link in links if _matches(link.url, self.allow_res)] - if self.deny_res: - links = [link for link in links if not _matches(link.url, self.deny_res)] - if self.allow_domains: - links = [link for link in links if url_is_from_any_domain(link.url, self.allow_domains)] - if self.deny_domains: - links = [link for link in links if not url_is_from_any_domain(link.url, self.deny_domains)] - - if self.canonicalize: - for link in links: - link.url = canonicalize_url(link.url) - + links = [x for x in links if self._link_allowed(x)] links = BaseSgmlLinkExtractor._process_links(self, links) return links + def _link_allowed(self, link): + parsed_url = urlparse(link.url) + allowed = _is_valid_url(link.url) + if self.allow_res: + allowed &= _matches(link.url, self.allow_res) + if self.deny_res: + allowed &= not _matches(link.url, self.deny_res) + if self.allow_domains: + allowed &= url_is_from_any_domain(parsed_url, self.allow_domains) + if self.deny_domains: + allowed &= not url_is_from_any_domain(parsed_url, self.deny_domains) + if self.deny_extensions: + allowed &= not url_has_any_extension(parsed_url, self.deny_extensions) + if allowed and self.canonicalize: + link.url = canonicalize_url(parsed_url) + return allowed + def matches(self, url): if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py new file mode 100644 index 000000000..5c2100d74 --- /dev/null +++ b/scrapy/linkextractor.py @@ -0,0 +1,21 @@ +""" +Common code and definitions used by Link extractors (located in +scrapy.contrib.linkextractor). +""" + +# common file extensions that are not followed if they occur in links +IGNORED_EXTENSIONS = [ + # images + 'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif', + 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', + + # audio + 'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff', + + # video + '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', + 'm4a', + + # other + 'css', 'pdf', 'doc', 'exe', 'bin', 'rss', 'zip', 'rar', +] diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 0a4f23814..6133d4c6d 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -135,10 +135,6 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://www.google.com/something', text=u'') ]) - lx = SgmlLinkExtractor(tags=('img', ), attrs=('src', )) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample2.jpg', text=u'') ]) - def test_extraction_using_single_values(self): '''Test the extractor's behaviour among different situations''' @@ -211,6 +207,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + def test_deny_extensions(self): + html = """asd and """ + response = HtmlResponse("http://example.org/", body=html) + lx = SgmlLinkExtractor() + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/page.html', text=u'asd')]) + def test_process_value(self): """Test restrict_xpaths with encodings""" html = """ diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index da5d33622..696d8b288 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,11 +6,8 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ -import os -import re import urlparse import urllib -import posixpath import cgi from w3lib.url import * @@ -18,7 +15,7 @@ from scrapy.utils.python import unicode_to_str def url_is_from_any_domain(url, domains): """Return True if the url belongs to any of the given domains""" - host = urlparse.urlparse(url).hostname + host = parse_url(url).hostname if host: return any(((host == d) or (host.endswith('.%s' % d)) for d in domains)) @@ -30,6 +27,9 @@ def url_is_from_spider(url, spider): return url_is_from_any_domain(url, [spider.name] + \ getattr(spider, 'allowed_domains', [])) +def url_has_any_extension(url, extensions): + return posixpath.splitext(parse_url(url).path)[1].lower() in extensions + def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ encoding=None): """Canonicalize the given url by applying the following procedures: @@ -48,11 +48,17 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ For examples see the tests in scrapy.tests.test_utils_url """ - url = unicode_to_str(url, encoding) - scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) + scheme, netloc, path, params, query, fragment = parse_url(url) keyvals = cgi.parse_qsl(query, keep_blank_values) keyvals.sort() query = urllib.urlencode(keyvals) path = safe_url_string(urllib.unquote(path)) fragment = '' if not keep_fragments else fragment return urlparse.urlunparse((scheme, netloc.lower(), path, params, query, fragment)) + +def parse_url(url, encoding=None): + """Return urlparsed url from the given argument (which could be an already + parsed url) + """ + return url if isinstance(url, urlparse.ParseResult) else \ + urlparse.urlparse(unicode_to_str(url, encoding))