From d4cc91ae5443db37afe32813d2d6ba6d008a2edd Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 29 Apr 2009 23:16:02 +0000 Subject: [PATCH] removed redundant link extractors documentation in docstrings, and some updated to link extractors doc --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401101 --- scrapy/trunk/docs/ref/link-extractors.rst | 86 +++++++++++++------- scrapy/trunk/docs/topics/link-extractors.rst | 30 ++++--- scrapy/trunk/scrapy/link/__init__.py | 25 ------ scrapy/trunk/scrapy/link/extractors.py | 24 +----- 4 files changed, 76 insertions(+), 89 deletions(-) diff --git a/scrapy/trunk/docs/ref/link-extractors.rst b/scrapy/trunk/docs/ref/link-extractors.rst index dec5bef44..71afef4e7 100644 --- a/scrapy/trunk/docs/ref/link-extractors.rst +++ b/scrapy/trunk/docs/ref/link-extractors.rst @@ -4,55 +4,85 @@ Available Link Extractors ========================= +.. module:: scrapy.link + :synopsis: Link extractors classes + LinkExtractor ============= .. class:: LinkExtractor(tag="a", href="href", unique=False) -This is the most basic Link Extractor which extracts links from a response with -by looking at the given attributes inside the given tags. + This is the most basic Link Extractor which extracts links from a response with + by looking at the given attributes inside the given tags. -``tag`` is either a string (with the name of a tag) or a function that receives -a tag name and returns True if links should be extracted from it, or False if -they shouldn't. Defaults to 'a'. + The constructor arguments are: -``attr`` is either a string (with the name of an tag attribute), or a function -that receives a an attribute name and returns True if links should be extracted from it, or False if the shouldn't. + :param tag: either a string (with the name of a tag) or a function that + receives a tag name and returns ``True`` if links should be extracted + from those tag, or ``False`` if they shouldn't. Defaults to ``'a'``. + request (once its downloaded) as its first parameter. For more + information see :ref:`ref-request-callback-arguments` below. + :type tag: str or callable -``unique`` is a boolean that specifies if a duplicate filtering should be -applied to links extracted. + :param attr: either string (with the name of a tag attribute), or a + function that receives a an attribute name and returns ``True`` if + links should be extracted from it, or ``False`` if the shouldn't. + Defaults to ``href``. + :type attr: str or callable + + :param unique: is a boolean that specifies if a duplicate filtering should + be applied to links extracted. + :type unique: boolean RegexLinkExtractor ================== .. class:: RegexLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True) -This Link Extractor extracts links from a response by applying several filters -that you can specify, including regular expressions that match (or don't match) -the extracted links. These parameters are configured when instantiating the -RegexLinkExtractor object. + The RegexLinkExtractor extends the base :class:`LinkExtractor` by providing + additional filters that you can specify to extract links, including regular + expressions patterns that the links must match to be extracted. All those + filters are configured through these constructor paramters: -``allow`` is a single value or a list of regular expressions that the (absolute) urls must match -in order to be extracted. + :param allow: a single regular expression (or list of regular expressions) + that the (absolute) urls must match in order to be extracted. If not + given (or empty), it will match all links. + :type allow: a regular expression (or list of) -``deny`` is a single value or list of regular expressions that makes any url -matching them be ignored. + :param deny: a single regular expression (or list of regular expressions) + that the (absolute) urls must match in order to be excluded (ie. not + extracted). It has precedence over the ``allow`` parameter. If not + given (or empty) it won't exclude any links. + :type allow: a regular expression (or list of) -``allow_domains`` is single value or a list of string containing domains which will be -considered for extracting the links + :param allow_domains: is single value or a list of string containing + domains which will be considered for extracting the links + :type allow: str or list -``deny_domains`` is single value or a list of strings containing domains which which won't be -considered for extracting the links + :param deny_domains: is single value or a list of strings containing + domains which which won't be considered for extracting the links + :type allow: str or list -``restrict_xpaths`` is single value or a list of string with XPath's. If specified, links will -only be looked inside the sections of the pages specified by those XPaths. + :param restrict_xpaths: is a XPath (or list of XPath's) which defines + regions inside the response where links should be extracted from. + If given, only the text selected by those XPath will be scanned for + links. See examples below. + :type restrict_xpaths: str or list -``tags`` is an iterable with the name of the tags where links should be extracted from. Defaults to ('a', 'area'). + :param tags: a tag or a list of tags to consider when extracting links. + Defaults to ``('a', 'area')``. + :type tags: str or list -``attrs`` is an interable with the name of the attributes where links should be extracted from. Defaults to ('href',) + :param attrs: list of attrbitues which should be considered when looking + for links to extract (only for those tags specified in the ``tags`` + parameter). Defaults to ``('href',)`` + :type attrs: boolean -``canonicalize`` canonicalize each extracted url (using scrapy.utils.url.canonicalize_url). Defaults to True. + :param canonicalize: canonicalize each extracted url (using + scrapy.utils.url.canonicalize_url). Defaults to ``True``. + :type canonicalize: boolean -``unique`` is a boolean that specifies if a duplicate filtering should be -applied to links extracted. + :param unique: whether duplicate filtering should be applied to extracted + links. + :type unique: boolean diff --git a/scrapy/trunk/docs/topics/link-extractors.rst b/scrapy/trunk/docs/topics/link-extractors.rst index d0fe55b5c..48d87f107 100644 --- a/scrapy/trunk/docs/topics/link-extractors.rst +++ b/scrapy/trunk/docs/topics/link-extractors.rst @@ -4,22 +4,26 @@ Link Extractors =============== -.. module:: scrapy.link +LinkExtractors are objects whose only purpose is to extract links from web +pages (:class:`scrapy.http.Response` objects) which will be eventually +followed. -LinkExtractors are objects whose purpose is to extract links from web pages. -They're used in the :class:`~scrapy.contrib.spiders.CrawlSpider`, for defining -crawling rules, among other places. +There are two Link Extractors available in Scrapy by default, but you create +your own custom Link Extractors to suit your needs by implanting a simple +interface. -There are two different LinkExtractors available in Scrapy by default, but you -create your own custom Link Extractor to suit your needs. +The only public method that every LinkExtractor have is ``extract_links``, +which receives a :class:`~scrapy.http.Response` object and returns a list +of links. Link Extractors are meant to be instantiated once and their +``extract_links`` method called several times with different responses, to +extract links to follow. -The only public method that every LinkExtractor has is ``extract_links``, which -always receives a response, independently of which LinkExtractor are you using. -This method should be called by you in case you want to extract links from a -response yourself. In the case of rules, however, you'll only have to define -your rules with the corresponding LinkExtractors, and the CrawlSpider will take -care of extracting them for each response arriving. +Link extractors are used in the :class:`~scrapy.contrib.spiders.CrawlSpider` +class (available in Scrapy), through a set of rules, but you can also use it in +your spiders even if you don't subclass from +:class:`~scrapy.contrib.spiders.CrawlSpider`, as its purpose is very simple: to +extract links. See :ref:`ref-link-extractors` for the list of available built-in Link -Extractors. +Extractors, including some examples. diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index e59f1d75d..91bdf2ad3 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -8,31 +8,6 @@ from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_u from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin class LinkExtractor(FixedSGMLParser): - """LinkExtractor are used to extract links from web pages. They are - instantiated and later "applied" to a Response using the extract_links - method which must receive a Response object and return a list of Link objects - containing the (absolute) urls to follow, and the links texts. - - This is the base LinkExtractor class that provides enough basic - functionality for extracting links to follow, but you could override this - class or create a new one if you need some additional functionality. The - only requisite is that the new (or overrided) class must provide a - extract_links method that receives a Response and returns a list of Link objects. - - This LinkExtractor always returns percent-encoded URLs, using the detected encoding - from the response. - - The constructor arguments are: - - * tag (string or function) - * a tag name which is used to search for links (defaults to "a") - * a function which receives a tag name and returns whether to scan it - * attr (string or function) - * an attribute name which is used to search for links (defaults to "href") - * a function which receives an attribute name and returns whether to scan it - * unique - if True the same urls won't be extracted twice, otherwise the - same urls will be extracted multiple times (with potentially different link texts) - """ def __init__(self, tag="a", attr="href", unique=False): FixedSGMLParser.__init__(self) diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 88a3d8855..1b67da7c3 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -1,6 +1,6 @@ """ This module provides some LinkExtractors, which extend the base LinkExtractor -(scrapy.link.LinkExtractor) with some addutional useful features. +(scrapy.link.LinkExtractor) with some additional useful features. See documentation in docs/ref/link-extractors.rst """ @@ -18,28 +18,6 @@ _matches = lambda url, regexs: any((r.search(url) for r in regexs)) _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file']) class RegexLinkExtractor(LinkExtractor): - """RegexLinkExtractor implements extends the base LinkExtractor by - providing several mechanisms to extract the links. - - It's constructor parameters are: - - allow - list of regexes that the (absolute urls) must match to be extracted - deny - ignore urls that match any of these regexes - allow_domains - only extract urls from these domains - deny_domains - ignore urls from these dmoains - restrict_xpaths - restrict url extraction to given xpaths contents - tags - look for urls in this tags - attrs - look for urls in this attrs - canonicalize - canonicalize all extracted urls using scrapy.utils.url.canonicalize_url - - Both 'allow' and 'deny' arguments can be a list of regexes strings or regex - python objects (already compiled) - - Url matching is always performed against the absolute urls, never the - relative urls found in pages. - - If no allow/deny arguments are given, match all links. - """ def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True):