From 20a8237910dbdde78fcda2e9101e175432b41b44 Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Thu, 6 Mar 2014 12:41:21 -0300 Subject: [PATCH] Running lucasdemarchi/codespell to fix typos in SEPs --- sep/sep-003.trac | 306 ++++++------ sep/sep-014.trac | 1224 +++++++++++++++++++++++----------------------- sep/sep-016.trac | 528 ++++++++++---------- sep/sep-018.trac | 1102 ++++++++++++++++++++--------------------- 4 files changed, 1580 insertions(+), 1580 deletions(-) diff --git a/sep/sep-003.trac b/sep/sep-003.trac index f0b910f61..dd40d9510 100644 --- a/sep/sep-003.trac +++ b/sep/sep-003.trac @@ -1,153 +1,153 @@ -= SEP-003 - Nested items API (!ItemField) = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||3|| -||'''Title:'''||Nested items API (!ItemField) || -||'''Author:'''||Pablo Hoffman|| -||'''Created:'''||2009-07-19|| -||'''Status'''||Obsoleted by [wiki:SEP-008]|| - -== Introduction == - -This page presents different usage scenarios for the new nested items field API called !ItemField. - -== Prerequisites == - -This API proposal relies on the following API: - - 1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance) - 2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}} - -== Proposed Implementation of !ItemField == - -{{{ -#!python -from scrapy.item.fields import BaseField - -class ItemField(BaseField): - def __init__(self, item_type, default=None): - self._item_type = item_type - super(ItemField, self).__init__(default) - - def to_python(self, value): - return self._item_type(value) if not isinstance(value, self._item_type) else value - - def get_default(self): - # WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors - # always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour - return self._default -}}} - -== Usage Scenarios == - -=== Defining an item containing !ItemField's === - -{{{ -#!python -from scrapy.item.models import Item -from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField - -class Supplier(Item): - name = TextField(default="anonymous supplier") - url = UrlField() - -class Variant(Item): - name = TextField(required=True) - url = UrlField() - price = DecimalField() - -class Product(Variant): - supplier = ItemField(Supplier, default=Supplier(name="default supplier") - variants = ListField(ItemField(Variant)) - - # these ones are used for documenting default value examples - supplier2 = ItemField(Supplier) - variants2 = ListField(ItemField(Variant), default=[]) -}}} - -It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant -relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile: - -{{{ -#!python -class Product(Item): - variants = ItemField(Product) # Fails to compile -}}} - -=== Assigning an item field === - -{{{ -#!python - -supplier = Supplier(name="Supplier 1", url="http://example.com") - -p = Product() - -# standard assignment -p['supplier'] = supplier -# this also works as it tries to instantiate a Supplier with the given dict -p['supplier'] = {'name': 'Supplier 1' url='http://example.com'} -# this fails because it can't instantiate a Supplier -p['supplier'] = 'Supplier 1' -# this fails because url doesn't have the valid type -p['supplier'] = {'name': 'Supplier 1' url=123} - -v1 = Variant() -v1['name'] = "lala" -v1['price'] = Decimal("100") - -v2 = Variant() -v2['name'] = "lolo" -v2['price'] = Decimal("150") - -# standard assignment -p['variants'] = [v1, v2] # OK -# can also instantiate at assignment time -p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")] -# this also works as it tries to instantiate a Variant with the given dict -p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")] -# this fails because it can't instantiate a Variant -p['variants'] = [v1, 'test'] -# this fails beacuse 'coco' is not a valid value for price -p['variants'] = [v1, {'name': 'lolo', 'price': 'coco'] -}}} - -=== Default values === - -{{{ -#!python -p = Product() - -p['supplier'] # returns: Supplier(name='default supplier') -p['supplier2'] # raises KeyError -p['supplier2'] = Supplier() -p['supplier2'] # returns: Supplier(name='anonymous supplier') - -p['variants'] # raises KeyError -p['variants2'] # returns [] - -p['categories'] # raises KeyError -p.get('categories') # returns None - -p['numbers'] # returns [] -}}} - -=== Accesing and changing nested item values === - -{{{ -#!python - -p = Product(supplier=Supplier(name="some name", url="http://example.com")) -p['supplier']['url'] # returns 'http://example.com' -p['supplier']['url'] = "http://www.other.com" # works as expected -p['supplier']['url'] = 123 # fails: wrong type for supplier url - -p['variants'] = [v1, v2] -p['variants'][0]['name'] # returns v1 name -p['variants'][1]['name'] # returns v2 name - -# XXX: decide what to do about these cases: -p['variants'].append(v3) # works but doesn't check type of v3 -p['variants'].append(1) # works but shouldn't? -}}} += SEP-003 - Nested items API (!ItemField) = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||3|| +||'''Title:'''||Nested items API (!ItemField) || +||'''Author:'''||Pablo Hoffman|| +||'''Created:'''||2009-07-19|| +||'''Status'''||Obsoleted by [wiki:SEP-008]|| + +== Introduction == + +This page presents different usage scenarios for the new nested items field API called !ItemField. + +== Prerequisites == + +This API proposal relies on the following API: + + 1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance) + 2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}} + +== Proposed Implementation of !ItemField == + +{{{ +#!python +from scrapy.item.fields import BaseField + +class ItemField(BaseField): + def __init__(self, item_type, default=None): + self._item_type = item_type + super(ItemField, self).__init__(default) + + def to_python(self, value): + return self._item_type(value) if not isinstance(value, self._item_type) else value + + def get_default(self): + # WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors + # always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour + return self._default +}}} + +== Usage Scenarios == + +=== Defining an item containing !ItemField's === + +{{{ +#!python +from scrapy.item.models import Item +from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField + +class Supplier(Item): + name = TextField(default="anonymous supplier") + url = UrlField() + +class Variant(Item): + name = TextField(required=True) + url = UrlField() + price = DecimalField() + +class Product(Variant): + supplier = ItemField(Supplier, default=Supplier(name="default supplier") + variants = ListField(ItemField(Variant)) + + # these ones are used for documenting default value examples + supplier2 = ItemField(Supplier) + variants2 = ListField(ItemField(Variant), default=[]) +}}} + +It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant +relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile: + +{{{ +#!python +class Product(Item): + variants = ItemField(Product) # Fails to compile +}}} + +=== Assigning an item field === + +{{{ +#!python + +supplier = Supplier(name="Supplier 1", url="http://example.com") + +p = Product() + +# standard assignment +p['supplier'] = supplier +# this also works as it tries to instantiate a Supplier with the given dict +p['supplier'] = {'name': 'Supplier 1' url='http://example.com'} +# this fails because it can't instantiate a Supplier +p['supplier'] = 'Supplier 1' +# this fails because url doesn't have the valid type +p['supplier'] = {'name': 'Supplier 1' url=123} + +v1 = Variant() +v1['name'] = "lala" +v1['price'] = Decimal("100") + +v2 = Variant() +v2['name'] = "lolo" +v2['price'] = Decimal("150") + +# standard assignment +p['variants'] = [v1, v2] # OK +# can also instantiate at assignment time +p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")] +# this also works as it tries to instantiate a Variant with the given dict +p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")] +# this fails because it can't instantiate a Variant +p['variants'] = [v1, 'test'] +# this fails because 'coco' is not a valid value for price +p['variants'] = [v1, {'name': 'lolo', 'price': 'coco'] +}}} + +=== Default values === + +{{{ +#!python +p = Product() + +p['supplier'] # returns: Supplier(name='default supplier') +p['supplier2'] # raises KeyError +p['supplier2'] = Supplier() +p['supplier2'] # returns: Supplier(name='anonymous supplier') + +p['variants'] # raises KeyError +p['variants2'] # returns [] + +p['categories'] # raises KeyError +p.get('categories') # returns None + +p['numbers'] # returns [] +}}} + +=== Accesing and changing nested item values === + +{{{ +#!python + +p = Product(supplier=Supplier(name="some name", url="http://example.com")) +p['supplier']['url'] # returns 'http://example.com' +p['supplier']['url'] = "http://www.other.com" # works as expected +p['supplier']['url'] = 123 # fails: wrong type for supplier url + +p['variants'] = [v1, v2] +p['variants'][0]['name'] # returns v1 name +p['variants'][1]['name'] # returns v2 name + +# XXX: decide what to do about these cases: +p['variants'].append(v3) # works but doesn't check type of v3 +p['variants'].append(1) # works but shouldn't? +}}} diff --git a/sep/sep-014.trac b/sep/sep-014.trac index 5b19784f0..1df94b3a4 100644 --- a/sep/sep-014.trac +++ b/sep/sep-014.trac @@ -1,612 +1,612 @@ -= SEP-014 - !CrawlSpider v2 = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||14|| -||'''Title:'''||!CrawlSpider v2|| -||'''Author:'''||Insophia Team|| -||'''Created:'''||2010-01-22|| -||'''Updated:'''||2010-02-04|| -||'''Status'''||Final. Partially implemented but discarded because of lack of use in r2632|| - -== Introduction == - -This SEP proposes a rewrite of Scrapy !CrawlSpider and related components - -== Current flaws and inconsistencies == - - 1. Request's callbacks are hard to persist. - 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) - 3. Isn't possible to crawl an url directly from command line because the Spider does not know which callback use. - -These flaws will be corrected by the changes proposed in this SEP. - -== Proposed API Changes == - - * Separate the functionality of Rule-!LinkExtractor-Callback - * Separate the functionality of !LinkExtractor to Request Extractor and Request Processor - * Separate the process of determining response callback and the extraction of new requests (link extractors) - * The callback will be determine by Matcher Objects on request/response objects - -=== Matcher Objects === - -Matcher Objects (aka Matcher) are responsible for determining if given request or response matches an arbitrary criteria. -The Matcher receives as argument the request or the response, giving a powerful access to all request/response attributes. - -In the current !CrawlSpider, the Rule Object has the responsability to determine the callback of given extractor, -and the link extractor contains the url pattern (aka regex). Now the Matcher will contain only the pattern or criteria -to determine which request/response will execute any action. See below Spider Rules. - -=== Request Extractors === - -Request Extractors takes response object and determines which requests follow. - -This is an enhancemente to !LinkExtractors which returns urls (links), Request Extractors -return Request objects. - -=== Request Processors === - -Request Processors takes requests objects and can perform any action to them, like filtering -or modifying on the fly. - -The current !LinkExtractor had integrated link processing, like canonicalize. Request Processors -can be reutilized and applied in serie. - -=== Request Generator === - -Request Generator is the decoupling of the !CrawlSpider's method _request_to_follow(). -Request Generator takes the response object and applies the Request Extractors and Request Processors. - -=== Rules Manager === - -The Rules are a definition of Rule objects containing Matcher Objects and callback. - -The Legacy Rules were used to perform the link extraction and attach the callback to the generated Request object. -The proposed new Rules will be used to determine the callback for given response. This opens a whole of opportunities, -like determine the callback for given url, and persist the queue of Request objects because the callback is determined -the matching the Response object against the Rules. - -== Usage Examples == - -=== Basic Crawling === - -{{{ -#!python -# -# Basic Crawling -# -class SampleSpider(CrawlSpider): - rules = [ - # The dispatcher uses first-match policy - Rule(UrlRegexMatch(r'product\.html\?id=\d+'), 'parse_item', follow=False), - # by default, if the first param is string is wrapped into UrlRegexMatch - Rule(r'.+', 'parse_page'), - ] - - request_extractors = [ - # crawl all links looking for products and images - SgmlRequestExtractor(), - ] - - request_processors = [ - # canonicalize all requests' urls - Canonicalize(), - ] - - def parse_item(self, response): - # parse and extract items from response - pass - - def parse_page(self, response): - # extract images on all pages - pass -}}} - -=== Custom Processor and External Callback === - -{{{ -#!python -# -# Using external callbacks -# - -# Custom Processor -def filter_today_links(requests): - # only crawl today links - today = datetime.datetime.today().strftime('%Y-%m-%d') - return [r for r in requests if today in r.url] - -# Callback defined out of spider -def my_external_callback(response): - # process item - pass - -class SampleSpider(CrawlSpider): - rules = [ - # The dispatcher uses first-match policy - Rule(UrlRegexMatch(r'/news/(.+)/'), my_external_callback), - ] - - request_extractors = [ - RegexRequestExtractor(r'/sections/.+'), - RegexRequestExtractor(r'/news/.+'), - ] - - request_processors = [ - # canonicalize all requests' urls - Canonicalize(), - filter_today_links, - ] - -}}} - -== Implementation == - -*Work-in-progress* - -=== Package Structure === -{{{ -contrib_exp - |- crawlspider/ - |- spider.py - |- CrawlSpider - |- rules.py - |- Rule - |- CompiledRule - |- RulesManager - |- reqgen.py - |- RequestGenerator - |- reqproc.py - |- Canonicalize - |- Unique - |- ... - |- reqext.py - |- SgmlRequestExtractor - |- RegexRequestExtractor - |- ... - |- matchers.py - |- BaseMatcher - |- UrlMatcher - |- UrlRegexMatcher - |- ... -}}} - -=== Request/Response Matchers === -{{{ -#!python -""" -Request/Response Matchers - -Perform evaluation to Request or Response attributes -""" - -class BaseMatcher(object): - """Base matcher. Returns True by default.""" - - def matches_request(self, request): - """Performs Request Matching""" - return True - - def matches_response(self, response): - """Performs Response Matching""" - return True - - -class UrlMatcher(BaseMatcher): - """Matches URL attribute""" - - def __init__(self, url): - """Initialize url attribute""" - self._url = url - - def matches_url(self, url): - """Returns True if given url is equal to matcher's url""" - return self._url == url - - def matches_request(self, request): - """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) - - def matches_response(self, response): - """REturns True if Response's url matches initial url""" - return self.matches_url(response.url) - - -class UrlRegexMatcher(UrlMatcher): - """Matches URL using regular expression""" - - def __init__(self, regex, flags=0): - """Initialize regular expression""" - self._regex = re.compile(regex, flags) - - def matches_url(self, url): - """Returns True if url matches regular expression""" - return self._regex.search(url) is not None -}}} - -=== Request Extractor === -{{{ -#!python -# -# Requests Extractor -# Extractors receive response and return list of Requests -# - -class BaseSgmlRequestExtractor(FixedSGMLParser): - """Base SGML Request Extractor""" - - def __init__(self, tag='a', attr='href'): - """Initialize attributes""" - FixedSGMLParser.__init__(self) - - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.current_request = None - - def extract_requests(self, response): - """Returns list of requests extracted from response""" - return self._extract_requests(response.body, response.url, - response.encoding) - - def _extract_requests(self, response_text, response_url, response_encoding): - """Extract requests with absolute urls""" - self.reset() - self.feed(response_text) - self.close() - - base_url = self.base_url if self.base_url else response_url - self._make_absolute_urls(base_url, response_encoding) - self._fix_link_text_encoding(response_encoding) - - return self.requests - - def _make_absolute_urls(self, base_url, encoding): - """Makes all request's urls absolute""" - for req in self.requests: - url = req.url - # make absolute url - url = urljoin_rfc(base_url, url, encoding) - url = safe_url_string(url, encoding) - # replace in-place request's url - req.url = url - - def _fix_link_text_encoding(self, encoding): - """Convert link_text to unicode for each request""" - for req in self.requests: - req.meta.setdefault('link_text', '') - req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) - - def reset(self): - """Reset state""" - FixedSGMLParser.reset(self) - self.requests = [] - self.base_url = None - - def unknown_starttag(self, tag, attrs): - """Process unknown start tag""" - if 'base' == tag: - self.base_url = dict(attrs).get('href') - - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - if value is not None: - req = Request(url=value) - self.requests.append(req) - self.current_request = req - - def unknown_endtag(self, tag): - """Process unknown end tag""" - self.current_request = None - - def handle_data(self, data): - """Process data""" - current = self.current_request - if current and not 'link_text' in current.meta: - current.meta['link_text'] = data.strip() - - -class SgmlRequestExtractor(BaseSgmlRequestExtractor): - """SGML Request Extractor""" - - def __init__(self, tags=None, attrs=None): - """Initialize with custom tag & attribute function checkers""" - # defaults - tags = tuple(tags) if tags else ('a', 'area') - attrs = tuple(attrs) if attrs else ('href', ) - - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs - BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) - - -class XPathRequestExtractor(SgmlRequestExtractor): - """SGML Request Extractor with XPath restriction""" - - def __init__(self, restrict_xpaths, tags=None, attrs=None): - """Initialize XPath restrictions""" - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - SgmlRequestExtractor.__init__(self, tags, attrs) - - def extract_requests(self, response): - """Restrict to XPath regions""" - hxs = HtmlXPathSelector(response) - fragments = (''.join( - html_frag for html_frag in hxs.select(xpath).extract() - ) for xpath in self.restrict_xpaths) - html_slice = ''.join(html_frag for html_frag in fragments) - return self._extract_requests(html_slice, response.url, - response.encoding) - -}}} - -=== Request Processor === -{{{ -#!python -# -# Request Processors -# Processors receive list of requests and return list of requests -# -"""Request Processors""" - -class Canonicalize(object): - """Canonicalize Request Processor""" - - def __call__(self, requests): - """Canonicalize all requests' urls""" - for req in requests: - # replace in-place - req.url = canonicalize_url(req.url) - yield req - - -class Unique(object): - """Filter duplicate Requests""" - - def __init__(self, *attributes): - """Initialize comparison attributes""" - self._attributes = attributes or ['url'] - - def _requests_equal(self, req1, req2): - """Attribute comparison helper""" - for attr in self._attributes: - if getattr(req1, attr) != getattr(req2, attr): - return False - # all attributes equal - return True - - def _request_in(self, request, requests_seen): - """Check if request is in given requests seen list""" - for seen in requests_seen: - if self._requests_equal(request, seen): - return True - # request not seen - return False - - def __call__(self, requests): - """Filter seen requests""" - # per-call duplicates filter - requests_seen = set() - for req in requests: - if not self._request_in(req, requests_seen): - yield req - # registry seen request - requests_seen.add(req) - - -class FilterDomain(object): - """Filter request's domain""" - - def __init__(self, allow=(), deny=()): - """Initialize allow/deny attributes""" - self.allow = tuple(arg_to_iter(allow)) - self.deny = tuple(arg_to_iter(deny)) - - def __call__(self, requests): - """Filter domains""" - processed = (req for req in requests) - - if self.allow: - processed = (req for req in requests - if url_is_from_any_domain(req.url, self.allow)) - if self.deny: - processed = (req for req in requests - if not url_is_from_any_domain(req.url, self.deny)) - - return processed - - -class FilterUrl(object): - """Filter request's url""" - - def __init__(self, allow=(), deny=()): - """Initialize allow/deny attributes""" - _re_type = type(re.compile('', 0)) - - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] - - def __call__(self, requests): - """Filter request's url based on allow/deny rules""" - #TODO: filter valid urls here? - processed = (req for req in requests) - - if self.allow_res: - processed = (req for req in requests - if self._matches(req.url, self.allow_res)) - if self.deny_res: - processed = (req for req in requests - if not self._matches(req.url, self.deny_res)) - - return processed - - def _matches(self, url, regexs): - """Returns True if url matches any regex in given list""" - return any(r.search(url) for r in regexs) - - -}}} - -=== Rule Object === -{{{ -#!python -# -# Dispatch Rules classes -# Manage Rules (Matchers + Callbacks) -# -class Rule(object): - """Crawler Rule""" - def __init__(self, matcher, callback=None, cb_args=None, - cb_kwargs=None, follow=True): - """Store attributes""" - self.matcher = matcher - self.callback = callback - self.cb_args = cb_args if cb_args else () - self.cb_kwargs = cb_kwargs if cb_kwargs else {} - self.follow = follow - -# -# Rules Manager takes list of Rule objects and normalize matcher and callback -# into CompiledRule -# -class CompiledRule(object): - """Compiled version of Rule""" - def __init__(self, matcher, callback=None, follow=False): - """Initialize attributes checking type""" - assert isinstance(matcher, BaseMatcher) - assert callback is None or callable(callback) - assert isinstance(follow, bool) - - self.matcher = matcher - self.callback = callback - self.follow = follow -}}} - -=== Rules Manager === -{{{ -#!python -# -# Handles rules matcher/callbacks -# Resolve rule for given response -# -class RulesManager(object): - """Rules Manager""" - def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): - """Initialize rules using spider and default matcher""" - self._rules = tuple() - - # compile absolute/relative-to-spider callbacks""" - for rule in rules: - # prepare matcher - if isinstance(rule.matcher, BaseMatcher): - matcher = rule.matcher - else: - # matcher not BaseMatcher, check for string - if isinstance(rule.matcher, basestring): - # instance default matcher - matcher = default_matcher(rule.matcher) - else: - raise ValueError('Not valid matcher given %r in %r' \ - % (rule.matcher, rule)) - - # prepare callback - if callable(rule.callback): - callback = rule.callback - elif not rule.callback is None: - # callback from spider - callback = getattr(spider, rule.callback) - - if not callable(callback): - raise AttributeError('Invalid callback %r can not be resolved' \ - % callback) - else: - callback = None - - if rule.cb_args or rule.cb_kwargs: - # build partial callback - callback = partial(callback, *rule.cb_args, **rule.cb_kwargs) - - # append compiled rule to rules list - crule = CompiledRule(matcher, callback, follow=rule.follow) - self._rules += (crule, ) - - def get_rule(self, response): - """Returns first rule that matches response""" - for rule in self._rules: - if rule.matcher.matches_response(response): - return rule - -}}} - -=== Request Generator === -{{{ -#!python -# -# Request Generator -# Takes response and generate requests using extractors and processors -# -class RequestGenerator(object): - def __init__(self, req_extractors, req_processors, callback): - self._request_extractors = req_extractors - self._request_processors = req_processors - self.callback = callback - - def generate_requests(self, response): - """ - Extract and process new requets from response - """ - requests = [] - for ext in self._request_extractors: - requets.extend(ext.extract_requests(response)) - - for proc in self._request_processors: - requests = proc(requests) - - for request in requests: - yield request.replace(callback=self.callback) -}}} - -=== !CrawlSpider === -{{{ -#!python -# -# Spider -# -class CrawlSpider(InitSpider): - """CrawlSpider v2""" - - request_extractors = [] - request_processors = [] - rules = [] - - def __init__(self): - """Initialize dispatcher""" - super(CrawlSpider, self).__init__() - - # wrap rules - self._rulesman = RulesManager(self.rules, spider=self) - # generates new requests with given callback - self._reqgen = RequestGenerator(self.request_extractors, - self.request_processors, - self.parse) - - def parse(self, response): - """Dispatch callback and generate requests""" - # get rule for response - rule = self._rulesman.get_rule(response) - if rule: - # dispatch callback if set - if rule.callback: - output = iterate_spider_output(rule.callback(response)) - for req_or_item in output: - yield req_or_item - - if rule.follow: - for req in self._reqgen.generate_requests(response): - yield req - -}}} - += SEP-014 - !CrawlSpider v2 = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||14|| +||'''Title:'''||!CrawlSpider v2|| +||'''Author:'''||Insophia Team|| +||'''Created:'''||2010-01-22|| +||'''Updated:'''||2010-02-04|| +||'''Status'''||Final. Partially implemented but discarded because of lack of use in r2632|| + +== Introduction == + +This SEP proposes a rewrite of Scrapy !CrawlSpider and related components + +== Current flaws and inconsistencies == + + 1. Request's callbacks are hard to persist. + 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) + 3. Isn't possible to crawl an url directly from command line because the Spider does not know which callback use. + +These flaws will be corrected by the changes proposed in this SEP. + +== Proposed API Changes == + + * Separate the functionality of Rule-!LinkExtractor-Callback + * Separate the functionality of !LinkExtractor to Request Extractor and Request Processor + * Separate the process of determining response callback and the extraction of new requests (link extractors) + * The callback will be determine by Matcher Objects on request/response objects + +=== Matcher Objects === + +Matcher Objects (aka Matcher) are responsible for determining if given request or response matches an arbitrary criteria. +The Matcher receives as argument the request or the response, giving a powerful access to all request/response attributes. + +In the current !CrawlSpider, the Rule Object has the responsibility to determine the callback of given extractor, +and the link extractor contains the url pattern (aka regex). Now the Matcher will contain only the pattern or criteria +to determine which request/response will execute any action. See below Spider Rules. + +=== Request Extractors === + +Request Extractors takes response object and determines which requests follow. + +This is an enhancemente to !LinkExtractors which returns urls (links), Request Extractors +return Request objects. + +=== Request Processors === + +Request Processors takes requests objects and can perform any action to them, like filtering +or modifying on the fly. + +The current !LinkExtractor had integrated link processing, like canonicalize. Request Processors +can be reutilized and applied in serie. + +=== Request Generator === + +Request Generator is the decoupling of the !CrawlSpider's method _request_to_follow(). +Request Generator takes the response object and applies the Request Extractors and Request Processors. + +=== Rules Manager === + +The Rules are a definition of Rule objects containing Matcher Objects and callback. + +The Legacy Rules were used to perform the link extraction and attach the callback to the generated Request object. +The proposed new Rules will be used to determine the callback for given response. This opens a whole of opportunities, +like determine the callback for given url, and persist the queue of Request objects because the callback is determined +the matching the Response object against the Rules. + +== Usage Examples == + +=== Basic Crawling === + +{{{ +#!python +# +# Basic Crawling +# +class SampleSpider(CrawlSpider): + rules = [ + # The dispatcher uses first-match policy + Rule(UrlRegexMatch(r'product\.html\?id=\d+'), 'parse_item', follow=False), + # by default, if the first param is string is wrapped into UrlRegexMatch + Rule(r'.+', 'parse_page'), + ] + + request_extractors = [ + # crawl all links looking for products and images + SgmlRequestExtractor(), + ] + + request_processors = [ + # canonicalize all requests' urls + Canonicalize(), + ] + + def parse_item(self, response): + # parse and extract items from response + pass + + def parse_page(self, response): + # extract images on all pages + pass +}}} + +=== Custom Processor and External Callback === + +{{{ +#!python +# +# Using external callbacks +# + +# Custom Processor +def filter_today_links(requests): + # only crawl today links + today = datetime.datetime.today().strftime('%Y-%m-%d') + return [r for r in requests if today in r.url] + +# Callback defined out of spider +def my_external_callback(response): + # process item + pass + +class SampleSpider(CrawlSpider): + rules = [ + # The dispatcher uses first-match policy + Rule(UrlRegexMatch(r'/news/(.+)/'), my_external_callback), + ] + + request_extractors = [ + RegexRequestExtractor(r'/sections/.+'), + RegexRequestExtractor(r'/news/.+'), + ] + + request_processors = [ + # canonicalize all requests' urls + Canonicalize(), + filter_today_links, + ] + +}}} + +== Implementation == + +*Work-in-progress* + +=== Package Structure === +{{{ +contrib_exp + |- crawlspider/ + |- spider.py + |- CrawlSpider + |- rules.py + |- Rule + |- CompiledRule + |- RulesManager + |- reqgen.py + |- RequestGenerator + |- reqproc.py + |- Canonicalize + |- Unique + |- ... + |- reqext.py + |- SgmlRequestExtractor + |- RegexRequestExtractor + |- ... + |- matchers.py + |- BaseMatcher + |- UrlMatcher + |- UrlRegexMatcher + |- ... +}}} + +=== Request/Response Matchers === +{{{ +#!python +""" +Request/Response Matchers + +Perform evaluation to Request or Response attributes +""" + +class BaseMatcher(object): + """Base matcher. Returns True by default.""" + + def matches_request(self, request): + """Performs Request Matching""" + return True + + def matches_response(self, response): + """Performs Response Matching""" + return True + + +class UrlMatcher(BaseMatcher): + """Matches URL attribute""" + + def __init__(self, url): + """Initialize url attribute""" + self._url = url + + def matches_url(self, url): + """Returns True if given url is equal to matcher's url""" + return self._url == url + + def matches_request(self, request): + """Returns True if Request's url matches initial url""" + return self.matches_url(request.url) + + def matches_response(self, response): + """REturns True if Response's url matches initial url""" + return self.matches_url(response.url) + + +class UrlRegexMatcher(UrlMatcher): + """Matches URL using regular expression""" + + def __init__(self, regex, flags=0): + """Initialize regular expression""" + self._regex = re.compile(regex, flags) + + def matches_url(self, url): + """Returns True if url matches regular expression""" + return self._regex.search(url) is not None +}}} + +=== Request Extractor === +{{{ +#!python +# +# Requests Extractor +# Extractors receive response and return list of Requests +# + +class BaseSgmlRequestExtractor(FixedSGMLParser): + """Base SGML Request Extractor""" + + def __init__(self, tag='a', attr='href'): + """Initialize attributes""" + FixedSGMLParser.__init__(self) + + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.current_request = None + + def extract_requests(self, response): + """Returns list of requests extracted from response""" + return self._extract_requests(response.body, response.url, + response.encoding) + + def _extract_requests(self, response_text, response_url, response_encoding): + """Extract requests with absolute urls""" + self.reset() + self.feed(response_text) + self.close() + + base_url = self.base_url if self.base_url else response_url + self._make_absolute_urls(base_url, response_encoding) + self._fix_link_text_encoding(response_encoding) + + return self.requests + + def _make_absolute_urls(self, base_url, encoding): + """Makes all request's urls absolute""" + for req in self.requests: + url = req.url + # make absolute url + url = urljoin_rfc(base_url, url, encoding) + url = safe_url_string(url, encoding) + # replace in-place request's url + req.url = url + + def _fix_link_text_encoding(self, encoding): + """Convert link_text to unicode for each request""" + for req in self.requests: + req.meta.setdefault('link_text', '') + req.meta['link_text'] = str_to_unicode(req.meta['link_text'], + encoding) + + def reset(self): + """Reset state""" + FixedSGMLParser.reset(self) + self.requests = [] + self.base_url = None + + def unknown_starttag(self, tag, attrs): + """Process unknown start tag""" + if 'base' == tag: + self.base_url = dict(attrs).get('href') + + if self.scan_tag(tag): + for attr, value in attrs: + if self.scan_attr(attr): + if value is not None: + req = Request(url=value) + self.requests.append(req) + self.current_request = req + + def unknown_endtag(self, tag): + """Process unknown end tag""" + self.current_request = None + + def handle_data(self, data): + """Process data""" + current = self.current_request + if current and not 'link_text' in current.meta: + current.meta['link_text'] = data.strip() + + +class SgmlRequestExtractor(BaseSgmlRequestExtractor): + """SGML Request Extractor""" + + def __init__(self, tags=None, attrs=None): + """Initialize with custom tag & attribute function checkers""" + # defaults + tags = tuple(tags) if tags else ('a', 'area') + attrs = tuple(attrs) if attrs else ('href', ) + + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) + + +class XPathRequestExtractor(SgmlRequestExtractor): + """SGML Request Extractor with XPath restriction""" + + def __init__(self, restrict_xpaths, tags=None, attrs=None): + """Initialize XPath restrictions""" + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + SgmlRequestExtractor.__init__(self, tags, attrs) + + def extract_requests(self, response): + """Restrict to XPath regions""" + hxs = HtmlXPathSelector(response) + fragments = (''.join( + html_frag for html_frag in hxs.select(xpath).extract() + ) for xpath in self.restrict_xpaths) + html_slice = ''.join(html_frag for html_frag in fragments) + return self._extract_requests(html_slice, response.url, + response.encoding) + +}}} + +=== Request Processor === +{{{ +#!python +# +# Request Processors +# Processors receive list of requests and return list of requests +# +"""Request Processors""" + +class Canonicalize(object): + """Canonicalize Request Processor""" + + def __call__(self, requests): + """Canonicalize all requests' urls""" + for req in requests: + # replace in-place + req.url = canonicalize_url(req.url) + yield req + + +class Unique(object): + """Filter duplicate Requests""" + + def __init__(self, *attributes): + """Initialize comparison attributes""" + self._attributes = attributes or ['url'] + + def _requests_equal(self, req1, req2): + """Attribute comparison helper""" + for attr in self._attributes: + if getattr(req1, attr) != getattr(req2, attr): + return False + # all attributes equal + return True + + def _request_in(self, request, requests_seen): + """Check if request is in given requests seen list""" + for seen in requests_seen: + if self._requests_equal(request, seen): + return True + # request not seen + return False + + def __call__(self, requests): + """Filter seen requests""" + # per-call duplicates filter + requests_seen = set() + for req in requests: + if not self._request_in(req, requests_seen): + yield req + # registry seen request + requests_seen.add(req) + + +class FilterDomain(object): + """Filter request's domain""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + self.allow = tuple(arg_to_iter(allow)) + self.deny = tuple(arg_to_iter(deny)) + + def __call__(self, requests): + """Filter domains""" + processed = (req for req in requests) + + if self.allow: + processed = (req for req in requests + if url_is_from_any_domain(req.url, self.allow)) + if self.deny: + processed = (req for req in requests + if not url_is_from_any_domain(req.url, self.deny)) + + return processed + + +class FilterUrl(object): + """Filter request's url""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + _re_type = type(re.compile('', 0)) + + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + def __call__(self, requests): + """Filter request's url based on allow/deny rules""" + #TODO: filter valid urls here? + processed = (req for req in requests) + + if self.allow_res: + processed = (req for req in requests + if self._matches(req.url, self.allow_res)) + if self.deny_res: + processed = (req for req in requests + if not self._matches(req.url, self.deny_res)) + + return processed + + def _matches(self, url, regexs): + """Returns True if url matches any regex in given list""" + return any(r.search(url) for r in regexs) + + +}}} + +=== Rule Object === +{{{ +#!python +# +# Dispatch Rules classes +# Manage Rules (Matchers + Callbacks) +# +class Rule(object): + """Crawler Rule""" + def __init__(self, matcher, callback=None, cb_args=None, + cb_kwargs=None, follow=True): + """Store attributes""" + self.matcher = matcher + self.callback = callback + self.cb_args = cb_args if cb_args else () + self.cb_kwargs = cb_kwargs if cb_kwargs else {} + self.follow = follow + +# +# Rules Manager takes list of Rule objects and normalize matcher and callback +# into CompiledRule +# +class CompiledRule(object): + """Compiled version of Rule""" + def __init__(self, matcher, callback=None, follow=False): + """Initialize attributes checking type""" + assert isinstance(matcher, BaseMatcher) + assert callback is None or callable(callback) + assert isinstance(follow, bool) + + self.matcher = matcher + self.callback = callback + self.follow = follow +}}} + +=== Rules Manager === +{{{ +#!python +# +# Handles rules matcher/callbacks +# Resolve rule for given response +# +class RulesManager(object): + """Rules Manager""" + def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): + """Initialize rules using spider and default matcher""" + self._rules = tuple() + + # compile absolute/relative-to-spider callbacks""" + for rule in rules: + # prepare matcher + if isinstance(rule.matcher, BaseMatcher): + matcher = rule.matcher + else: + # matcher not BaseMatcher, check for string + if isinstance(rule.matcher, basestring): + # instance default matcher + matcher = default_matcher(rule.matcher) + else: + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) + + # prepare callback + if callable(rule.callback): + callback = rule.callback + elif not rule.callback is None: + # callback from spider + callback = getattr(spider, rule.callback) + + if not callable(callback): + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) + else: + callback = None + + if rule.cb_args or rule.cb_kwargs: + # build partial callback + callback = partial(callback, *rule.cb_args, **rule.cb_kwargs) + + # append compiled rule to rules list + crule = CompiledRule(matcher, callback, follow=rule.follow) + self._rules += (crule, ) + + def get_rule(self, response): + """Returns first rule that matches response""" + for rule in self._rules: + if rule.matcher.matches_response(response): + return rule + +}}} + +=== Request Generator === +{{{ +#!python +# +# Request Generator +# Takes response and generate requests using extractors and processors +# +class RequestGenerator(object): + def __init__(self, req_extractors, req_processors, callback): + self._request_extractors = req_extractors + self._request_processors = req_processors + self.callback = callback + + def generate_requests(self, response): + """ + Extract and process new requets from response + """ + requests = [] + for ext in self._request_extractors: + requets.extend(ext.extract_requests(response)) + + for proc in self._request_processors: + requests = proc(requests) + + for request in requests: + yield request.replace(callback=self.callback) +}}} + +=== !CrawlSpider === +{{{ +#!python +# +# Spider +# +class CrawlSpider(InitSpider): + """CrawlSpider v2""" + + request_extractors = [] + request_processors = [] + rules = [] + + def __init__(self): + """Initialize dispatcher""" + super(CrawlSpider, self).__init__() + + # wrap rules + self._rulesman = RulesManager(self.rules, spider=self) + # generates new requests with given callback + self._reqgen = RequestGenerator(self.request_extractors, + self.request_processors, + self.parse) + + def parse(self, response): + """Dispatch callback and generate requests""" + # get rule for response + rule = self._rulesman.get_rule(response) + if rule: + # dispatch callback if set + if rule.callback: + output = iterate_spider_output(rule.callback(response)) + for req_or_item in output: + yield req_or_item + + if rule.follow: + for req in self._reqgen.generate_requests(response): + yield req + +}}} + diff --git a/sep/sep-016.trac b/sep/sep-016.trac index 9f7ada3c6..e9a0e5df0 100644 --- a/sep/sep-016.trac +++ b/sep/sep-016.trac @@ -1,265 +1,265 @@ -= SEP-016: Leg Spider = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||16|| -||'''Title:'''||Leg Spider|| -||'''Author:'''||Insophia Team|| -||'''Created:'''||2010-06-03|| -||'''Status'''||Superseded by [wiki:SEP-018]|| - -== Introduction == - -This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders. - -== Rationale == - -The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. - -The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider". - -== {{{LegSpider}}} API == - -A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods: - - * {{{legs}}} - * legs composing this spider - * {{{process_response(response)}}} - * Process a (downloaded) response and return a list of requests and items - * {{{process_request(request)}}} - * Process a request after it has been extracted and before returning it from the spider - * {{{process_item(item)}}} - * Process an item after it has been extracted and before returning it from the spider - * {{{set_spider()}}} - * Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior. - -== How Leg Spiders work == - - 1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output. - 2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself. - -== Leg Spider examples == - -=== Regex (HTML) Link Extractor === - -A typical application of LegSpider's is to build Link Extractors. For example: - -{{{ -#!python -class RegexHtmlLinkExtractor(LegSpider): - - def process_response(self, response): - if isinstance(response, HtmlResponse): - allowed_regexes = self.spider.url_regexes_to_follow - # extract urls to follow using allowed_regexes - return [Request(x) for x in urls_to_follow] - -class MySpider(LegSpider): - - legs = [RegexHtmlLinkExtractor()] - url_regexes_to_follow = ['/product.php?.*'] - - def parse_response(self, response): - # parse response and extract items - return items -}}} - -=== RSS2 link extractor === - -This is a Leg Spider that can be used for following links from RSS2 feeds. - -{{{ -#!python -class Rss2LinkExtractor(LegSpider): - - def process_response(self, response): - if response.headers.get('Content-type') == 'application/rss+xml': - xs = XmlXPathSelector(response) - urls = xs.select("//item/link/text()").extract() - return [Request(x) for x in urls] -}}} - -=== Callback dispatcher based on rules === - -Another example could be to build a callback dispatcher based on rules: - -{{{ -#!python -class CallbackRules(LegSpider): - - def __init__(self, *a, **kw): - super(CallbackRules, self).__init__(*a, **kw) - for regex, method_name in self.spider.callback_rules.items(): - r = re.compile(regex) - m = getattr(self.spider, method_name, None) - if m: - self._rules[r] = m - - def process_response(self, response): - for regex, method in self._rules.items(): - m = regex.search(response.url) - if m: - return method(response) - return [] - -class MySpider(LegSpider): - - legs = [CallbackRules()] - callback_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - def parse_product(self, response): - # parse reponse and populate item - return item -}}} - -=== URL Canonicalizers === - -Another example could be for building URL canonicalizers: - -{{{ -#!python -class CanonializeUrl(LegSpider): - - def process_request(self, request): - curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules) - return request.replace(url=curl) - -class MySpider(LegSpider): - - legs = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - # ... -}}} - -=== Setting item identifier === - -Another example could be for setting a unique identifier to items, based on certain fields: - -{{{ -#!python -class ItemIdSetter(LegSpider): - - def process_item(self, item): - id_field = self.spider.id_field - id_fields_to_hash = self.spider.id_fields_to_hash - item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) - return item - -class MySpider(LegSpider): - - legs = [ItemIdSetter()] - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def process_response(self, item): - # extract item from response - return item -}}} - -=== Combining multiple leg spiders === - -Here's an example that combines functionality from multiple leg spiders: - -{{{ -#!python -class MySpider(LegSpider): - - legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()] - - url_regexes_to_follow = ['/product.php?.*'] - - parse_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def process_product(self, item): - # extract item from response - return item - - def process_category(self, item): - # extract item from response - return item -}}} - - - -== Leg Spiders vs Spider middlewares == - -A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality. - -== When not to use Leg Spiders == - -Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as: - - * Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware. - * Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit. - -== {{{LegSpider}}} proof-of-concept implementation == - -Here's a proof-of-concept implementation of {{{LegSpider}}}: - -{{{ -#!python -from scrapy.http import Request -from scrapy.item import BaseItem -from scrapy.spider import BaseSpider -from scrapy.utils.spider import iterate_spider_output - - -class LegSpider(BaseSpider): - """A spider made of legs""" - - legs = [] - - def __init__(self, *args, **kwargs): - super(LegSpider, self).__init__(*args, **kwargs) - self._legs = [self] + self.legs[:] - for l in self._legs: - l.set_spider(self) - - def parse(self, response): - res = self._process_response(response) - for r in res: - if isinstance(r, BaseItem): - yield self._process_item(r) - else: - yield self._process_request(r) - - def process_response(self, response): - return [] - - def process_request(self, request): - return request - - def process_item(self, item): - return item - - def set_spider(self, spider): - self.spider = spider - - def _process_response(self, response): - res = [] - for l in self._legs: - res.extend(iterate_spider_output(l.process_response(response))) - return res - - def _process_request(self, request): - for l in self._legs: - request = l.process_request(request) - return request - - def _process_item(self, item): - for l in self._legs: - item = l.process_item(item) - return item += SEP-016: Leg Spider = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||16|| +||'''Title:'''||Leg Spider|| +||'''Author:'''||Insophia Team|| +||'''Created:'''||2010-06-03|| +||'''Status'''||Superseded by [wiki:SEP-018]|| + +== Introduction == + +This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders. + +== Rationale == + +The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. + +The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider". + +== {{{LegSpider}}} API == + +A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods: + + * {{{legs}}} + * legs composing this spider + * {{{process_response(response)}}} + * Process a (downloaded) response and return a list of requests and items + * {{{process_request(request)}}} + * Process a request after it has been extracted and before returning it from the spider + * {{{process_item(item)}}} + * Process an item after it has been extracted and before returning it from the spider + * {{{set_spider()}}} + * Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior. + +== How Leg Spiders work == + + 1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output. + 2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself. + +== Leg Spider examples == + +=== Regex (HTML) Link Extractor === + +A typical application of LegSpider's is to build Link Extractors. For example: + +{{{ +#!python +class RegexHtmlLinkExtractor(LegSpider): + + def process_response(self, response): + if isinstance(response, HtmlResponse): + allowed_regexes = self.spider.url_regexes_to_follow + # extract urls to follow using allowed_regexes + return [Request(x) for x in urls_to_follow] + +class MySpider(LegSpider): + + legs = [RegexHtmlLinkExtractor()] + url_regexes_to_follow = ['/product.php?.*'] + + def parse_response(self, response): + # parse response and extract items + return items +}}} + +=== RSS2 link extractor === + +This is a Leg Spider that can be used for following links from RSS2 feeds. + +{{{ +#!python +class Rss2LinkExtractor(LegSpider): + + def process_response(self, response): + if response.headers.get('Content-type') == 'application/rss+xml': + xs = XmlXPathSelector(response) + urls = xs.select("//item/link/text()").extract() + return [Request(x) for x in urls] +}}} + +=== Callback dispatcher based on rules === + +Another example could be to build a callback dispatcher based on rules: + +{{{ +#!python +class CallbackRules(LegSpider): + + def __init__(self, *a, **kw): + super(CallbackRules, self).__init__(*a, **kw) + for regex, method_name in self.spider.callback_rules.items(): + r = re.compile(regex) + m = getattr(self.spider, method_name, None) + if m: + self._rules[r] = m + + def process_response(self, response): + for regex, method in self._rules.items(): + m = regex.search(response.url) + if m: + return method(response) + return [] + +class MySpider(LegSpider): + + legs = [CallbackRules()] + callback_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + def parse_product(self, response): + # parse response and populate item + return item +}}} + +=== URL Canonicalizers === + +Another example could be for building URL canonicalizers: + +{{{ +#!python +class CanonializeUrl(LegSpider): + + def process_request(self, request): + curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules) + return request.replace(url=curl) + +class MySpider(LegSpider): + + legs = [CanonicalizeUrl()] + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + # ... +}}} + +=== Setting item identifier === + +Another example could be for setting a unique identifier to items, based on certain fields: + +{{{ +#!python +class ItemIdSetter(LegSpider): + + def process_item(self, item): + id_field = self.spider.id_field + id_fields_to_hash = self.spider.id_fields_to_hash + item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) + return item + +class MySpider(LegSpider): + + legs = [ItemIdSetter()] + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def process_response(self, item): + # extract item from response + return item +}}} + +=== Combining multiple leg spiders === + +Here's an example that combines functionality from multiple leg spiders: + +{{{ +#!python +class MySpider(LegSpider): + + legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()] + + url_regexes_to_follow = ['/product.php?.*'] + + parse_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def process_product(self, item): + # extract item from response + return item + + def process_category(self, item): + # extract item from response + return item +}}} + + + +== Leg Spiders vs Spider middlewares == + +A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality. + +== When not to use Leg Spiders == + +Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as: + + * Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware. + * Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit. + +== {{{LegSpider}}} proof-of-concept implementation == + +Here's a proof-of-concept implementation of {{{LegSpider}}}: + +{{{ +#!python +from scrapy.http import Request +from scrapy.item import BaseItem +from scrapy.spider import BaseSpider +from scrapy.utils.spider import iterate_spider_output + + +class LegSpider(BaseSpider): + """A spider made of legs""" + + legs = [] + + def __init__(self, *args, **kwargs): + super(LegSpider, self).__init__(*args, **kwargs) + self._legs = [self] + self.legs[:] + for l in self._legs: + l.set_spider(self) + + def parse(self, response): + res = self._process_response(response) + for r in res: + if isinstance(r, BaseItem): + yield self._process_item(r) + else: + yield self._process_request(r) + + def process_response(self, response): + return [] + + def process_request(self, request): + return request + + def process_item(self, item): + return item + + def set_spider(self, spider): + self.spider = spider + + def _process_response(self, response): + res = [] + for l in self._legs: + res.extend(iterate_spider_output(l.process_response(response))) + return res + + def _process_request(self, request): + for l in self._legs: + request = l.process_request(request) + return request + + def _process_item(self, item): + for l in self._legs: + item = l.process_item(item) + return item }}} \ No newline at end of file diff --git a/sep/sep-018.trac b/sep/sep-018.trac index 9c2a0349b..e09356d49 100644 --- a/sep/sep-018.trac +++ b/sep/sep-018.trac @@ -1,551 +1,551 @@ -= SEP-018: Spider middleware v2 = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||18|| -||'''Title:'''||Spider Middleware v2|| -||'''Author:'''||Insophia Team|| -||'''Created:'''||2010-06-20|| -||'''Status'''||Draft (in progress)|| - -This SEP introduces a new architecture for spider middlewares which provides a greater degree of modularity to combine functionality which can be plugged in from different (reusable) middlewares. - -The purpose of !SpiderMiddleware-v2 is to define an architecture that encourages more re-usability for building spiders based on smaller well-tested components. Those components can be global (similar to current spider middlewares) or per-spider that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. - -Unless explicitly stated, in this document "spider middleware" refers to the '''new''' spider middleware v2, not the old one. - -This document is a work in progress, see [#Pendingissues Pending issues] below. - -== New spider middleware API == - -A spider middleware can implement any of the following methods: - - * `process_response(response, request, spider)` - * Process a (downloaded) response - * Receives: The `response` to process, the `request` used to download the response (not necessarily the request sent from the spider), and the `spider` that requested it. - * Returns: A list containing requests and/or items - * `process_error(error, request, spider)`: - * Process a error when trying to download a request, such as DNS errors, timeout errors, etc. - * Receives: The `error` caused, the `request` that caused it (not necessarily the request sent from the spider), and then `spider` that requested it. - * Returns: A list containing request and/or items - * `process_request(request, response, spider)` - * Process a request after it has been extracted from the spider or previous middleware `process_response()` methods. - * Receives: The `request` to process, the `response` where the request was extracted from, and the `spider` that extracted it. - * Note: `response` is `None` for start requests, or requests injected directly (through `manager.scraper.process_request()` without specifying a response (see below) - * Returns: A `Request` object (not necessarily the same received), or `None` in which case the request is dropped. - * `process_item(item, response, spider)` - * Process an item after it has been extracted from the spider or previous middleware `process_response()` methods. - * Receives: The `item` to process, the `response` where the item was extracted from, and the `spider` that extracted it. - * Returns: An `Item` object (not necessarily the same received), or `None` in which case the item is dropped. - * `next_request(spider)` - * Returns a the next request to crawl with this spider. This method is called when the spider is opened, and when it gets idle. - * Receives: The `spider` to return the next request for. - * Returns: A `Request` object. - * `open_spider(spider)` - * This can be used to allocate resources when a spider is opened. - * Receives: The `spider` that has been opened. - * Returns: nothing - * `close_spider(spider)` - * This can be used to free resources when a spider is closed. - * Receives: The `spider` that has been closed. - * Returns: nothing - -== Changes to core API == - -=== Injecting requests to crawl === - -To inject start requests (or new requests without a response) to crawl, you used before: - - * `manager.engine.crawl(request, spider)` - -Now you'll use: - - * `manager.scraper.process_request(request, spider, response=None)` - -Which (unlike the old `engine.crawl` will make the requests pass through the spider middleware `process_request()` method). - -=== Scheduler middleware to be removed === - -We're gonna remove the Scheduler Middleware, and move the duplicates filter to a new spider middleware. - -== Scraper high-level API == - -There is a simpler high-level API - the Scraper API - which is the API used by the engine and other core components. This is also the API implemented by this new middleware, with its own internal architecture and hooks. Here is the Scraper API: - - * `process_response(response, request, spider)` - * returns iterable of items and requests - * `process_error(error, request, spider)` - * returns iterable of items and requests - * `process_request(request, spider, response=None)` - * injects a request to crawl for the given spider - * `process_item(item, spider, response) - * injects a item to process with the item processor (typically the item pipeline) - * `next_request(spider)` - * returns the next request to process for the given spider - * `open_spider(spider)` - * opens a spider - * `close_spider(spider)` - * closes a spider - -== How it works == - -The spider middlewares are defined in certain order with the top-most being the one closer to the engine, and the bottom-most being the one closed to the spider. - -Example: - - * Engine - * Global spider Middleware 3 - * Global spider Middleware 2 - * Global spider Middleware 1 - * Spider-specific middlewares (defined in `Spider.middlewares`) - * Spider-specific middleware 3 - * Spider-specific middleware 2 - * Spider-specific middleware 1 - * Spider - -The data flow with Spider Middleware v2 is as follows: - - 1. When a response arrives from the engine, it it passed through all the spider middlewares (in descending order). The result of each middleware `process_response` is kept and then returned along with the spider callback result - 2. Each item of the aggregated result from previous point is passed through all middlewares (in ascending order) calling the `process_request` or `process_item` method accordingly, and their results are kept for passing to the following middlewares - -One of the spider middlewares (typically - but not necessarily - the last spider middleware closer to the spider, as shown in the example) will be a "spider-specific spider middleware" which would take care of calling the additional spider middlewares defined in the `Spider.middlewares` attribute, hence providing support for per-spider middlewares. If the middleware is well written, it should work both globally and per-spider. - -== Spider-specific middlewares == - -You can define in the spider itself a list of additional middlewares that will be used for this spider, and only this spider. If the middleware is well written, it should work both globally and per spider. - -Here's an example that combines functionality from multiple middlewares into the same spider: - -{{{ -#!python -class MySpider(BaseSpider): - - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] - - allowed_domains = ['example.com', 'sub.example.com'] - - url_regexes_to_follow = ['/product.php?.*'] - - callback_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def parse_product(self, item): - # extract item from response - return item - - def parse_category(self, item): - # extract item from response - return item -}}} - -== The Spider Middleware that implements spider code == - -There's gonna be one middleware that will take care of calling the proper spider methods on each event such as: - - * call `Request.callback` (for 200 responses) or `Request.errback` for non-200 responses and other errors. this behaviour can be changed through the `handle_httpstatus_list` spider attribute. - * if `Request.callback` is not set it will use `Spider.parse` - * if `Request.errback` is not set it will use `Spider.errback` - * call additional spider middlewares defined in the `Spider.middlewares` attribute - * call `Spider.next_request()` and `Spider.start_requests()` on `next_request()` middleware method (this would implicitly support backwards compatibility) - -== Differences with Spider middleware v1 == - - * adds support for per-spider middlewares through the `Spider.middlewares` attribute - * allows processing initial requests (those returned from `Spider.start_requests()`) - -== Use cases and examples == - -This section contains several examples and use cases for Spider Middlewares. Imports are intentionally removed for conciseness and clarity. - -=== Regex (HTML) Link Extractor === - -A typical application of spider middlewares could be to build Link Extractors. For example: - -{{{ -#!python -class RegexHtmlLinkExtractor(object): - - def process_response(self, response, request, spider): - if isinstance(response, HtmlResponse): - allowed_regexes = spider.url_regexes_to_follow - # extract urls to follow using allowed_regexes - return [Request(x) for x in urls_to_follow] - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [RegexHtmlLinkExtractor()] - url_regexes_to_follow = ['/product.php?.*'] - - # parsing callbacks below -}}} - -=== RSS2 link extractor === - -{{{ -#!python -class Rss2LinkExtractor(object): - - def process_response(self, response, request, spider): - if response.headers.get('Content-type') == 'application/rss+xml': - xs = XmlXPathSelector(response) - urls = xs.select("//item/link/text()").extract() - return [Request(x) for x in urls] -}}} - -=== Callback dispatcher based on rules === - -Another example could be to build a callback dispatcher based on rules: - -{{{ -#!python -class CallbackRules(object): - - def __init__(self): - self.rules = {} - dispatcher.connect(signals.spider_opened, self.spider_opened) - dispatcher.connect(signals.spider_closed, self.spider_closed) - - def spider_opened(self, spider): - self.rules[spider] = {} - for regex, method_name in spider.callback_rules.items(): - r = re.compile(regex) - m = getattr(self.spider, method_name, None) - if m: - self.rules[spider][r] = m - - def spider_closed(self, spider): - del self.rules[spider] - - def process_response(self, response, request, spider): - for regex, method in self.rules[spider].items(): - m = regex.search(response.url) - if m: - return method(response) - return [] - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [CallbackRules()] - callback_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - def parse_product(self, response): - # parse reponse and populate item - return item -}}} - -=== URL Canonicalizers === - -Another example could be for building URL canonicalizers: - -{{{ -#!python -class CanonializeUrl(object): - - def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) - return request.replace(url=curl) - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - # ... -}}} - -=== Setting item identifier === - -Another example could be for setting a unique identifier to items, based on certain fields: - -{{{ -#!python -class ItemIdSetter(object): - - def process_item(self, item, response, spider): - id_field = spider.id_field - id_fields_to_hash = spider.id_fields_to_hash - item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) - return item - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [ItemIdSetter()] - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def parse(self, response): - # extract item from response - return item -}}} - -=== robots.txt exclusion === - -A spider middleware to avoid visiting pages forbidden by robots.txt: - -{{{ -#!python -class SpiderInfo(object): - - def __init__(self, useragent): - self.useragent = useragent - self.parsers = {} - self.pending = defaultdict(list) - - -class AllowAllParser(object): - - def can_fetch(useragent, url): - return True - - -class RobotsTxtMiddleware(object): - - REQUEST_PRIORITY = 1000 - - def __init__(self): - self.spiders = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_request(self, request, response, spider): - return self.process_start_request(self, request) - - def process_start_request(self, request, spider): - info = self.spiders[spider] - url = urlparse_cached(request) - netloc = url.netloc - if netloc in info.parsers: - rp = info.parsers[netloc] - if rp.can_fetch(info.useragent, request.url): - res = request - else: - spider.log("Forbidden by robots.txt: %s" % request) - res = None - else: - if netloc in info.pending: - res = None - else: - robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) - meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} - res = Request(robotsurl, callback=self.parse_robots, - meta=meta, priority=self.REQUEST_PRIORITY) - info.pending[netloc].append(request) - return res - - def parse_robots(self, response): - spider = response.request.meta['spider'] - netloc urlparse_cached(response).netloc - info = self.spiders[spider] - if response.status == 200; - rp = robotparser.RobotFileParser(response.url) - rp.parse(response.body.splitlines()) - info.parsers[netloc] = rp - else: - info.parsers[netloc] = AllowAllParser() - return info.pending[netloc] - - def spider_opened(self, spider): - ua = getattr(spider, 'user_agent', None) or settings['USER_AGENT'] - self.spiders[spider] = SpiderInfo(ua) - - def spider_closed(self, spider): - del self.spiders[spider] -}}} - -=== Offsite middleware === - -This is a port of the Offsite middleware to the new spider middleware API: - -{{{ -#!python -class SpiderInfo(object): - - def __init__(self, host_regex): - self.host_regex = host_regex - self.hosts_seen = set() - - -class OffsiteMiddleware(object): - - def __init__(self): - self.spiders = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_request(self, request, response, spider): - return self.process_start_request(self, request) - - def process_start_request(self, request, spider): - if self.should_follow(request, spider): - return request - else: - info = self.spiders[spider] - host = urlparse_cached(x).hostname - if host and host not in info.hosts_seen: - spider.log("Filtered offsite request to %r: %s" % (host, request)) - info.hosts_seen.add(host) - - def should_follow(self, request, spider): - info = self.spiders[spider] - # hostanme can be None for wrong urls (like javascript links) - host = urlparse_cached(request).hostname or '' - return bool(info.regex.search(host)) - - def get_host_regex(self, spider): - """Override this method to implement a different offsite policy""" - domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) - return re.compile(regex) - - def spider_opened(self, spider): - info = SpiderInfo(self.get_host_regex(spider)) - self.spiders[spider] = info - - def spider_closed(self, spider): - del self.spiders[spider] - -}}} - -=== Limit URL length === - -A middleware to filter out requests with long urls: - -{{{ -#!python - -class LimitUrlLength(object): - - def __init__(self): - self.maxlength = settings.getint('URLLENGTH_LIMIT') - - def process_request(self, request, response, spider): - return self.process_start_request(self, request) - - def process_start_request(self, request, spider): - if len(request.url) <= self.maxlength: - return request - spider.log("Ignoring request (url length > %d): %s " % (self.maxlength, request.url)) -}}} - -=== Set Referer === - -A middleware to set the Referer: - -{{{ -#!python -class SetReferer(object): - - def process_request(self, request, response, spider): - request.headers.setdefault('Referer', response.url) - return request -}}} - -=== Set and limit crawling depth === - -A middleware to set (and limit) the request/response depth, taken from the start requests: - -{{{ -#!python -class SetLimitDepth(object): - - def __init__(self, maxdepth=0): - self.maxdepth = maxdepth or settings.getint('DEPTH_LIMIT') - - def process_request(self, request, response, spider): - depth = response.request.meta['depth'] + 1 - request.meta['depth'] = depth - if not self.maxdepth or depth <= self.maxdepth: - return request - spider.log("Ignoring link (depth > %d): %s " % (self.maxdepth, request) - - def process_start_request(self, request, spider): - request.meta['depth'] = 0 - return request -}}} - -=== Filter duplicate requests === - -A middleware to filter out requests already seen: - -{{{ -#!python -class FilterDuplicates(object): - - def __init__(self): - clspath = settings.get('DUPEFILTER_CLASS') - self.dupefilter = load_object(clspath)() - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def enqueue_request(self, spider, request): - seen = self.dupefilter.request_seen(spider, request) - if not seen or request.dont_filter: - return request - - def spider_opened(self, spider): - self.dupefilter.open_spider(spider) - - def spider_closed(self, spider): - self.dupefilter.close_spider(spider) -}}} - -=== Scrape data using Parsley === - -A middleware to Scrape data using Parsley as described in UsingParsley - -{{{ -#!python -from pyparsley import PyParsley - -class ParsleyExtractor(object): - - def __init__(self, parslet_json_code): - parslet = json.loads(parselet_json_code) - class ParsleyItem(Item): - def __init__(self, *a, **kw): - for name in parslet.keys(): - self.fields[name] = Field() - super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') - - def process_response(self, response, request, spider): - return self.item_class(self.parsly.parse(string=response.body)) -}}} - - - -== Pending issues == - -Resolved: - - * how to make `start_requests()` output pass through spider middleware `process_request()`? - * Start requests will be injected through `manager.scraper.process_request()` instead of `manager.engine.crawl()` - * should we support adding additional start requests from a spider middleware? - * Yes - there is a spider middleware method (`start_requests`) for that - * should `process_response()` receive a `request` argument with the `request` that originated it?. `response.request` is the latest request, not the original one (think of redirections), but it does carry the `meta` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be the deserialized request from the persistent scheduler queue. - * No - this would make implementation more complex and we're not sure it's really needed - * how to make sure `Request.errback` is always called if there is a problem with the request?. Do we need to ensure that?. Requests filtered out (by returning `None`) in the `process_request()` method will never be callback-ed or even errback-ed. this could be a problem for spiders that want to be notified if their requests are dropped. should we support this notification somehow or document (the lack of) it properly? - * We won't support notifications of dropped requests, because: 1. it's hard to implement and unreliable, 2. it's against not friendly with request persistence, 3. we can't come up with a good api. - * should we make the list of default spider middlewares empty? (or the "per-spider" spider middleware alone) - * No - there are some useful spider middlewares that it's worth enabling by default like referer, duplicates, robots2 - * should we allow returning deferreds in spider middleware methods? - * Yes - we should build a Deferred with the spider middleware methods as callbacks and that would implicitly support returning Deferreds - * should we support processing responses before they're processed by the spider, because `process_response` runs "in parallel" to the spider callback, and can't stop from running it. - * No - we haven't seen a practical use case for this, so we won't add an additional hook. It should be trivial to add it later, if needed. - * should we make a spider middleware to handle calling the request and spider callback, instead of letting the Scraper component do it? - * Yes - there's gonna a spider middleware for execution spider-specific code such as callbacks and also custom middlewares += SEP-018: Spider middleware v2 = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||18|| +||'''Title:'''||Spider Middleware v2|| +||'''Author:'''||Insophia Team|| +||'''Created:'''||2010-06-20|| +||'''Status'''||Draft (in progress)|| + +This SEP introduces a new architecture for spider middlewares which provides a greater degree of modularity to combine functionality which can be plugged in from different (reusable) middlewares. + +The purpose of !SpiderMiddleware-v2 is to define an architecture that encourages more re-usability for building spiders based on smaller well-tested components. Those components can be global (similar to current spider middlewares) or per-spider that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. + +Unless explicitly stated, in this document "spider middleware" refers to the '''new''' spider middleware v2, not the old one. + +This document is a work in progress, see [#Pendingissues Pending issues] below. + +== New spider middleware API == + +A spider middleware can implement any of the following methods: + + * `process_response(response, request, spider)` + * Process a (downloaded) response + * Receives: The `response` to process, the `request` used to download the response (not necessarily the request sent from the spider), and the `spider` that requested it. + * Returns: A list containing requests and/or items + * `process_error(error, request, spider)`: + * Process a error when trying to download a request, such as DNS errors, timeout errors, etc. + * Receives: The `error` caused, the `request` that caused it (not necessarily the request sent from the spider), and then `spider` that requested it. + * Returns: A list containing request and/or items + * `process_request(request, response, spider)` + * Process a request after it has been extracted from the spider or previous middleware `process_response()` methods. + * Receives: The `request` to process, the `response` where the request was extracted from, and the `spider` that extracted it. + * Note: `response` is `None` for start requests, or requests injected directly (through `manager.scraper.process_request()` without specifying a response (see below) + * Returns: A `Request` object (not necessarily the same received), or `None` in which case the request is dropped. + * `process_item(item, response, spider)` + * Process an item after it has been extracted from the spider or previous middleware `process_response()` methods. + * Receives: The `item` to process, the `response` where the item was extracted from, and the `spider` that extracted it. + * Returns: An `Item` object (not necessarily the same received), or `None` in which case the item is dropped. + * `next_request(spider)` + * Returns a the next request to crawl with this spider. This method is called when the spider is opened, and when it gets idle. + * Receives: The `spider` to return the next request for. + * Returns: A `Request` object. + * `open_spider(spider)` + * This can be used to allocate resources when a spider is opened. + * Receives: The `spider` that has been opened. + * Returns: nothing + * `close_spider(spider)` + * This can be used to free resources when a spider is closed. + * Receives: The `spider` that has been closed. + * Returns: nothing + +== Changes to core API == + +=== Injecting requests to crawl === + +To inject start requests (or new requests without a response) to crawl, you used before: + + * `manager.engine.crawl(request, spider)` + +Now you'll use: + + * `manager.scraper.process_request(request, spider, response=None)` + +Which (unlike the old `engine.crawl` will make the requests pass through the spider middleware `process_request()` method). + +=== Scheduler middleware to be removed === + +We're gonna remove the Scheduler Middleware, and move the duplicates filter to a new spider middleware. + +== Scraper high-level API == + +There is a simpler high-level API - the Scraper API - which is the API used by the engine and other core components. This is also the API implemented by this new middleware, with its own internal architecture and hooks. Here is the Scraper API: + + * `process_response(response, request, spider)` + * returns iterable of items and requests + * `process_error(error, request, spider)` + * returns iterable of items and requests + * `process_request(request, spider, response=None)` + * injects a request to crawl for the given spider + * `process_item(item, spider, response) + * injects a item to process with the item processor (typically the item pipeline) + * `next_request(spider)` + * returns the next request to process for the given spider + * `open_spider(spider)` + * opens a spider + * `close_spider(spider)` + * closes a spider + +== How it works == + +The spider middlewares are defined in certain order with the top-most being the one closer to the engine, and the bottom-most being the one closed to the spider. + +Example: + + * Engine + * Global spider Middleware 3 + * Global spider Middleware 2 + * Global spider Middleware 1 + * Spider-specific middlewares (defined in `Spider.middlewares`) + * Spider-specific middleware 3 + * Spider-specific middleware 2 + * Spider-specific middleware 1 + * Spider + +The data flow with Spider Middleware v2 is as follows: + + 1. When a response arrives from the engine, it it passed through all the spider middlewares (in descending order). The result of each middleware `process_response` is kept and then returned along with the spider callback result + 2. Each item of the aggregated result from previous point is passed through all middlewares (in ascending order) calling the `process_request` or `process_item` method accordingly, and their results are kept for passing to the following middlewares + +One of the spider middlewares (typically - but not necessarily - the last spider middleware closer to the spider, as shown in the example) will be a "spider-specific spider middleware" which would take care of calling the additional spider middlewares defined in the `Spider.middlewares` attribute, hence providing support for per-spider middlewares. If the middleware is well written, it should work both globally and per-spider. + +== Spider-specific middlewares == + +You can define in the spider itself a list of additional middlewares that will be used for this spider, and only this spider. If the middleware is well written, it should work both globally and per spider. + +Here's an example that combines functionality from multiple middlewares into the same spider: + +{{{ +#!python +class MySpider(BaseSpider): + + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] + + allowed_domains = ['example.com', 'sub.example.com'] + + url_regexes_to_follow = ['/product.php?.*'] + + callback_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def parse_product(self, item): + # extract item from response + return item + + def parse_category(self, item): + # extract item from response + return item +}}} + +== The Spider Middleware that implements spider code == + +There's gonna be one middleware that will take care of calling the proper spider methods on each event such as: + + * call `Request.callback` (for 200 responses) or `Request.errback` for non-200 responses and other errors. this behaviour can be changed through the `handle_httpstatus_list` spider attribute. + * if `Request.callback` is not set it will use `Spider.parse` + * if `Request.errback` is not set it will use `Spider.errback` + * call additional spider middlewares defined in the `Spider.middlewares` attribute + * call `Spider.next_request()` and `Spider.start_requests()` on `next_request()` middleware method (this would implicitly support backwards compatibility) + +== Differences with Spider middleware v1 == + + * adds support for per-spider middlewares through the `Spider.middlewares` attribute + * allows processing initial requests (those returned from `Spider.start_requests()`) + +== Use cases and examples == + +This section contains several examples and use cases for Spider Middlewares. Imports are intentionally removed for conciseness and clarity. + +=== Regex (HTML) Link Extractor === + +A typical application of spider middlewares could be to build Link Extractors. For example: + +{{{ +#!python +class RegexHtmlLinkExtractor(object): + + def process_response(self, response, request, spider): + if isinstance(response, HtmlResponse): + allowed_regexes = spider.url_regexes_to_follow + # extract urls to follow using allowed_regexes + return [Request(x) for x in urls_to_follow] + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [RegexHtmlLinkExtractor()] + url_regexes_to_follow = ['/product.php?.*'] + + # parsing callbacks below +}}} + +=== RSS2 link extractor === + +{{{ +#!python +class Rss2LinkExtractor(object): + + def process_response(self, response, request, spider): + if response.headers.get('Content-type') == 'application/rss+xml': + xs = XmlXPathSelector(response) + urls = xs.select("//item/link/text()").extract() + return [Request(x) for x in urls] +}}} + +=== Callback dispatcher based on rules === + +Another example could be to build a callback dispatcher based on rules: + +{{{ +#!python +class CallbackRules(object): + + def __init__(self): + self.rules = {} + dispatcher.connect(signals.spider_opened, self.spider_opened) + dispatcher.connect(signals.spider_closed, self.spider_closed) + + def spider_opened(self, spider): + self.rules[spider] = {} + for regex, method_name in spider.callback_rules.items(): + r = re.compile(regex) + m = getattr(self.spider, method_name, None) + if m: + self.rules[spider][r] = m + + def spider_closed(self, spider): + del self.rules[spider] + + def process_response(self, response, request, spider): + for regex, method in self.rules[spider].items(): + m = regex.search(response.url) + if m: + return method(response) + return [] + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [CallbackRules()] + callback_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + def parse_product(self, response): + # parse response and populate item + return item +}}} + +=== URL Canonicalizers === + +Another example could be for building URL canonicalizers: + +{{{ +#!python +class CanonializeUrl(object): + + def process_request(self, request, response, spider): + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) + return request.replace(url=curl) + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [CanonicalizeUrl()] + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + # ... +}}} + +=== Setting item identifier === + +Another example could be for setting a unique identifier to items, based on certain fields: + +{{{ +#!python +class ItemIdSetter(object): + + def process_item(self, item, response, spider): + id_field = spider.id_field + id_fields_to_hash = spider.id_fields_to_hash + item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) + return item + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [ItemIdSetter()] + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def parse(self, response): + # extract item from response + return item +}}} + +=== robots.txt exclusion === + +A spider middleware to avoid visiting pages forbidden by robots.txt: + +{{{ +#!python +class SpiderInfo(object): + + def __init__(self, useragent): + self.useragent = useragent + self.parsers = {} + self.pending = defaultdict(list) + + +class AllowAllParser(object): + + def can_fetch(useragent, url): + return True + + +class RobotsTxtMiddleware(object): + + REQUEST_PRIORITY = 1000 + + def __init__(self): + self.spiders = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + def process_request(self, request, response, spider): + return self.process_start_request(self, request) + + def process_start_request(self, request, spider): + info = self.spiders[spider] + url = urlparse_cached(request) + netloc = url.netloc + if netloc in info.parsers: + rp = info.parsers[netloc] + if rp.can_fetch(info.useragent, request.url): + res = request + else: + spider.log("Forbidden by robots.txt: %s" % request) + res = None + else: + if netloc in info.pending: + res = None + else: + robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) + meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} + res = Request(robotsurl, callback=self.parse_robots, + meta=meta, priority=self.REQUEST_PRIORITY) + info.pending[netloc].append(request) + return res + + def parse_robots(self, response): + spider = response.request.meta['spider'] + netloc urlparse_cached(response).netloc + info = self.spiders[spider] + if response.status == 200; + rp = robotparser.RobotFileParser(response.url) + rp.parse(response.body.splitlines()) + info.parsers[netloc] = rp + else: + info.parsers[netloc] = AllowAllParser() + return info.pending[netloc] + + def spider_opened(self, spider): + ua = getattr(spider, 'user_agent', None) or settings['USER_AGENT'] + self.spiders[spider] = SpiderInfo(ua) + + def spider_closed(self, spider): + del self.spiders[spider] +}}} + +=== Offsite middleware === + +This is a port of the Offsite middleware to the new spider middleware API: + +{{{ +#!python +class SpiderInfo(object): + + def __init__(self, host_regex): + self.host_regex = host_regex + self.hosts_seen = set() + + +class OffsiteMiddleware(object): + + def __init__(self): + self.spiders = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + def process_request(self, request, response, spider): + return self.process_start_request(self, request) + + def process_start_request(self, request, spider): + if self.should_follow(request, spider): + return request + else: + info = self.spiders[spider] + host = urlparse_cached(x).hostname + if host and host not in info.hosts_seen: + spider.log("Filtered offsite request to %r: %s" % (host, request)) + info.hosts_seen.add(host) + + def should_follow(self, request, spider): + info = self.spiders[spider] + # hostanme can be None for wrong urls (like javascript links) + host = urlparse_cached(request).hostname or '' + return bool(info.regex.search(host)) + + def get_host_regex(self, spider): + """Override this method to implement a different offsite policy""" + domains = [d.replace('.', r'\.') for d in spider.allowed_domains] + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + return re.compile(regex) + + def spider_opened(self, spider): + info = SpiderInfo(self.get_host_regex(spider)) + self.spiders[spider] = info + + def spider_closed(self, spider): + del self.spiders[spider] + +}}} + +=== Limit URL length === + +A middleware to filter out requests with long urls: + +{{{ +#!python + +class LimitUrlLength(object): + + def __init__(self): + self.maxlength = settings.getint('URLLENGTH_LIMIT') + + def process_request(self, request, response, spider): + return self.process_start_request(self, request) + + def process_start_request(self, request, spider): + if len(request.url) <= self.maxlength: + return request + spider.log("Ignoring request (url length > %d): %s " % (self.maxlength, request.url)) +}}} + +=== Set Referer === + +A middleware to set the Referer: + +{{{ +#!python +class SetReferer(object): + + def process_request(self, request, response, spider): + request.headers.setdefault('Referer', response.url) + return request +}}} + +=== Set and limit crawling depth === + +A middleware to set (and limit) the request/response depth, taken from the start requests: + +{{{ +#!python +class SetLimitDepth(object): + + def __init__(self, maxdepth=0): + self.maxdepth = maxdepth or settings.getint('DEPTH_LIMIT') + + def process_request(self, request, response, spider): + depth = response.request.meta['depth'] + 1 + request.meta['depth'] = depth + if not self.maxdepth or depth <= self.maxdepth: + return request + spider.log("Ignoring link (depth > %d): %s " % (self.maxdepth, request) + + def process_start_request(self, request, spider): + request.meta['depth'] = 0 + return request +}}} + +=== Filter duplicate requests === + +A middleware to filter out requests already seen: + +{{{ +#!python +class FilterDuplicates(object): + + def __init__(self): + clspath = settings.get('DUPEFILTER_CLASS') + self.dupefilter = load_object(clspath)() + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + def enqueue_request(self, spider, request): + seen = self.dupefilter.request_seen(spider, request) + if not seen or request.dont_filter: + return request + + def spider_opened(self, spider): + self.dupefilter.open_spider(spider) + + def spider_closed(self, spider): + self.dupefilter.close_spider(spider) +}}} + +=== Scrape data using Parsley === + +A middleware to Scrape data using Parsley as described in UsingParsley + +{{{ +#!python +from pyparsley import PyParsley + +class ParsleyExtractor(object): + + def __init__(self, parslet_json_code): + parslet = json.loads(parselet_json_code) + class ParsleyItem(Item): + def __init__(self, *a, **kw): + for name in parslet.keys(): + self.fields[name] = Field() + super(ParsleyItem, self).__init__(*a, **kw) + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') + + def process_response(self, response, request, spider): + return self.item_class(self.parsly.parse(string=response.body)) +}}} + + + +== Pending issues == + +Resolved: + + * how to make `start_requests()` output pass through spider middleware `process_request()`? + * Start requests will be injected through `manager.scraper.process_request()` instead of `manager.engine.crawl()` + * should we support adding additional start requests from a spider middleware? + * Yes - there is a spider middleware method (`start_requests`) for that + * should `process_response()` receive a `request` argument with the `request` that originated it?. `response.request` is the latest request, not the original one (think of redirections), but it does carry the `meta` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be the deserialized request from the persistent scheduler queue. + * No - this would make implementation more complex and we're not sure it's really needed + * how to make sure `Request.errback` is always called if there is a problem with the request?. Do we need to ensure that?. Requests filtered out (by returning `None`) in the `process_request()` method will never be callback-ed or even errback-ed. this could be a problem for spiders that want to be notified if their requests are dropped. should we support this notification somehow or document (the lack of) it properly? + * We won't support notifications of dropped requests, because: 1. it's hard to implement and unreliable, 2. it's against not friendly with request persistence, 3. we can't come up with a good api. + * should we make the list of default spider middlewares empty? (or the "per-spider" spider middleware alone) + * No - there are some useful spider middlewares that it's worth enabling by default like referer, duplicates, robots2 + * should we allow returning deferreds in spider middleware methods? + * Yes - we should build a Deferred with the spider middleware methods as callbacks and that would implicitly support returning Deferreds + * should we support processing responses before they're processed by the spider, because `process_response` runs "in parallel" to the spider callback, and can't stop from running it. + * No - we haven't seen a practical use case for this, so we won't add an additional hook. It should be trivial to add it later, if needed. + * should we make a spider middleware to handle calling the request and spider callback, instead of letting the Scraper component do it? + * Yes - there's gonna a spider middleware for execution spider-specific code such as callbacks and also custom middlewares