From 72b6c96d9a30f7f93db66b33f9a9299f72dc9928 Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Thu, 6 Mar 2014 12:15:42 -0300 Subject: [PATCH 1/3] Running lucasdemarchi/codespell to fix typos in docs --- docs/faq.rst | 2 +- docs/news.rst | 8 ++++---- docs/topics/request-response.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9cab09574..5728f2b15 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -172,7 +172,7 @@ the :ref:`topics-signals-ref` to know which ones. What does the response status code 999 means? --------------------------------------------- -999 is a custom reponse status code used by Yahoo sites to throttle requests. +999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or higher) in your spider:: diff --git a/docs/news.rst b/docs/news.rst index fe6339857..15b1840dc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -63,7 +63,7 @@ Enhancements - Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`) - Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`) - Support defining http error handling using settings (:issue:`466`) -- Use moderm python idioms wherever you find legacies (:issue:`497`) +- Use modern python idioms wherever you find legacies (:issue:`497`) - Improve and correct documentation (:issue:`527`, :issue:`524`, :issue:`521`, :issue:`517`, :issue:`512`, :issue:`505`, :issue:`502`, :issue:`489`, :issue:`465`, :issue:`460`, :issue:`425`, :issue:`536`) @@ -118,7 +118,7 @@ Enhancements - Remove multi spider support from multiple core components (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`) - Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages. -- Add pypy 2.1 to continous integration tests (:commit:`ecfa7431`) +- Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`) - Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`) - Use importlib for parametric imports (:issue:`445`) - Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`) @@ -188,7 +188,7 @@ List of contributors sorted by number of commits:: - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests lazyness and early hangs (:commit:`89faf52`) +- Fix start_requests laziness and early hangs (:commit:`89faf52`) 0.18.3 (released 2013-10-03) ---------------------------- @@ -330,7 +330,7 @@ contributors sorted by number of commits:: - fixes spelling errors in documentation (:commit:`6d2b3aa`) - add doc about disabling an extension. refs #132 (:commit:`c90de33`) -- Fixed error message formatting. log.err() doesn't support cool formatting and when error occured, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`) +- Fixed error message formatting. log.err() doesn't support cool formatting and when error occurred, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`) - lint and improve images pipeline error logging (:commit:`56b45fc`) - fixed doc typos (:commit:`243be84`) - add documentation topics: Broad Crawls & Common Practies (:commit:`1fbb715`) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 280e9fd60..a142f6690 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -411,7 +411,7 @@ Response objects .. attribute:: Response.body - A str containing the body of this Response. Keep in mind that Reponse.body + A str containing the body of this Response. Keep in mind that Response.body is always a str. If you want the unicode version use :meth:`TextResponse.body_as_unicode` (only available in :class:`TextResponse` and subclasses). From 20a8237910dbdde78fcda2e9101e175432b41b44 Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Thu, 6 Mar 2014 12:41:21 -0300 Subject: [PATCH 2/3] Running lucasdemarchi/codespell to fix typos in SEPs --- sep/sep-003.trac | 306 ++++++------ sep/sep-014.trac | 1224 +++++++++++++++++++++++----------------------- sep/sep-016.trac | 528 ++++++++++---------- sep/sep-018.trac | 1102 ++++++++++++++++++++--------------------- 4 files changed, 1580 insertions(+), 1580 deletions(-) diff --git a/sep/sep-003.trac b/sep/sep-003.trac index f0b910f61..dd40d9510 100644 --- a/sep/sep-003.trac +++ b/sep/sep-003.trac @@ -1,153 +1,153 @@ -= SEP-003 - Nested items API (!ItemField) = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||3|| -||'''Title:'''||Nested items API (!ItemField) || -||'''Author:'''||Pablo Hoffman|| -||'''Created:'''||2009-07-19|| -||'''Status'''||Obsoleted by [wiki:SEP-008]|| - -== Introduction == - -This page presents different usage scenarios for the new nested items field API called !ItemField. - -== Prerequisites == - -This API proposal relies on the following API: - - 1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance) - 2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}} - -== Proposed Implementation of !ItemField == - -{{{ -#!python -from scrapy.item.fields import BaseField - -class ItemField(BaseField): - def __init__(self, item_type, default=None): - self._item_type = item_type - super(ItemField, self).__init__(default) - - def to_python(self, value): - return self._item_type(value) if not isinstance(value, self._item_type) else value - - def get_default(self): - # WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors - # always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour - return self._default -}}} - -== Usage Scenarios == - -=== Defining an item containing !ItemField's === - -{{{ -#!python -from scrapy.item.models import Item -from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField - -class Supplier(Item): - name = TextField(default="anonymous supplier") - url = UrlField() - -class Variant(Item): - name = TextField(required=True) - url = UrlField() - price = DecimalField() - -class Product(Variant): - supplier = ItemField(Supplier, default=Supplier(name="default supplier") - variants = ListField(ItemField(Variant)) - - # these ones are used for documenting default value examples - supplier2 = ItemField(Supplier) - variants2 = ListField(ItemField(Variant), default=[]) -}}} - -It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant -relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile: - -{{{ -#!python -class Product(Item): - variants = ItemField(Product) # Fails to compile -}}} - -=== Assigning an item field === - -{{{ -#!python - -supplier = Supplier(name="Supplier 1", url="http://example.com") - -p = Product() - -# standard assignment -p['supplier'] = supplier -# this also works as it tries to instantiate a Supplier with the given dict -p['supplier'] = {'name': 'Supplier 1' url='http://example.com'} -# this fails because it can't instantiate a Supplier -p['supplier'] = 'Supplier 1' -# this fails because url doesn't have the valid type -p['supplier'] = {'name': 'Supplier 1' url=123} - -v1 = Variant() -v1['name'] = "lala" -v1['price'] = Decimal("100") - -v2 = Variant() -v2['name'] = "lolo" -v2['price'] = Decimal("150") - -# standard assignment -p['variants'] = [v1, v2] # OK -# can also instantiate at assignment time -p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")] -# this also works as it tries to instantiate a Variant with the given dict -p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")] -# this fails because it can't instantiate a Variant -p['variants'] = [v1, 'test'] -# this fails beacuse 'coco' is not a valid value for price -p['variants'] = [v1, {'name': 'lolo', 'price': 'coco'] -}}} - -=== Default values === - -{{{ -#!python -p = Product() - -p['supplier'] # returns: Supplier(name='default supplier') -p['supplier2'] # raises KeyError -p['supplier2'] = Supplier() -p['supplier2'] # returns: Supplier(name='anonymous supplier') - -p['variants'] # raises KeyError -p['variants2'] # returns [] - -p['categories'] # raises KeyError -p.get('categories') # returns None - -p['numbers'] # returns [] -}}} - -=== Accesing and changing nested item values === - -{{{ -#!python - -p = Product(supplier=Supplier(name="some name", url="http://example.com")) -p['supplier']['url'] # returns 'http://example.com' -p['supplier']['url'] = "http://www.other.com" # works as expected -p['supplier']['url'] = 123 # fails: wrong type for supplier url - -p['variants'] = [v1, v2] -p['variants'][0]['name'] # returns v1 name -p['variants'][1]['name'] # returns v2 name - -# XXX: decide what to do about these cases: -p['variants'].append(v3) # works but doesn't check type of v3 -p['variants'].append(1) # works but shouldn't? -}}} += SEP-003 - Nested items API (!ItemField) = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||3|| +||'''Title:'''||Nested items API (!ItemField) || +||'''Author:'''||Pablo Hoffman|| +||'''Created:'''||2009-07-19|| +||'''Status'''||Obsoleted by [wiki:SEP-008]|| + +== Introduction == + +This page presents different usage scenarios for the new nested items field API called !ItemField. + +== Prerequisites == + +This API proposal relies on the following API: + + 1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance) + 2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}} + +== Proposed Implementation of !ItemField == + +{{{ +#!python +from scrapy.item.fields import BaseField + +class ItemField(BaseField): + def __init__(self, item_type, default=None): + self._item_type = item_type + super(ItemField, self).__init__(default) + + def to_python(self, value): + return self._item_type(value) if not isinstance(value, self._item_type) else value + + def get_default(self): + # WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors + # always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour + return self._default +}}} + +== Usage Scenarios == + +=== Defining an item containing !ItemField's === + +{{{ +#!python +from scrapy.item.models import Item +from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField + +class Supplier(Item): + name = TextField(default="anonymous supplier") + url = UrlField() + +class Variant(Item): + name = TextField(required=True) + url = UrlField() + price = DecimalField() + +class Product(Variant): + supplier = ItemField(Supplier, default=Supplier(name="default supplier") + variants = ListField(ItemField(Variant)) + + # these ones are used for documenting default value examples + supplier2 = ItemField(Supplier) + variants2 = ListField(ItemField(Variant), default=[]) +}}} + +It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant +relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile: + +{{{ +#!python +class Product(Item): + variants = ItemField(Product) # Fails to compile +}}} + +=== Assigning an item field === + +{{{ +#!python + +supplier = Supplier(name="Supplier 1", url="http://example.com") + +p = Product() + +# standard assignment +p['supplier'] = supplier +# this also works as it tries to instantiate a Supplier with the given dict +p['supplier'] = {'name': 'Supplier 1' url='http://example.com'} +# this fails because it can't instantiate a Supplier +p['supplier'] = 'Supplier 1' +# this fails because url doesn't have the valid type +p['supplier'] = {'name': 'Supplier 1' url=123} + +v1 = Variant() +v1['name'] = "lala" +v1['price'] = Decimal("100") + +v2 = Variant() +v2['name'] = "lolo" +v2['price'] = Decimal("150") + +# standard assignment +p['variants'] = [v1, v2] # OK +# can also instantiate at assignment time +p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")] +# this also works as it tries to instantiate a Variant with the given dict +p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")] +# this fails because it can't instantiate a Variant +p['variants'] = [v1, 'test'] +# this fails because 'coco' is not a valid value for price +p['variants'] = [v1, {'name': 'lolo', 'price': 'coco'] +}}} + +=== Default values === + +{{{ +#!python +p = Product() + +p['supplier'] # returns: Supplier(name='default supplier') +p['supplier2'] # raises KeyError +p['supplier2'] = Supplier() +p['supplier2'] # returns: Supplier(name='anonymous supplier') + +p['variants'] # raises KeyError +p['variants2'] # returns [] + +p['categories'] # raises KeyError +p.get('categories') # returns None + +p['numbers'] # returns [] +}}} + +=== Accesing and changing nested item values === + +{{{ +#!python + +p = Product(supplier=Supplier(name="some name", url="http://example.com")) +p['supplier']['url'] # returns 'http://example.com' +p['supplier']['url'] = "http://www.other.com" # works as expected +p['supplier']['url'] = 123 # fails: wrong type for supplier url + +p['variants'] = [v1, v2] +p['variants'][0]['name'] # returns v1 name +p['variants'][1]['name'] # returns v2 name + +# XXX: decide what to do about these cases: +p['variants'].append(v3) # works but doesn't check type of v3 +p['variants'].append(1) # works but shouldn't? +}}} diff --git a/sep/sep-014.trac b/sep/sep-014.trac index 5b19784f0..1df94b3a4 100644 --- a/sep/sep-014.trac +++ b/sep/sep-014.trac @@ -1,612 +1,612 @@ -= SEP-014 - !CrawlSpider v2 = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||14|| -||'''Title:'''||!CrawlSpider v2|| -||'''Author:'''||Insophia Team|| -||'''Created:'''||2010-01-22|| -||'''Updated:'''||2010-02-04|| -||'''Status'''||Final. Partially implemented but discarded because of lack of use in r2632|| - -== Introduction == - -This SEP proposes a rewrite of Scrapy !CrawlSpider and related components - -== Current flaws and inconsistencies == - - 1. Request's callbacks are hard to persist. - 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) - 3. Isn't possible to crawl an url directly from command line because the Spider does not know which callback use. - -These flaws will be corrected by the changes proposed in this SEP. - -== Proposed API Changes == - - * Separate the functionality of Rule-!LinkExtractor-Callback - * Separate the functionality of !LinkExtractor to Request Extractor and Request Processor - * Separate the process of determining response callback and the extraction of new requests (link extractors) - * The callback will be determine by Matcher Objects on request/response objects - -=== Matcher Objects === - -Matcher Objects (aka Matcher) are responsible for determining if given request or response matches an arbitrary criteria. -The Matcher receives as argument the request or the response, giving a powerful access to all request/response attributes. - -In the current !CrawlSpider, the Rule Object has the responsability to determine the callback of given extractor, -and the link extractor contains the url pattern (aka regex). Now the Matcher will contain only the pattern or criteria -to determine which request/response will execute any action. See below Spider Rules. - -=== Request Extractors === - -Request Extractors takes response object and determines which requests follow. - -This is an enhancemente to !LinkExtractors which returns urls (links), Request Extractors -return Request objects. - -=== Request Processors === - -Request Processors takes requests objects and can perform any action to them, like filtering -or modifying on the fly. - -The current !LinkExtractor had integrated link processing, like canonicalize. Request Processors -can be reutilized and applied in serie. - -=== Request Generator === - -Request Generator is the decoupling of the !CrawlSpider's method _request_to_follow(). -Request Generator takes the response object and applies the Request Extractors and Request Processors. - -=== Rules Manager === - -The Rules are a definition of Rule objects containing Matcher Objects and callback. - -The Legacy Rules were used to perform the link extraction and attach the callback to the generated Request object. -The proposed new Rules will be used to determine the callback for given response. This opens a whole of opportunities, -like determine the callback for given url, and persist the queue of Request objects because the callback is determined -the matching the Response object against the Rules. - -== Usage Examples == - -=== Basic Crawling === - -{{{ -#!python -# -# Basic Crawling -# -class SampleSpider(CrawlSpider): - rules = [ - # The dispatcher uses first-match policy - Rule(UrlRegexMatch(r'product\.html\?id=\d+'), 'parse_item', follow=False), - # by default, if the first param is string is wrapped into UrlRegexMatch - Rule(r'.+', 'parse_page'), - ] - - request_extractors = [ - # crawl all links looking for products and images - SgmlRequestExtractor(), - ] - - request_processors = [ - # canonicalize all requests' urls - Canonicalize(), - ] - - def parse_item(self, response): - # parse and extract items from response - pass - - def parse_page(self, response): - # extract images on all pages - pass -}}} - -=== Custom Processor and External Callback === - -{{{ -#!python -# -# Using external callbacks -# - -# Custom Processor -def filter_today_links(requests): - # only crawl today links - today = datetime.datetime.today().strftime('%Y-%m-%d') - return [r for r in requests if today in r.url] - -# Callback defined out of spider -def my_external_callback(response): - # process item - pass - -class SampleSpider(CrawlSpider): - rules = [ - # The dispatcher uses first-match policy - Rule(UrlRegexMatch(r'/news/(.+)/'), my_external_callback), - ] - - request_extractors = [ - RegexRequestExtractor(r'/sections/.+'), - RegexRequestExtractor(r'/news/.+'), - ] - - request_processors = [ - # canonicalize all requests' urls - Canonicalize(), - filter_today_links, - ] - -}}} - -== Implementation == - -*Work-in-progress* - -=== Package Structure === -{{{ -contrib_exp - |- crawlspider/ - |- spider.py - |- CrawlSpider - |- rules.py - |- Rule - |- CompiledRule - |- RulesManager - |- reqgen.py - |- RequestGenerator - |- reqproc.py - |- Canonicalize - |- Unique - |- ... - |- reqext.py - |- SgmlRequestExtractor - |- RegexRequestExtractor - |- ... - |- matchers.py - |- BaseMatcher - |- UrlMatcher - |- UrlRegexMatcher - |- ... -}}} - -=== Request/Response Matchers === -{{{ -#!python -""" -Request/Response Matchers - -Perform evaluation to Request or Response attributes -""" - -class BaseMatcher(object): - """Base matcher. Returns True by default.""" - - def matches_request(self, request): - """Performs Request Matching""" - return True - - def matches_response(self, response): - """Performs Response Matching""" - return True - - -class UrlMatcher(BaseMatcher): - """Matches URL attribute""" - - def __init__(self, url): - """Initialize url attribute""" - self._url = url - - def matches_url(self, url): - """Returns True if given url is equal to matcher's url""" - return self._url == url - - def matches_request(self, request): - """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) - - def matches_response(self, response): - """REturns True if Response's url matches initial url""" - return self.matches_url(response.url) - - -class UrlRegexMatcher(UrlMatcher): - """Matches URL using regular expression""" - - def __init__(self, regex, flags=0): - """Initialize regular expression""" - self._regex = re.compile(regex, flags) - - def matches_url(self, url): - """Returns True if url matches regular expression""" - return self._regex.search(url) is not None -}}} - -=== Request Extractor === -{{{ -#!python -# -# Requests Extractor -# Extractors receive response and return list of Requests -# - -class BaseSgmlRequestExtractor(FixedSGMLParser): - """Base SGML Request Extractor""" - - def __init__(self, tag='a', attr='href'): - """Initialize attributes""" - FixedSGMLParser.__init__(self) - - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.current_request = None - - def extract_requests(self, response): - """Returns list of requests extracted from response""" - return self._extract_requests(response.body, response.url, - response.encoding) - - def _extract_requests(self, response_text, response_url, response_encoding): - """Extract requests with absolute urls""" - self.reset() - self.feed(response_text) - self.close() - - base_url = self.base_url if self.base_url else response_url - self._make_absolute_urls(base_url, response_encoding) - self._fix_link_text_encoding(response_encoding) - - return self.requests - - def _make_absolute_urls(self, base_url, encoding): - """Makes all request's urls absolute""" - for req in self.requests: - url = req.url - # make absolute url - url = urljoin_rfc(base_url, url, encoding) - url = safe_url_string(url, encoding) - # replace in-place request's url - req.url = url - - def _fix_link_text_encoding(self, encoding): - """Convert link_text to unicode for each request""" - for req in self.requests: - req.meta.setdefault('link_text', '') - req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) - - def reset(self): - """Reset state""" - FixedSGMLParser.reset(self) - self.requests = [] - self.base_url = None - - def unknown_starttag(self, tag, attrs): - """Process unknown start tag""" - if 'base' == tag: - self.base_url = dict(attrs).get('href') - - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - if value is not None: - req = Request(url=value) - self.requests.append(req) - self.current_request = req - - def unknown_endtag(self, tag): - """Process unknown end tag""" - self.current_request = None - - def handle_data(self, data): - """Process data""" - current = self.current_request - if current and not 'link_text' in current.meta: - current.meta['link_text'] = data.strip() - - -class SgmlRequestExtractor(BaseSgmlRequestExtractor): - """SGML Request Extractor""" - - def __init__(self, tags=None, attrs=None): - """Initialize with custom tag & attribute function checkers""" - # defaults - tags = tuple(tags) if tags else ('a', 'area') - attrs = tuple(attrs) if attrs else ('href', ) - - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs - BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) - - -class XPathRequestExtractor(SgmlRequestExtractor): - """SGML Request Extractor with XPath restriction""" - - def __init__(self, restrict_xpaths, tags=None, attrs=None): - """Initialize XPath restrictions""" - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - SgmlRequestExtractor.__init__(self, tags, attrs) - - def extract_requests(self, response): - """Restrict to XPath regions""" - hxs = HtmlXPathSelector(response) - fragments = (''.join( - html_frag for html_frag in hxs.select(xpath).extract() - ) for xpath in self.restrict_xpaths) - html_slice = ''.join(html_frag for html_frag in fragments) - return self._extract_requests(html_slice, response.url, - response.encoding) - -}}} - -=== Request Processor === -{{{ -#!python -# -# Request Processors -# Processors receive list of requests and return list of requests -# -"""Request Processors""" - -class Canonicalize(object): - """Canonicalize Request Processor""" - - def __call__(self, requests): - """Canonicalize all requests' urls""" - for req in requests: - # replace in-place - req.url = canonicalize_url(req.url) - yield req - - -class Unique(object): - """Filter duplicate Requests""" - - def __init__(self, *attributes): - """Initialize comparison attributes""" - self._attributes = attributes or ['url'] - - def _requests_equal(self, req1, req2): - """Attribute comparison helper""" - for attr in self._attributes: - if getattr(req1, attr) != getattr(req2, attr): - return False - # all attributes equal - return True - - def _request_in(self, request, requests_seen): - """Check if request is in given requests seen list""" - for seen in requests_seen: - if self._requests_equal(request, seen): - return True - # request not seen - return False - - def __call__(self, requests): - """Filter seen requests""" - # per-call duplicates filter - requests_seen = set() - for req in requests: - if not self._request_in(req, requests_seen): - yield req - # registry seen request - requests_seen.add(req) - - -class FilterDomain(object): - """Filter request's domain""" - - def __init__(self, allow=(), deny=()): - """Initialize allow/deny attributes""" - self.allow = tuple(arg_to_iter(allow)) - self.deny = tuple(arg_to_iter(deny)) - - def __call__(self, requests): - """Filter domains""" - processed = (req for req in requests) - - if self.allow: - processed = (req for req in requests - if url_is_from_any_domain(req.url, self.allow)) - if self.deny: - processed = (req for req in requests - if not url_is_from_any_domain(req.url, self.deny)) - - return processed - - -class FilterUrl(object): - """Filter request's url""" - - def __init__(self, allow=(), deny=()): - """Initialize allow/deny attributes""" - _re_type = type(re.compile('', 0)) - - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] - - def __call__(self, requests): - """Filter request's url based on allow/deny rules""" - #TODO: filter valid urls here? - processed = (req for req in requests) - - if self.allow_res: - processed = (req for req in requests - if self._matches(req.url, self.allow_res)) - if self.deny_res: - processed = (req for req in requests - if not self._matches(req.url, self.deny_res)) - - return processed - - def _matches(self, url, regexs): - """Returns True if url matches any regex in given list""" - return any(r.search(url) for r in regexs) - - -}}} - -=== Rule Object === -{{{ -#!python -# -# Dispatch Rules classes -# Manage Rules (Matchers + Callbacks) -# -class Rule(object): - """Crawler Rule""" - def __init__(self, matcher, callback=None, cb_args=None, - cb_kwargs=None, follow=True): - """Store attributes""" - self.matcher = matcher - self.callback = callback - self.cb_args = cb_args if cb_args else () - self.cb_kwargs = cb_kwargs if cb_kwargs else {} - self.follow = follow - -# -# Rules Manager takes list of Rule objects and normalize matcher and callback -# into CompiledRule -# -class CompiledRule(object): - """Compiled version of Rule""" - def __init__(self, matcher, callback=None, follow=False): - """Initialize attributes checking type""" - assert isinstance(matcher, BaseMatcher) - assert callback is None or callable(callback) - assert isinstance(follow, bool) - - self.matcher = matcher - self.callback = callback - self.follow = follow -}}} - -=== Rules Manager === -{{{ -#!python -# -# Handles rules matcher/callbacks -# Resolve rule for given response -# -class RulesManager(object): - """Rules Manager""" - def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): - """Initialize rules using spider and default matcher""" - self._rules = tuple() - - # compile absolute/relative-to-spider callbacks""" - for rule in rules: - # prepare matcher - if isinstance(rule.matcher, BaseMatcher): - matcher = rule.matcher - else: - # matcher not BaseMatcher, check for string - if isinstance(rule.matcher, basestring): - # instance default matcher - matcher = default_matcher(rule.matcher) - else: - raise ValueError('Not valid matcher given %r in %r' \ - % (rule.matcher, rule)) - - # prepare callback - if callable(rule.callback): - callback = rule.callback - elif not rule.callback is None: - # callback from spider - callback = getattr(spider, rule.callback) - - if not callable(callback): - raise AttributeError('Invalid callback %r can not be resolved' \ - % callback) - else: - callback = None - - if rule.cb_args or rule.cb_kwargs: - # build partial callback - callback = partial(callback, *rule.cb_args, **rule.cb_kwargs) - - # append compiled rule to rules list - crule = CompiledRule(matcher, callback, follow=rule.follow) - self._rules += (crule, ) - - def get_rule(self, response): - """Returns first rule that matches response""" - for rule in self._rules: - if rule.matcher.matches_response(response): - return rule - -}}} - -=== Request Generator === -{{{ -#!python -# -# Request Generator -# Takes response and generate requests using extractors and processors -# -class RequestGenerator(object): - def __init__(self, req_extractors, req_processors, callback): - self._request_extractors = req_extractors - self._request_processors = req_processors - self.callback = callback - - def generate_requests(self, response): - """ - Extract and process new requets from response - """ - requests = [] - for ext in self._request_extractors: - requets.extend(ext.extract_requests(response)) - - for proc in self._request_processors: - requests = proc(requests) - - for request in requests: - yield request.replace(callback=self.callback) -}}} - -=== !CrawlSpider === -{{{ -#!python -# -# Spider -# -class CrawlSpider(InitSpider): - """CrawlSpider v2""" - - request_extractors = [] - request_processors = [] - rules = [] - - def __init__(self): - """Initialize dispatcher""" - super(CrawlSpider, self).__init__() - - # wrap rules - self._rulesman = RulesManager(self.rules, spider=self) - # generates new requests with given callback - self._reqgen = RequestGenerator(self.request_extractors, - self.request_processors, - self.parse) - - def parse(self, response): - """Dispatch callback and generate requests""" - # get rule for response - rule = self._rulesman.get_rule(response) - if rule: - # dispatch callback if set - if rule.callback: - output = iterate_spider_output(rule.callback(response)) - for req_or_item in output: - yield req_or_item - - if rule.follow: - for req in self._reqgen.generate_requests(response): - yield req - -}}} - += SEP-014 - !CrawlSpider v2 = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||14|| +||'''Title:'''||!CrawlSpider v2|| +||'''Author:'''||Insophia Team|| +||'''Created:'''||2010-01-22|| +||'''Updated:'''||2010-02-04|| +||'''Status'''||Final. Partially implemented but discarded because of lack of use in r2632|| + +== Introduction == + +This SEP proposes a rewrite of Scrapy !CrawlSpider and related components + +== Current flaws and inconsistencies == + + 1. Request's callbacks are hard to persist. + 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) + 3. Isn't possible to crawl an url directly from command line because the Spider does not know which callback use. + +These flaws will be corrected by the changes proposed in this SEP. + +== Proposed API Changes == + + * Separate the functionality of Rule-!LinkExtractor-Callback + * Separate the functionality of !LinkExtractor to Request Extractor and Request Processor + * Separate the process of determining response callback and the extraction of new requests (link extractors) + * The callback will be determine by Matcher Objects on request/response objects + +=== Matcher Objects === + +Matcher Objects (aka Matcher) are responsible for determining if given request or response matches an arbitrary criteria. +The Matcher receives as argument the request or the response, giving a powerful access to all request/response attributes. + +In the current !CrawlSpider, the Rule Object has the responsibility to determine the callback of given extractor, +and the link extractor contains the url pattern (aka regex). Now the Matcher will contain only the pattern or criteria +to determine which request/response will execute any action. See below Spider Rules. + +=== Request Extractors === + +Request Extractors takes response object and determines which requests follow. + +This is an enhancemente to !LinkExtractors which returns urls (links), Request Extractors +return Request objects. + +=== Request Processors === + +Request Processors takes requests objects and can perform any action to them, like filtering +or modifying on the fly. + +The current !LinkExtractor had integrated link processing, like canonicalize. Request Processors +can be reutilized and applied in serie. + +=== Request Generator === + +Request Generator is the decoupling of the !CrawlSpider's method _request_to_follow(). +Request Generator takes the response object and applies the Request Extractors and Request Processors. + +=== Rules Manager === + +The Rules are a definition of Rule objects containing Matcher Objects and callback. + +The Legacy Rules were used to perform the link extraction and attach the callback to the generated Request object. +The proposed new Rules will be used to determine the callback for given response. This opens a whole of opportunities, +like determine the callback for given url, and persist the queue of Request objects because the callback is determined +the matching the Response object against the Rules. + +== Usage Examples == + +=== Basic Crawling === + +{{{ +#!python +# +# Basic Crawling +# +class SampleSpider(CrawlSpider): + rules = [ + # The dispatcher uses first-match policy + Rule(UrlRegexMatch(r'product\.html\?id=\d+'), 'parse_item', follow=False), + # by default, if the first param is string is wrapped into UrlRegexMatch + Rule(r'.+', 'parse_page'), + ] + + request_extractors = [ + # crawl all links looking for products and images + SgmlRequestExtractor(), + ] + + request_processors = [ + # canonicalize all requests' urls + Canonicalize(), + ] + + def parse_item(self, response): + # parse and extract items from response + pass + + def parse_page(self, response): + # extract images on all pages + pass +}}} + +=== Custom Processor and External Callback === + +{{{ +#!python +# +# Using external callbacks +# + +# Custom Processor +def filter_today_links(requests): + # only crawl today links + today = datetime.datetime.today().strftime('%Y-%m-%d') + return [r for r in requests if today in r.url] + +# Callback defined out of spider +def my_external_callback(response): + # process item + pass + +class SampleSpider(CrawlSpider): + rules = [ + # The dispatcher uses first-match policy + Rule(UrlRegexMatch(r'/news/(.+)/'), my_external_callback), + ] + + request_extractors = [ + RegexRequestExtractor(r'/sections/.+'), + RegexRequestExtractor(r'/news/.+'), + ] + + request_processors = [ + # canonicalize all requests' urls + Canonicalize(), + filter_today_links, + ] + +}}} + +== Implementation == + +*Work-in-progress* + +=== Package Structure === +{{{ +contrib_exp + |- crawlspider/ + |- spider.py + |- CrawlSpider + |- rules.py + |- Rule + |- CompiledRule + |- RulesManager + |- reqgen.py + |- RequestGenerator + |- reqproc.py + |- Canonicalize + |- Unique + |- ... + |- reqext.py + |- SgmlRequestExtractor + |- RegexRequestExtractor + |- ... + |- matchers.py + |- BaseMatcher + |- UrlMatcher + |- UrlRegexMatcher + |- ... +}}} + +=== Request/Response Matchers === +{{{ +#!python +""" +Request/Response Matchers + +Perform evaluation to Request or Response attributes +""" + +class BaseMatcher(object): + """Base matcher. Returns True by default.""" + + def matches_request(self, request): + """Performs Request Matching""" + return True + + def matches_response(self, response): + """Performs Response Matching""" + return True + + +class UrlMatcher(BaseMatcher): + """Matches URL attribute""" + + def __init__(self, url): + """Initialize url attribute""" + self._url = url + + def matches_url(self, url): + """Returns True if given url is equal to matcher's url""" + return self._url == url + + def matches_request(self, request): + """Returns True if Request's url matches initial url""" + return self.matches_url(request.url) + + def matches_response(self, response): + """REturns True if Response's url matches initial url""" + return self.matches_url(response.url) + + +class UrlRegexMatcher(UrlMatcher): + """Matches URL using regular expression""" + + def __init__(self, regex, flags=0): + """Initialize regular expression""" + self._regex = re.compile(regex, flags) + + def matches_url(self, url): + """Returns True if url matches regular expression""" + return self._regex.search(url) is not None +}}} + +=== Request Extractor === +{{{ +#!python +# +# Requests Extractor +# Extractors receive response and return list of Requests +# + +class BaseSgmlRequestExtractor(FixedSGMLParser): + """Base SGML Request Extractor""" + + def __init__(self, tag='a', attr='href'): + """Initialize attributes""" + FixedSGMLParser.__init__(self) + + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.current_request = None + + def extract_requests(self, response): + """Returns list of requests extracted from response""" + return self._extract_requests(response.body, response.url, + response.encoding) + + def _extract_requests(self, response_text, response_url, response_encoding): + """Extract requests with absolute urls""" + self.reset() + self.feed(response_text) + self.close() + + base_url = self.base_url if self.base_url else response_url + self._make_absolute_urls(base_url, response_encoding) + self._fix_link_text_encoding(response_encoding) + + return self.requests + + def _make_absolute_urls(self, base_url, encoding): + """Makes all request's urls absolute""" + for req in self.requests: + url = req.url + # make absolute url + url = urljoin_rfc(base_url, url, encoding) + url = safe_url_string(url, encoding) + # replace in-place request's url + req.url = url + + def _fix_link_text_encoding(self, encoding): + """Convert link_text to unicode for each request""" + for req in self.requests: + req.meta.setdefault('link_text', '') + req.meta['link_text'] = str_to_unicode(req.meta['link_text'], + encoding) + + def reset(self): + """Reset state""" + FixedSGMLParser.reset(self) + self.requests = [] + self.base_url = None + + def unknown_starttag(self, tag, attrs): + """Process unknown start tag""" + if 'base' == tag: + self.base_url = dict(attrs).get('href') + + if self.scan_tag(tag): + for attr, value in attrs: + if self.scan_attr(attr): + if value is not None: + req = Request(url=value) + self.requests.append(req) + self.current_request = req + + def unknown_endtag(self, tag): + """Process unknown end tag""" + self.current_request = None + + def handle_data(self, data): + """Process data""" + current = self.current_request + if current and not 'link_text' in current.meta: + current.meta['link_text'] = data.strip() + + +class SgmlRequestExtractor(BaseSgmlRequestExtractor): + """SGML Request Extractor""" + + def __init__(self, tags=None, attrs=None): + """Initialize with custom tag & attribute function checkers""" + # defaults + tags = tuple(tags) if tags else ('a', 'area') + attrs = tuple(attrs) if attrs else ('href', ) + + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) + + +class XPathRequestExtractor(SgmlRequestExtractor): + """SGML Request Extractor with XPath restriction""" + + def __init__(self, restrict_xpaths, tags=None, attrs=None): + """Initialize XPath restrictions""" + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + SgmlRequestExtractor.__init__(self, tags, attrs) + + def extract_requests(self, response): + """Restrict to XPath regions""" + hxs = HtmlXPathSelector(response) + fragments = (''.join( + html_frag for html_frag in hxs.select(xpath).extract() + ) for xpath in self.restrict_xpaths) + html_slice = ''.join(html_frag for html_frag in fragments) + return self._extract_requests(html_slice, response.url, + response.encoding) + +}}} + +=== Request Processor === +{{{ +#!python +# +# Request Processors +# Processors receive list of requests and return list of requests +# +"""Request Processors""" + +class Canonicalize(object): + """Canonicalize Request Processor""" + + def __call__(self, requests): + """Canonicalize all requests' urls""" + for req in requests: + # replace in-place + req.url = canonicalize_url(req.url) + yield req + + +class Unique(object): + """Filter duplicate Requests""" + + def __init__(self, *attributes): + """Initialize comparison attributes""" + self._attributes = attributes or ['url'] + + def _requests_equal(self, req1, req2): + """Attribute comparison helper""" + for attr in self._attributes: + if getattr(req1, attr) != getattr(req2, attr): + return False + # all attributes equal + return True + + def _request_in(self, request, requests_seen): + """Check if request is in given requests seen list""" + for seen in requests_seen: + if self._requests_equal(request, seen): + return True + # request not seen + return False + + def __call__(self, requests): + """Filter seen requests""" + # per-call duplicates filter + requests_seen = set() + for req in requests: + if not self._request_in(req, requests_seen): + yield req + # registry seen request + requests_seen.add(req) + + +class FilterDomain(object): + """Filter request's domain""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + self.allow = tuple(arg_to_iter(allow)) + self.deny = tuple(arg_to_iter(deny)) + + def __call__(self, requests): + """Filter domains""" + processed = (req for req in requests) + + if self.allow: + processed = (req for req in requests + if url_is_from_any_domain(req.url, self.allow)) + if self.deny: + processed = (req for req in requests + if not url_is_from_any_domain(req.url, self.deny)) + + return processed + + +class FilterUrl(object): + """Filter request's url""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + _re_type = type(re.compile('', 0)) + + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + def __call__(self, requests): + """Filter request's url based on allow/deny rules""" + #TODO: filter valid urls here? + processed = (req for req in requests) + + if self.allow_res: + processed = (req for req in requests + if self._matches(req.url, self.allow_res)) + if self.deny_res: + processed = (req for req in requests + if not self._matches(req.url, self.deny_res)) + + return processed + + def _matches(self, url, regexs): + """Returns True if url matches any regex in given list""" + return any(r.search(url) for r in regexs) + + +}}} + +=== Rule Object === +{{{ +#!python +# +# Dispatch Rules classes +# Manage Rules (Matchers + Callbacks) +# +class Rule(object): + """Crawler Rule""" + def __init__(self, matcher, callback=None, cb_args=None, + cb_kwargs=None, follow=True): + """Store attributes""" + self.matcher = matcher + self.callback = callback + self.cb_args = cb_args if cb_args else () + self.cb_kwargs = cb_kwargs if cb_kwargs else {} + self.follow = follow + +# +# Rules Manager takes list of Rule objects and normalize matcher and callback +# into CompiledRule +# +class CompiledRule(object): + """Compiled version of Rule""" + def __init__(self, matcher, callback=None, follow=False): + """Initialize attributes checking type""" + assert isinstance(matcher, BaseMatcher) + assert callback is None or callable(callback) + assert isinstance(follow, bool) + + self.matcher = matcher + self.callback = callback + self.follow = follow +}}} + +=== Rules Manager === +{{{ +#!python +# +# Handles rules matcher/callbacks +# Resolve rule for given response +# +class RulesManager(object): + """Rules Manager""" + def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): + """Initialize rules using spider and default matcher""" + self._rules = tuple() + + # compile absolute/relative-to-spider callbacks""" + for rule in rules: + # prepare matcher + if isinstance(rule.matcher, BaseMatcher): + matcher = rule.matcher + else: + # matcher not BaseMatcher, check for string + if isinstance(rule.matcher, basestring): + # instance default matcher + matcher = default_matcher(rule.matcher) + else: + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) + + # prepare callback + if callable(rule.callback): + callback = rule.callback + elif not rule.callback is None: + # callback from spider + callback = getattr(spider, rule.callback) + + if not callable(callback): + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) + else: + callback = None + + if rule.cb_args or rule.cb_kwargs: + # build partial callback + callback = partial(callback, *rule.cb_args, **rule.cb_kwargs) + + # append compiled rule to rules list + crule = CompiledRule(matcher, callback, follow=rule.follow) + self._rules += (crule, ) + + def get_rule(self, response): + """Returns first rule that matches response""" + for rule in self._rules: + if rule.matcher.matches_response(response): + return rule + +}}} + +=== Request Generator === +{{{ +#!python +# +# Request Generator +# Takes response and generate requests using extractors and processors +# +class RequestGenerator(object): + def __init__(self, req_extractors, req_processors, callback): + self._request_extractors = req_extractors + self._request_processors = req_processors + self.callback = callback + + def generate_requests(self, response): + """ + Extract and process new requets from response + """ + requests = [] + for ext in self._request_extractors: + requets.extend(ext.extract_requests(response)) + + for proc in self._request_processors: + requests = proc(requests) + + for request in requests: + yield request.replace(callback=self.callback) +}}} + +=== !CrawlSpider === +{{{ +#!python +# +# Spider +# +class CrawlSpider(InitSpider): + """CrawlSpider v2""" + + request_extractors = [] + request_processors = [] + rules = [] + + def __init__(self): + """Initialize dispatcher""" + super(CrawlSpider, self).__init__() + + # wrap rules + self._rulesman = RulesManager(self.rules, spider=self) + # generates new requests with given callback + self._reqgen = RequestGenerator(self.request_extractors, + self.request_processors, + self.parse) + + def parse(self, response): + """Dispatch callback and generate requests""" + # get rule for response + rule = self._rulesman.get_rule(response) + if rule: + # dispatch callback if set + if rule.callback: + output = iterate_spider_output(rule.callback(response)) + for req_or_item in output: + yield req_or_item + + if rule.follow: + for req in self._reqgen.generate_requests(response): + yield req + +}}} + diff --git a/sep/sep-016.trac b/sep/sep-016.trac index 9f7ada3c6..e9a0e5df0 100644 --- a/sep/sep-016.trac +++ b/sep/sep-016.trac @@ -1,265 +1,265 @@ -= SEP-016: Leg Spider = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||16|| -||'''Title:'''||Leg Spider|| -||'''Author:'''||Insophia Team|| -||'''Created:'''||2010-06-03|| -||'''Status'''||Superseded by [wiki:SEP-018]|| - -== Introduction == - -This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders. - -== Rationale == - -The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. - -The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider". - -== {{{LegSpider}}} API == - -A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods: - - * {{{legs}}} - * legs composing this spider - * {{{process_response(response)}}} - * Process a (downloaded) response and return a list of requests and items - * {{{process_request(request)}}} - * Process a request after it has been extracted and before returning it from the spider - * {{{process_item(item)}}} - * Process an item after it has been extracted and before returning it from the spider - * {{{set_spider()}}} - * Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior. - -== How Leg Spiders work == - - 1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output. - 2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself. - -== Leg Spider examples == - -=== Regex (HTML) Link Extractor === - -A typical application of LegSpider's is to build Link Extractors. For example: - -{{{ -#!python -class RegexHtmlLinkExtractor(LegSpider): - - def process_response(self, response): - if isinstance(response, HtmlResponse): - allowed_regexes = self.spider.url_regexes_to_follow - # extract urls to follow using allowed_regexes - return [Request(x) for x in urls_to_follow] - -class MySpider(LegSpider): - - legs = [RegexHtmlLinkExtractor()] - url_regexes_to_follow = ['/product.php?.*'] - - def parse_response(self, response): - # parse response and extract items - return items -}}} - -=== RSS2 link extractor === - -This is a Leg Spider that can be used for following links from RSS2 feeds. - -{{{ -#!python -class Rss2LinkExtractor(LegSpider): - - def process_response(self, response): - if response.headers.get('Content-type') == 'application/rss+xml': - xs = XmlXPathSelector(response) - urls = xs.select("//item/link/text()").extract() - return [Request(x) for x in urls] -}}} - -=== Callback dispatcher based on rules === - -Another example could be to build a callback dispatcher based on rules: - -{{{ -#!python -class CallbackRules(LegSpider): - - def __init__(self, *a, **kw): - super(CallbackRules, self).__init__(*a, **kw) - for regex, method_name in self.spider.callback_rules.items(): - r = re.compile(regex) - m = getattr(self.spider, method_name, None) - if m: - self._rules[r] = m - - def process_response(self, response): - for regex, method in self._rules.items(): - m = regex.search(response.url) - if m: - return method(response) - return [] - -class MySpider(LegSpider): - - legs = [CallbackRules()] - callback_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - def parse_product(self, response): - # parse reponse and populate item - return item -}}} - -=== URL Canonicalizers === - -Another example could be for building URL canonicalizers: - -{{{ -#!python -class CanonializeUrl(LegSpider): - - def process_request(self, request): - curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules) - return request.replace(url=curl) - -class MySpider(LegSpider): - - legs = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - # ... -}}} - -=== Setting item identifier === - -Another example could be for setting a unique identifier to items, based on certain fields: - -{{{ -#!python -class ItemIdSetter(LegSpider): - - def process_item(self, item): - id_field = self.spider.id_field - id_fields_to_hash = self.spider.id_fields_to_hash - item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) - return item - -class MySpider(LegSpider): - - legs = [ItemIdSetter()] - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def process_response(self, item): - # extract item from response - return item -}}} - -=== Combining multiple leg spiders === - -Here's an example that combines functionality from multiple leg spiders: - -{{{ -#!python -class MySpider(LegSpider): - - legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()] - - url_regexes_to_follow = ['/product.php?.*'] - - parse_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def process_product(self, item): - # extract item from response - return item - - def process_category(self, item): - # extract item from response - return item -}}} - - - -== Leg Spiders vs Spider middlewares == - -A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality. - -== When not to use Leg Spiders == - -Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as: - - * Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware. - * Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit. - -== {{{LegSpider}}} proof-of-concept implementation == - -Here's a proof-of-concept implementation of {{{LegSpider}}}: - -{{{ -#!python -from scrapy.http import Request -from scrapy.item import BaseItem -from scrapy.spider import BaseSpider -from scrapy.utils.spider import iterate_spider_output - - -class LegSpider(BaseSpider): - """A spider made of legs""" - - legs = [] - - def __init__(self, *args, **kwargs): - super(LegSpider, self).__init__(*args, **kwargs) - self._legs = [self] + self.legs[:] - for l in self._legs: - l.set_spider(self) - - def parse(self, response): - res = self._process_response(response) - for r in res: - if isinstance(r, BaseItem): - yield self._process_item(r) - else: - yield self._process_request(r) - - def process_response(self, response): - return [] - - def process_request(self, request): - return request - - def process_item(self, item): - return item - - def set_spider(self, spider): - self.spider = spider - - def _process_response(self, response): - res = [] - for l in self._legs: - res.extend(iterate_spider_output(l.process_response(response))) - return res - - def _process_request(self, request): - for l in self._legs: - request = l.process_request(request) - return request - - def _process_item(self, item): - for l in self._legs: - item = l.process_item(item) - return item += SEP-016: Leg Spider = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||16|| +||'''Title:'''||Leg Spider|| +||'''Author:'''||Insophia Team|| +||'''Created:'''||2010-06-03|| +||'''Status'''||Superseded by [wiki:SEP-018]|| + +== Introduction == + +This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders. + +== Rationale == + +The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. + +The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider". + +== {{{LegSpider}}} API == + +A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods: + + * {{{legs}}} + * legs composing this spider + * {{{process_response(response)}}} + * Process a (downloaded) response and return a list of requests and items + * {{{process_request(request)}}} + * Process a request after it has been extracted and before returning it from the spider + * {{{process_item(item)}}} + * Process an item after it has been extracted and before returning it from the spider + * {{{set_spider()}}} + * Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior. + +== How Leg Spiders work == + + 1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output. + 2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself. + +== Leg Spider examples == + +=== Regex (HTML) Link Extractor === + +A typical application of LegSpider's is to build Link Extractors. For example: + +{{{ +#!python +class RegexHtmlLinkExtractor(LegSpider): + + def process_response(self, response): + if isinstance(response, HtmlResponse): + allowed_regexes = self.spider.url_regexes_to_follow + # extract urls to follow using allowed_regexes + return [Request(x) for x in urls_to_follow] + +class MySpider(LegSpider): + + legs = [RegexHtmlLinkExtractor()] + url_regexes_to_follow = ['/product.php?.*'] + + def parse_response(self, response): + # parse response and extract items + return items +}}} + +=== RSS2 link extractor === + +This is a Leg Spider that can be used for following links from RSS2 feeds. + +{{{ +#!python +class Rss2LinkExtractor(LegSpider): + + def process_response(self, response): + if response.headers.get('Content-type') == 'application/rss+xml': + xs = XmlXPathSelector(response) + urls = xs.select("//item/link/text()").extract() + return [Request(x) for x in urls] +}}} + +=== Callback dispatcher based on rules === + +Another example could be to build a callback dispatcher based on rules: + +{{{ +#!python +class CallbackRules(LegSpider): + + def __init__(self, *a, **kw): + super(CallbackRules, self).__init__(*a, **kw) + for regex, method_name in self.spider.callback_rules.items(): + r = re.compile(regex) + m = getattr(self.spider, method_name, None) + if m: + self._rules[r] = m + + def process_response(self, response): + for regex, method in self._rules.items(): + m = regex.search(response.url) + if m: + return method(response) + return [] + +class MySpider(LegSpider): + + legs = [CallbackRules()] + callback_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + def parse_product(self, response): + # parse response and populate item + return item +}}} + +=== URL Canonicalizers === + +Another example could be for building URL canonicalizers: + +{{{ +#!python +class CanonializeUrl(LegSpider): + + def process_request(self, request): + curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules) + return request.replace(url=curl) + +class MySpider(LegSpider): + + legs = [CanonicalizeUrl()] + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + # ... +}}} + +=== Setting item identifier === + +Another example could be for setting a unique identifier to items, based on certain fields: + +{{{ +#!python +class ItemIdSetter(LegSpider): + + def process_item(self, item): + id_field = self.spider.id_field + id_fields_to_hash = self.spider.id_fields_to_hash + item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) + return item + +class MySpider(LegSpider): + + legs = [ItemIdSetter()] + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def process_response(self, item): + # extract item from response + return item +}}} + +=== Combining multiple leg spiders === + +Here's an example that combines functionality from multiple leg spiders: + +{{{ +#!python +class MySpider(LegSpider): + + legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()] + + url_regexes_to_follow = ['/product.php?.*'] + + parse_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def process_product(self, item): + # extract item from response + return item + + def process_category(self, item): + # extract item from response + return item +}}} + + + +== Leg Spiders vs Spider middlewares == + +A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality. + +== When not to use Leg Spiders == + +Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as: + + * Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware. + * Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit. + +== {{{LegSpider}}} proof-of-concept implementation == + +Here's a proof-of-concept implementation of {{{LegSpider}}}: + +{{{ +#!python +from scrapy.http import Request +from scrapy.item import BaseItem +from scrapy.spider import BaseSpider +from scrapy.utils.spider import iterate_spider_output + + +class LegSpider(BaseSpider): + """A spider made of legs""" + + legs = [] + + def __init__(self, *args, **kwargs): + super(LegSpider, self).__init__(*args, **kwargs) + self._legs = [self] + self.legs[:] + for l in self._legs: + l.set_spider(self) + + def parse(self, response): + res = self._process_response(response) + for r in res: + if isinstance(r, BaseItem): + yield self._process_item(r) + else: + yield self._process_request(r) + + def process_response(self, response): + return [] + + def process_request(self, request): + return request + + def process_item(self, item): + return item + + def set_spider(self, spider): + self.spider = spider + + def _process_response(self, response): + res = [] + for l in self._legs: + res.extend(iterate_spider_output(l.process_response(response))) + return res + + def _process_request(self, request): + for l in self._legs: + request = l.process_request(request) + return request + + def _process_item(self, item): + for l in self._legs: + item = l.process_item(item) + return item }}} \ No newline at end of file diff --git a/sep/sep-018.trac b/sep/sep-018.trac index 9c2a0349b..e09356d49 100644 --- a/sep/sep-018.trac +++ b/sep/sep-018.trac @@ -1,551 +1,551 @@ -= SEP-018: Spider middleware v2 = - -[[PageOutline(2-5,Contents)]] - -||'''SEP:'''||18|| -||'''Title:'''||Spider Middleware v2|| -||'''Author:'''||Insophia Team|| -||'''Created:'''||2010-06-20|| -||'''Status'''||Draft (in progress)|| - -This SEP introduces a new architecture for spider middlewares which provides a greater degree of modularity to combine functionality which can be plugged in from different (reusable) middlewares. - -The purpose of !SpiderMiddleware-v2 is to define an architecture that encourages more re-usability for building spiders based on smaller well-tested components. Those components can be global (similar to current spider middlewares) or per-spider that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. - -Unless explicitly stated, in this document "spider middleware" refers to the '''new''' spider middleware v2, not the old one. - -This document is a work in progress, see [#Pendingissues Pending issues] below. - -== New spider middleware API == - -A spider middleware can implement any of the following methods: - - * `process_response(response, request, spider)` - * Process a (downloaded) response - * Receives: The `response` to process, the `request` used to download the response (not necessarily the request sent from the spider), and the `spider` that requested it. - * Returns: A list containing requests and/or items - * `process_error(error, request, spider)`: - * Process a error when trying to download a request, such as DNS errors, timeout errors, etc. - * Receives: The `error` caused, the `request` that caused it (not necessarily the request sent from the spider), and then `spider` that requested it. - * Returns: A list containing request and/or items - * `process_request(request, response, spider)` - * Process a request after it has been extracted from the spider or previous middleware `process_response()` methods. - * Receives: The `request` to process, the `response` where the request was extracted from, and the `spider` that extracted it. - * Note: `response` is `None` for start requests, or requests injected directly (through `manager.scraper.process_request()` without specifying a response (see below) - * Returns: A `Request` object (not necessarily the same received), or `None` in which case the request is dropped. - * `process_item(item, response, spider)` - * Process an item after it has been extracted from the spider or previous middleware `process_response()` methods. - * Receives: The `item` to process, the `response` where the item was extracted from, and the `spider` that extracted it. - * Returns: An `Item` object (not necessarily the same received), or `None` in which case the item is dropped. - * `next_request(spider)` - * Returns a the next request to crawl with this spider. This method is called when the spider is opened, and when it gets idle. - * Receives: The `spider` to return the next request for. - * Returns: A `Request` object. - * `open_spider(spider)` - * This can be used to allocate resources when a spider is opened. - * Receives: The `spider` that has been opened. - * Returns: nothing - * `close_spider(spider)` - * This can be used to free resources when a spider is closed. - * Receives: The `spider` that has been closed. - * Returns: nothing - -== Changes to core API == - -=== Injecting requests to crawl === - -To inject start requests (or new requests without a response) to crawl, you used before: - - * `manager.engine.crawl(request, spider)` - -Now you'll use: - - * `manager.scraper.process_request(request, spider, response=None)` - -Which (unlike the old `engine.crawl` will make the requests pass through the spider middleware `process_request()` method). - -=== Scheduler middleware to be removed === - -We're gonna remove the Scheduler Middleware, and move the duplicates filter to a new spider middleware. - -== Scraper high-level API == - -There is a simpler high-level API - the Scraper API - which is the API used by the engine and other core components. This is also the API implemented by this new middleware, with its own internal architecture and hooks. Here is the Scraper API: - - * `process_response(response, request, spider)` - * returns iterable of items and requests - * `process_error(error, request, spider)` - * returns iterable of items and requests - * `process_request(request, spider, response=None)` - * injects a request to crawl for the given spider - * `process_item(item, spider, response) - * injects a item to process with the item processor (typically the item pipeline) - * `next_request(spider)` - * returns the next request to process for the given spider - * `open_spider(spider)` - * opens a spider - * `close_spider(spider)` - * closes a spider - -== How it works == - -The spider middlewares are defined in certain order with the top-most being the one closer to the engine, and the bottom-most being the one closed to the spider. - -Example: - - * Engine - * Global spider Middleware 3 - * Global spider Middleware 2 - * Global spider Middleware 1 - * Spider-specific middlewares (defined in `Spider.middlewares`) - * Spider-specific middleware 3 - * Spider-specific middleware 2 - * Spider-specific middleware 1 - * Spider - -The data flow with Spider Middleware v2 is as follows: - - 1. When a response arrives from the engine, it it passed through all the spider middlewares (in descending order). The result of each middleware `process_response` is kept and then returned along with the spider callback result - 2. Each item of the aggregated result from previous point is passed through all middlewares (in ascending order) calling the `process_request` or `process_item` method accordingly, and their results are kept for passing to the following middlewares - -One of the spider middlewares (typically - but not necessarily - the last spider middleware closer to the spider, as shown in the example) will be a "spider-specific spider middleware" which would take care of calling the additional spider middlewares defined in the `Spider.middlewares` attribute, hence providing support for per-spider middlewares. If the middleware is well written, it should work both globally and per-spider. - -== Spider-specific middlewares == - -You can define in the spider itself a list of additional middlewares that will be used for this spider, and only this spider. If the middleware is well written, it should work both globally and per spider. - -Here's an example that combines functionality from multiple middlewares into the same spider: - -{{{ -#!python -class MySpider(BaseSpider): - - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] - - allowed_domains = ['example.com', 'sub.example.com'] - - url_regexes_to_follow = ['/product.php?.*'] - - callback_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def parse_product(self, item): - # extract item from response - return item - - def parse_category(self, item): - # extract item from response - return item -}}} - -== The Spider Middleware that implements spider code == - -There's gonna be one middleware that will take care of calling the proper spider methods on each event such as: - - * call `Request.callback` (for 200 responses) or `Request.errback` for non-200 responses and other errors. this behaviour can be changed through the `handle_httpstatus_list` spider attribute. - * if `Request.callback` is not set it will use `Spider.parse` - * if `Request.errback` is not set it will use `Spider.errback` - * call additional spider middlewares defined in the `Spider.middlewares` attribute - * call `Spider.next_request()` and `Spider.start_requests()` on `next_request()` middleware method (this would implicitly support backwards compatibility) - -== Differences with Spider middleware v1 == - - * adds support for per-spider middlewares through the `Spider.middlewares` attribute - * allows processing initial requests (those returned from `Spider.start_requests()`) - -== Use cases and examples == - -This section contains several examples and use cases for Spider Middlewares. Imports are intentionally removed for conciseness and clarity. - -=== Regex (HTML) Link Extractor === - -A typical application of spider middlewares could be to build Link Extractors. For example: - -{{{ -#!python -class RegexHtmlLinkExtractor(object): - - def process_response(self, response, request, spider): - if isinstance(response, HtmlResponse): - allowed_regexes = spider.url_regexes_to_follow - # extract urls to follow using allowed_regexes - return [Request(x) for x in urls_to_follow] - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [RegexHtmlLinkExtractor()] - url_regexes_to_follow = ['/product.php?.*'] - - # parsing callbacks below -}}} - -=== RSS2 link extractor === - -{{{ -#!python -class Rss2LinkExtractor(object): - - def process_response(self, response, request, spider): - if response.headers.get('Content-type') == 'application/rss+xml': - xs = XmlXPathSelector(response) - urls = xs.select("//item/link/text()").extract() - return [Request(x) for x in urls] -}}} - -=== Callback dispatcher based on rules === - -Another example could be to build a callback dispatcher based on rules: - -{{{ -#!python -class CallbackRules(object): - - def __init__(self): - self.rules = {} - dispatcher.connect(signals.spider_opened, self.spider_opened) - dispatcher.connect(signals.spider_closed, self.spider_closed) - - def spider_opened(self, spider): - self.rules[spider] = {} - for regex, method_name in spider.callback_rules.items(): - r = re.compile(regex) - m = getattr(self.spider, method_name, None) - if m: - self.rules[spider][r] = m - - def spider_closed(self, spider): - del self.rules[spider] - - def process_response(self, response, request, spider): - for regex, method in self.rules[spider].items(): - m = regex.search(response.url) - if m: - return method(response) - return [] - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [CallbackRules()] - callback_rules = { - '/product.php.*': 'parse_product', - '/category.php.*': 'parse_category', - } - - def parse_product(self, response): - # parse reponse and populate item - return item -}}} - -=== URL Canonicalizers === - -Another example could be for building URL canonicalizers: - -{{{ -#!python -class CanonializeUrl(object): - - def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) - return request.replace(url=curl) - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] - - # ... -}}} - -=== Setting item identifier === - -Another example could be for setting a unique identifier to items, based on certain fields: - -{{{ -#!python -class ItemIdSetter(object): - - def process_item(self, item, response, spider): - id_field = spider.id_field - id_fields_to_hash = spider.id_fields_to_hash - item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) - return item - -# Example spider using this middleware -class MySpider(BaseSpider): - - middlewares = [ItemIdSetter()] - id_field = 'guid' - id_fields_to_hash = ['supplier_name', 'supplier_id'] - - def parse(self, response): - # extract item from response - return item -}}} - -=== robots.txt exclusion === - -A spider middleware to avoid visiting pages forbidden by robots.txt: - -{{{ -#!python -class SpiderInfo(object): - - def __init__(self, useragent): - self.useragent = useragent - self.parsers = {} - self.pending = defaultdict(list) - - -class AllowAllParser(object): - - def can_fetch(useragent, url): - return True - - -class RobotsTxtMiddleware(object): - - REQUEST_PRIORITY = 1000 - - def __init__(self): - self.spiders = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_request(self, request, response, spider): - return self.process_start_request(self, request) - - def process_start_request(self, request, spider): - info = self.spiders[spider] - url = urlparse_cached(request) - netloc = url.netloc - if netloc in info.parsers: - rp = info.parsers[netloc] - if rp.can_fetch(info.useragent, request.url): - res = request - else: - spider.log("Forbidden by robots.txt: %s" % request) - res = None - else: - if netloc in info.pending: - res = None - else: - robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) - meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} - res = Request(robotsurl, callback=self.parse_robots, - meta=meta, priority=self.REQUEST_PRIORITY) - info.pending[netloc].append(request) - return res - - def parse_robots(self, response): - spider = response.request.meta['spider'] - netloc urlparse_cached(response).netloc - info = self.spiders[spider] - if response.status == 200; - rp = robotparser.RobotFileParser(response.url) - rp.parse(response.body.splitlines()) - info.parsers[netloc] = rp - else: - info.parsers[netloc] = AllowAllParser() - return info.pending[netloc] - - def spider_opened(self, spider): - ua = getattr(spider, 'user_agent', None) or settings['USER_AGENT'] - self.spiders[spider] = SpiderInfo(ua) - - def spider_closed(self, spider): - del self.spiders[spider] -}}} - -=== Offsite middleware === - -This is a port of the Offsite middleware to the new spider middleware API: - -{{{ -#!python -class SpiderInfo(object): - - def __init__(self, host_regex): - self.host_regex = host_regex - self.hosts_seen = set() - - -class OffsiteMiddleware(object): - - def __init__(self): - self.spiders = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_request(self, request, response, spider): - return self.process_start_request(self, request) - - def process_start_request(self, request, spider): - if self.should_follow(request, spider): - return request - else: - info = self.spiders[spider] - host = urlparse_cached(x).hostname - if host and host not in info.hosts_seen: - spider.log("Filtered offsite request to %r: %s" % (host, request)) - info.hosts_seen.add(host) - - def should_follow(self, request, spider): - info = self.spiders[spider] - # hostanme can be None for wrong urls (like javascript links) - host = urlparse_cached(request).hostname or '' - return bool(info.regex.search(host)) - - def get_host_regex(self, spider): - """Override this method to implement a different offsite policy""" - domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) - return re.compile(regex) - - def spider_opened(self, spider): - info = SpiderInfo(self.get_host_regex(spider)) - self.spiders[spider] = info - - def spider_closed(self, spider): - del self.spiders[spider] - -}}} - -=== Limit URL length === - -A middleware to filter out requests with long urls: - -{{{ -#!python - -class LimitUrlLength(object): - - def __init__(self): - self.maxlength = settings.getint('URLLENGTH_LIMIT') - - def process_request(self, request, response, spider): - return self.process_start_request(self, request) - - def process_start_request(self, request, spider): - if len(request.url) <= self.maxlength: - return request - spider.log("Ignoring request (url length > %d): %s " % (self.maxlength, request.url)) -}}} - -=== Set Referer === - -A middleware to set the Referer: - -{{{ -#!python -class SetReferer(object): - - def process_request(self, request, response, spider): - request.headers.setdefault('Referer', response.url) - return request -}}} - -=== Set and limit crawling depth === - -A middleware to set (and limit) the request/response depth, taken from the start requests: - -{{{ -#!python -class SetLimitDepth(object): - - def __init__(self, maxdepth=0): - self.maxdepth = maxdepth or settings.getint('DEPTH_LIMIT') - - def process_request(self, request, response, spider): - depth = response.request.meta['depth'] + 1 - request.meta['depth'] = depth - if not self.maxdepth or depth <= self.maxdepth: - return request - spider.log("Ignoring link (depth > %d): %s " % (self.maxdepth, request) - - def process_start_request(self, request, spider): - request.meta['depth'] = 0 - return request -}}} - -=== Filter duplicate requests === - -A middleware to filter out requests already seen: - -{{{ -#!python -class FilterDuplicates(object): - - def __init__(self): - clspath = settings.get('DUPEFILTER_CLASS') - self.dupefilter = load_object(clspath)() - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def enqueue_request(self, spider, request): - seen = self.dupefilter.request_seen(spider, request) - if not seen or request.dont_filter: - return request - - def spider_opened(self, spider): - self.dupefilter.open_spider(spider) - - def spider_closed(self, spider): - self.dupefilter.close_spider(spider) -}}} - -=== Scrape data using Parsley === - -A middleware to Scrape data using Parsley as described in UsingParsley - -{{{ -#!python -from pyparsley import PyParsley - -class ParsleyExtractor(object): - - def __init__(self, parslet_json_code): - parslet = json.loads(parselet_json_code) - class ParsleyItem(Item): - def __init__(self, *a, **kw): - for name in parslet.keys(): - self.fields[name] = Field() - super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') - - def process_response(self, response, request, spider): - return self.item_class(self.parsly.parse(string=response.body)) -}}} - - - -== Pending issues == - -Resolved: - - * how to make `start_requests()` output pass through spider middleware `process_request()`? - * Start requests will be injected through `manager.scraper.process_request()` instead of `manager.engine.crawl()` - * should we support adding additional start requests from a spider middleware? - * Yes - there is a spider middleware method (`start_requests`) for that - * should `process_response()` receive a `request` argument with the `request` that originated it?. `response.request` is the latest request, not the original one (think of redirections), but it does carry the `meta` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be the deserialized request from the persistent scheduler queue. - * No - this would make implementation more complex and we're not sure it's really needed - * how to make sure `Request.errback` is always called if there is a problem with the request?. Do we need to ensure that?. Requests filtered out (by returning `None`) in the `process_request()` method will never be callback-ed or even errback-ed. this could be a problem for spiders that want to be notified if their requests are dropped. should we support this notification somehow or document (the lack of) it properly? - * We won't support notifications of dropped requests, because: 1. it's hard to implement and unreliable, 2. it's against not friendly with request persistence, 3. we can't come up with a good api. - * should we make the list of default spider middlewares empty? (or the "per-spider" spider middleware alone) - * No - there are some useful spider middlewares that it's worth enabling by default like referer, duplicates, robots2 - * should we allow returning deferreds in spider middleware methods? - * Yes - we should build a Deferred with the spider middleware methods as callbacks and that would implicitly support returning Deferreds - * should we support processing responses before they're processed by the spider, because `process_response` runs "in parallel" to the spider callback, and can't stop from running it. - * No - we haven't seen a practical use case for this, so we won't add an additional hook. It should be trivial to add it later, if needed. - * should we make a spider middleware to handle calling the request and spider callback, instead of letting the Scraper component do it? - * Yes - there's gonna a spider middleware for execution spider-specific code such as callbacks and also custom middlewares += SEP-018: Spider middleware v2 = + +[[PageOutline(2-5,Contents)]] + +||'''SEP:'''||18|| +||'''Title:'''||Spider Middleware v2|| +||'''Author:'''||Insophia Team|| +||'''Created:'''||2010-06-20|| +||'''Status'''||Draft (in progress)|| + +This SEP introduces a new architecture for spider middlewares which provides a greater degree of modularity to combine functionality which can be plugged in from different (reusable) middlewares. + +The purpose of !SpiderMiddleware-v2 is to define an architecture that encourages more re-usability for building spiders based on smaller well-tested components. Those components can be global (similar to current spider middlewares) or per-spider that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components that can be shared among different spiders and projects. Some of them will come bundled with Scrapy. + +Unless explicitly stated, in this document "spider middleware" refers to the '''new''' spider middleware v2, not the old one. + +This document is a work in progress, see [#Pendingissues Pending issues] below. + +== New spider middleware API == + +A spider middleware can implement any of the following methods: + + * `process_response(response, request, spider)` + * Process a (downloaded) response + * Receives: The `response` to process, the `request` used to download the response (not necessarily the request sent from the spider), and the `spider` that requested it. + * Returns: A list containing requests and/or items + * `process_error(error, request, spider)`: + * Process a error when trying to download a request, such as DNS errors, timeout errors, etc. + * Receives: The `error` caused, the `request` that caused it (not necessarily the request sent from the spider), and then `spider` that requested it. + * Returns: A list containing request and/or items + * `process_request(request, response, spider)` + * Process a request after it has been extracted from the spider or previous middleware `process_response()` methods. + * Receives: The `request` to process, the `response` where the request was extracted from, and the `spider` that extracted it. + * Note: `response` is `None` for start requests, or requests injected directly (through `manager.scraper.process_request()` without specifying a response (see below) + * Returns: A `Request` object (not necessarily the same received), or `None` in which case the request is dropped. + * `process_item(item, response, spider)` + * Process an item after it has been extracted from the spider or previous middleware `process_response()` methods. + * Receives: The `item` to process, the `response` where the item was extracted from, and the `spider` that extracted it. + * Returns: An `Item` object (not necessarily the same received), or `None` in which case the item is dropped. + * `next_request(spider)` + * Returns a the next request to crawl with this spider. This method is called when the spider is opened, and when it gets idle. + * Receives: The `spider` to return the next request for. + * Returns: A `Request` object. + * `open_spider(spider)` + * This can be used to allocate resources when a spider is opened. + * Receives: The `spider` that has been opened. + * Returns: nothing + * `close_spider(spider)` + * This can be used to free resources when a spider is closed. + * Receives: The `spider` that has been closed. + * Returns: nothing + +== Changes to core API == + +=== Injecting requests to crawl === + +To inject start requests (or new requests without a response) to crawl, you used before: + + * `manager.engine.crawl(request, spider)` + +Now you'll use: + + * `manager.scraper.process_request(request, spider, response=None)` + +Which (unlike the old `engine.crawl` will make the requests pass through the spider middleware `process_request()` method). + +=== Scheduler middleware to be removed === + +We're gonna remove the Scheduler Middleware, and move the duplicates filter to a new spider middleware. + +== Scraper high-level API == + +There is a simpler high-level API - the Scraper API - which is the API used by the engine and other core components. This is also the API implemented by this new middleware, with its own internal architecture and hooks. Here is the Scraper API: + + * `process_response(response, request, spider)` + * returns iterable of items and requests + * `process_error(error, request, spider)` + * returns iterable of items and requests + * `process_request(request, spider, response=None)` + * injects a request to crawl for the given spider + * `process_item(item, spider, response) + * injects a item to process with the item processor (typically the item pipeline) + * `next_request(spider)` + * returns the next request to process for the given spider + * `open_spider(spider)` + * opens a spider + * `close_spider(spider)` + * closes a spider + +== How it works == + +The spider middlewares are defined in certain order with the top-most being the one closer to the engine, and the bottom-most being the one closed to the spider. + +Example: + + * Engine + * Global spider Middleware 3 + * Global spider Middleware 2 + * Global spider Middleware 1 + * Spider-specific middlewares (defined in `Spider.middlewares`) + * Spider-specific middleware 3 + * Spider-specific middleware 2 + * Spider-specific middleware 1 + * Spider + +The data flow with Spider Middleware v2 is as follows: + + 1. When a response arrives from the engine, it it passed through all the spider middlewares (in descending order). The result of each middleware `process_response` is kept and then returned along with the spider callback result + 2. Each item of the aggregated result from previous point is passed through all middlewares (in ascending order) calling the `process_request` or `process_item` method accordingly, and their results are kept for passing to the following middlewares + +One of the spider middlewares (typically - but not necessarily - the last spider middleware closer to the spider, as shown in the example) will be a "spider-specific spider middleware" which would take care of calling the additional spider middlewares defined in the `Spider.middlewares` attribute, hence providing support for per-spider middlewares. If the middleware is well written, it should work both globally and per-spider. + +== Spider-specific middlewares == + +You can define in the spider itself a list of additional middlewares that will be used for this spider, and only this spider. If the middleware is well written, it should work both globally and per spider. + +Here's an example that combines functionality from multiple middlewares into the same spider: + +{{{ +#!python +class MySpider(BaseSpider): + + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] + + allowed_domains = ['example.com', 'sub.example.com'] + + url_regexes_to_follow = ['/product.php?.*'] + + callback_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def parse_product(self, item): + # extract item from response + return item + + def parse_category(self, item): + # extract item from response + return item +}}} + +== The Spider Middleware that implements spider code == + +There's gonna be one middleware that will take care of calling the proper spider methods on each event such as: + + * call `Request.callback` (for 200 responses) or `Request.errback` for non-200 responses and other errors. this behaviour can be changed through the `handle_httpstatus_list` spider attribute. + * if `Request.callback` is not set it will use `Spider.parse` + * if `Request.errback` is not set it will use `Spider.errback` + * call additional spider middlewares defined in the `Spider.middlewares` attribute + * call `Spider.next_request()` and `Spider.start_requests()` on `next_request()` middleware method (this would implicitly support backwards compatibility) + +== Differences with Spider middleware v1 == + + * adds support for per-spider middlewares through the `Spider.middlewares` attribute + * allows processing initial requests (those returned from `Spider.start_requests()`) + +== Use cases and examples == + +This section contains several examples and use cases for Spider Middlewares. Imports are intentionally removed for conciseness and clarity. + +=== Regex (HTML) Link Extractor === + +A typical application of spider middlewares could be to build Link Extractors. For example: + +{{{ +#!python +class RegexHtmlLinkExtractor(object): + + def process_response(self, response, request, spider): + if isinstance(response, HtmlResponse): + allowed_regexes = spider.url_regexes_to_follow + # extract urls to follow using allowed_regexes + return [Request(x) for x in urls_to_follow] + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [RegexHtmlLinkExtractor()] + url_regexes_to_follow = ['/product.php?.*'] + + # parsing callbacks below +}}} + +=== RSS2 link extractor === + +{{{ +#!python +class Rss2LinkExtractor(object): + + def process_response(self, response, request, spider): + if response.headers.get('Content-type') == 'application/rss+xml': + xs = XmlXPathSelector(response) + urls = xs.select("//item/link/text()").extract() + return [Request(x) for x in urls] +}}} + +=== Callback dispatcher based on rules === + +Another example could be to build a callback dispatcher based on rules: + +{{{ +#!python +class CallbackRules(object): + + def __init__(self): + self.rules = {} + dispatcher.connect(signals.spider_opened, self.spider_opened) + dispatcher.connect(signals.spider_closed, self.spider_closed) + + def spider_opened(self, spider): + self.rules[spider] = {} + for regex, method_name in spider.callback_rules.items(): + r = re.compile(regex) + m = getattr(self.spider, method_name, None) + if m: + self.rules[spider][r] = m + + def spider_closed(self, spider): + del self.rules[spider] + + def process_response(self, response, request, spider): + for regex, method in self.rules[spider].items(): + m = regex.search(response.url) + if m: + return method(response) + return [] + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [CallbackRules()] + callback_rules = { + '/product.php.*': 'parse_product', + '/category.php.*': 'parse_category', + } + + def parse_product(self, response): + # parse response and populate item + return item +}}} + +=== URL Canonicalizers === + +Another example could be for building URL canonicalizers: + +{{{ +#!python +class CanonializeUrl(object): + + def process_request(self, request, response, spider): + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) + return request.replace(url=curl) + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [CanonicalizeUrl()] + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] + + # ... +}}} + +=== Setting item identifier === + +Another example could be for setting a unique identifier to items, based on certain fields: + +{{{ +#!python +class ItemIdSetter(object): + + def process_item(self, item, response, spider): + id_field = spider.id_field + id_fields_to_hash = spider.id_fields_to_hash + item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash) + return item + +# Example spider using this middleware +class MySpider(BaseSpider): + + middlewares = [ItemIdSetter()] + id_field = 'guid' + id_fields_to_hash = ['supplier_name', 'supplier_id'] + + def parse(self, response): + # extract item from response + return item +}}} + +=== robots.txt exclusion === + +A spider middleware to avoid visiting pages forbidden by robots.txt: + +{{{ +#!python +class SpiderInfo(object): + + def __init__(self, useragent): + self.useragent = useragent + self.parsers = {} + self.pending = defaultdict(list) + + +class AllowAllParser(object): + + def can_fetch(useragent, url): + return True + + +class RobotsTxtMiddleware(object): + + REQUEST_PRIORITY = 1000 + + def __init__(self): + self.spiders = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + def process_request(self, request, response, spider): + return self.process_start_request(self, request) + + def process_start_request(self, request, spider): + info = self.spiders[spider] + url = urlparse_cached(request) + netloc = url.netloc + if netloc in info.parsers: + rp = info.parsers[netloc] + if rp.can_fetch(info.useragent, request.url): + res = request + else: + spider.log("Forbidden by robots.txt: %s" % request) + res = None + else: + if netloc in info.pending: + res = None + else: + robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) + meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} + res = Request(robotsurl, callback=self.parse_robots, + meta=meta, priority=self.REQUEST_PRIORITY) + info.pending[netloc].append(request) + return res + + def parse_robots(self, response): + spider = response.request.meta['spider'] + netloc urlparse_cached(response).netloc + info = self.spiders[spider] + if response.status == 200; + rp = robotparser.RobotFileParser(response.url) + rp.parse(response.body.splitlines()) + info.parsers[netloc] = rp + else: + info.parsers[netloc] = AllowAllParser() + return info.pending[netloc] + + def spider_opened(self, spider): + ua = getattr(spider, 'user_agent', None) or settings['USER_AGENT'] + self.spiders[spider] = SpiderInfo(ua) + + def spider_closed(self, spider): + del self.spiders[spider] +}}} + +=== Offsite middleware === + +This is a port of the Offsite middleware to the new spider middleware API: + +{{{ +#!python +class SpiderInfo(object): + + def __init__(self, host_regex): + self.host_regex = host_regex + self.hosts_seen = set() + + +class OffsiteMiddleware(object): + + def __init__(self): + self.spiders = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + def process_request(self, request, response, spider): + return self.process_start_request(self, request) + + def process_start_request(self, request, spider): + if self.should_follow(request, spider): + return request + else: + info = self.spiders[spider] + host = urlparse_cached(x).hostname + if host and host not in info.hosts_seen: + spider.log("Filtered offsite request to %r: %s" % (host, request)) + info.hosts_seen.add(host) + + def should_follow(self, request, spider): + info = self.spiders[spider] + # hostanme can be None for wrong urls (like javascript links) + host = urlparse_cached(request).hostname or '' + return bool(info.regex.search(host)) + + def get_host_regex(self, spider): + """Override this method to implement a different offsite policy""" + domains = [d.replace('.', r'\.') for d in spider.allowed_domains] + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + return re.compile(regex) + + def spider_opened(self, spider): + info = SpiderInfo(self.get_host_regex(spider)) + self.spiders[spider] = info + + def spider_closed(self, spider): + del self.spiders[spider] + +}}} + +=== Limit URL length === + +A middleware to filter out requests with long urls: + +{{{ +#!python + +class LimitUrlLength(object): + + def __init__(self): + self.maxlength = settings.getint('URLLENGTH_LIMIT') + + def process_request(self, request, response, spider): + return self.process_start_request(self, request) + + def process_start_request(self, request, spider): + if len(request.url) <= self.maxlength: + return request + spider.log("Ignoring request (url length > %d): %s " % (self.maxlength, request.url)) +}}} + +=== Set Referer === + +A middleware to set the Referer: + +{{{ +#!python +class SetReferer(object): + + def process_request(self, request, response, spider): + request.headers.setdefault('Referer', response.url) + return request +}}} + +=== Set and limit crawling depth === + +A middleware to set (and limit) the request/response depth, taken from the start requests: + +{{{ +#!python +class SetLimitDepth(object): + + def __init__(self, maxdepth=0): + self.maxdepth = maxdepth or settings.getint('DEPTH_LIMIT') + + def process_request(self, request, response, spider): + depth = response.request.meta['depth'] + 1 + request.meta['depth'] = depth + if not self.maxdepth or depth <= self.maxdepth: + return request + spider.log("Ignoring link (depth > %d): %s " % (self.maxdepth, request) + + def process_start_request(self, request, spider): + request.meta['depth'] = 0 + return request +}}} + +=== Filter duplicate requests === + +A middleware to filter out requests already seen: + +{{{ +#!python +class FilterDuplicates(object): + + def __init__(self): + clspath = settings.get('DUPEFILTER_CLASS') + self.dupefilter = load_object(clspath)() + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + def enqueue_request(self, spider, request): + seen = self.dupefilter.request_seen(spider, request) + if not seen or request.dont_filter: + return request + + def spider_opened(self, spider): + self.dupefilter.open_spider(spider) + + def spider_closed(self, spider): + self.dupefilter.close_spider(spider) +}}} + +=== Scrape data using Parsley === + +A middleware to Scrape data using Parsley as described in UsingParsley + +{{{ +#!python +from pyparsley import PyParsley + +class ParsleyExtractor(object): + + def __init__(self, parslet_json_code): + parslet = json.loads(parselet_json_code) + class ParsleyItem(Item): + def __init__(self, *a, **kw): + for name in parslet.keys(): + self.fields[name] = Field() + super(ParsleyItem, self).__init__(*a, **kw) + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') + + def process_response(self, response, request, spider): + return self.item_class(self.parsly.parse(string=response.body)) +}}} + + + +== Pending issues == + +Resolved: + + * how to make `start_requests()` output pass through spider middleware `process_request()`? + * Start requests will be injected through `manager.scraper.process_request()` instead of `manager.engine.crawl()` + * should we support adding additional start requests from a spider middleware? + * Yes - there is a spider middleware method (`start_requests`) for that + * should `process_response()` receive a `request` argument with the `request` that originated it?. `response.request` is the latest request, not the original one (think of redirections), but it does carry the `meta` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be the deserialized request from the persistent scheduler queue. + * No - this would make implementation more complex and we're not sure it's really needed + * how to make sure `Request.errback` is always called if there is a problem with the request?. Do we need to ensure that?. Requests filtered out (by returning `None`) in the `process_request()` method will never be callback-ed or even errback-ed. this could be a problem for spiders that want to be notified if their requests are dropped. should we support this notification somehow or document (the lack of) it properly? + * We won't support notifications of dropped requests, because: 1. it's hard to implement and unreliable, 2. it's against not friendly with request persistence, 3. we can't come up with a good api. + * should we make the list of default spider middlewares empty? (or the "per-spider" spider middleware alone) + * No - there are some useful spider middlewares that it's worth enabling by default like referer, duplicates, robots2 + * should we allow returning deferreds in spider middleware methods? + * Yes - we should build a Deferred with the spider middleware methods as callbacks and that would implicitly support returning Deferreds + * should we support processing responses before they're processed by the spider, because `process_response` runs "in parallel" to the spider callback, and can't stop from running it. + * No - we haven't seen a practical use case for this, so we won't add an additional hook. It should be trivial to add it later, if needed. + * should we make a spider middleware to handle calling the request and spider callback, instead of letting the Scraper component do it? + * Yes - there's gonna a spider middleware for execution spider-specific code such as callbacks and also custom middlewares From 4de45c942e93845e2d6588c49a9431c5fd933909 Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Thu, 6 Mar 2014 12:41:38 -0300 Subject: [PATCH 3/3] Running lucasdemarchi/codespell to fix typos in code --- scrapy/contracts/default.py | 8 ++++---- scrapy/contrib/linkextractors/sgml.py | 2 +- scrapy/contrib/memusage.py | 2 +- scrapy/core/scraper.py | 2 +- scrapy/responsetypes.py | 16 ++++++++-------- scrapy/xlib/pydispatch/robustapply.py | 2 +- scrapy/xlib/tx/_newclient.py | 2 +- scrapy/xlib/tx/interfaces.py | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 73a1447f0..1d8367f82 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -57,12 +57,12 @@ class ReturnsContract(Contract): self.max_bound = float('inf') def post_process(self, output): - occurences = 0 + occurrences = 0 for x in output: if isinstance(x, self.obj_type): - occurences += 1 + occurrences += 1 - assertion = (self.min_bound <= occurences <= self.max_bound) + assertion = (self.min_bound <= occurrences <= self.max_bound) if not assertion: if self.min_bound == self.max_bound: @@ -71,7 +71,7 @@ class ReturnsContract(Contract): expected = '%s..%s' % (self.min_bound, self.max_bound) raise ContractFail("Returned %s %s, expected %s" % \ - (occurences, self.obj_name, expected)) + (occurrences, self.obj_name, expected)) class ScrapesContract(Contract): diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 72ebc743a..446769324 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -45,7 +45,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): def _process_links(self, links): """ Normalize and filter extracted links - The subclass should override it if neccessary + The subclass should override it if necessary """ links = unique_list(links, key=lambda link: link.url) if self.unique else links return links diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 6608632f0..6bcba8e11 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -21,7 +21,7 @@ class MemoryUsage(object): if not crawler.settings.getbool('MEMUSAGE_ENABLED'): raise NotConfigured try: - # stdlib's resource module is only availabe on unix platforms. + # stdlib's resource module is only available on unix platforms. self.resource = import_module('resource') except ImportError: raise NotConfigured diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index f0dcc7522..bc9979928 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -121,7 +121,7 @@ class Scraper(object): return dfd def _scrape2(self, request_result, request, spider): - """Handle the diferent cases of request's result been a Response or a + """Handle the different cases of request's result been a Response or a Failure""" if not isinstance(request_result, Failure): return self.spidermw.scrape_response(self.call_spider, \ diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index efb11a267..53f0ae426 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -1,6 +1,6 @@ """ -This module implements a class which returns the appropiate Response class -based on different criterias. +This module implements a class which returns the appropriate Response class +based on different criteria. """ @@ -38,7 +38,7 @@ class ResponseTypes(object): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): - """Return the most appropiate Response class for the given mimetype""" + """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response elif mimetype in self.classes: @@ -48,7 +48,7 @@ class ResponseTypes(object): return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): - """Return the most appropiate Response class from an HTTP Content-Type + """Return the most appropriate Response class from an HTTP Content-Type header """ if content_encoding: return Response @@ -64,7 +64,7 @@ class ResponseTypes(object): return Response def from_headers(self, headers): - """Return the most appropiate Response class by looking at the HTTP + """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response if 'Content-Type' in headers: @@ -75,7 +75,7 @@ class ResponseTypes(object): return cls def from_filename(self, filename): - """Return the most appropiate Response class from a file name""" + """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) @@ -83,7 +83,7 @@ class ResponseTypes(object): return Response def from_body(self, body): - """Try to guess the appropiate response based on the body content. + """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types cannot be guess using more straightforward methods.""" @@ -98,7 +98,7 @@ class ResponseTypes(object): return self.from_mimetype('text') def from_args(self, headers=None, url=None, filename=None, body=None): - """Guess the most appropiate Response class based on the given arguments""" + """Guess the most appropriate Response class based on the given arguments""" cls = Response if headers is not None: cls = self.from_headers(headers) diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py index 11d25d377..cfd41425c 100644 --- a/scrapy/xlib/pydispatch/robustapply.py +++ b/scrapy/xlib/pydispatch/robustapply.py @@ -27,7 +27,7 @@ def function(receiver): # an instance-method... return receiver, receiver.im_func.func_code, 1 elif not hasattr(receiver, 'func_code'): - raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver))) + raise ValueError('unknown receiver type %s %s'%(receiver, type(receiver))) return receiver, receiver.func_code, 0 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index e088a12d1..16d0ca6b4 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -581,7 +581,7 @@ class Request: # In the future, having the protocol version be a parameter to this # method would probably be good. It would be nice if this method - # weren't limited to issueing HTTP/1.1 requests. + # weren't limited to issuing HTTP/1.1 requests. requestLines = [] requestLines.append( '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index 7e294c450..f3e4ed5d8 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -849,7 +849,7 @@ class IReactorMulticast(Interface): UDP socket methods that support multicast. IMPORTANT: This is an experimental new interface. It may change - without backwards compatability. Suggestions are welcome. + without backwards compatibility. Suggestions are welcome. """ def listenMulticast(port, protocol, interface='', maxPacketSize=8192,