mirror of https://github.com/scrapy/scrapy.git
Running lucasdemarchi/codespell to fix typos in SEPs
This commit is contained in:
Capi Etheriel
parent
72b6c96d9a
commit
20a8237910
306
sep/sep-003.trac
306
sep/sep-003.trac
|
|
@ -1,153 +1,153 @@
|
|||
= SEP-003 - Nested items API (!ItemField) =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||3||
|
||||
||'''Title:'''||Nested items API (!ItemField) ||
|
||||
||'''Author:'''||Pablo Hoffman||
|
||||
||'''Created:'''||2009-07-19||
|
||||
||'''Status'''||Obsoleted by [wiki:SEP-008]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This page presents different usage scenarios for the new nested items field API called !ItemField.
|
||||
|
||||
== Prerequisites ==
|
||||
|
||||
This API proposal relies on the following API:
|
||||
|
||||
1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance)
|
||||
2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}}
|
||||
|
||||
== Proposed Implementation of !ItemField ==
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.fields import BaseField
|
||||
|
||||
class ItemField(BaseField):
|
||||
def __init__(self, item_type, default=None):
|
||||
self._item_type = item_type
|
||||
super(ItemField, self).__init__(default)
|
||||
|
||||
def to_python(self, value):
|
||||
return self._item_type(value) if not isinstance(value, self._item_type) else value
|
||||
|
||||
def get_default(self):
|
||||
# WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors
|
||||
# always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour
|
||||
return self._default
|
||||
}}}
|
||||
|
||||
== Usage Scenarios ==
|
||||
|
||||
=== Defining an item containing !ItemField's ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.models import Item
|
||||
from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField
|
||||
|
||||
class Supplier(Item):
|
||||
name = TextField(default="anonymous supplier")
|
||||
url = UrlField()
|
||||
|
||||
class Variant(Item):
|
||||
name = TextField(required=True)
|
||||
url = UrlField()
|
||||
price = DecimalField()
|
||||
|
||||
class Product(Variant):
|
||||
supplier = ItemField(Supplier, default=Supplier(name="default supplier")
|
||||
variants = ListField(ItemField(Variant))
|
||||
|
||||
# these ones are used for documenting default value examples
|
||||
supplier2 = ItemField(Supplier)
|
||||
variants2 = ListField(ItemField(Variant), default=[])
|
||||
}}}
|
||||
|
||||
It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant
|
||||
relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Product(Item):
|
||||
variants = ItemField(Product) # Fails to compile
|
||||
}}}
|
||||
|
||||
=== Assigning an item field ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
supplier = Supplier(name="Supplier 1", url="http://example.com")
|
||||
|
||||
p = Product()
|
||||
|
||||
# standard assignment
|
||||
p['supplier'] = supplier
|
||||
# this also works as it tries to instantiate a Supplier with the given dict
|
||||
p['supplier'] = {'name': 'Supplier 1' url='http://example.com'}
|
||||
# this fails because it can't instantiate a Supplier
|
||||
p['supplier'] = 'Supplier 1'
|
||||
# this fails because url doesn't have the valid type
|
||||
p['supplier'] = {'name': 'Supplier 1' url=123}
|
||||
|
||||
v1 = Variant()
|
||||
v1['name'] = "lala"
|
||||
v1['price'] = Decimal("100")
|
||||
|
||||
v2 = Variant()
|
||||
v2['name'] = "lolo"
|
||||
v2['price'] = Decimal("150")
|
||||
|
||||
# standard assignment
|
||||
p['variants'] = [v1, v2] # OK
|
||||
# can also instantiate at assignment time
|
||||
p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")]
|
||||
# this also works as it tries to instantiate a Variant with the given dict
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")]
|
||||
# this fails because it can't instantiate a Variant
|
||||
p['variants'] = [v1, 'test']
|
||||
# this fails beacuse 'coco' is not a valid value for price
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': 'coco']
|
||||
}}}
|
||||
|
||||
=== Default values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
p = Product()
|
||||
|
||||
p['supplier'] # returns: Supplier(name='default supplier')
|
||||
p['supplier2'] # raises KeyError
|
||||
p['supplier2'] = Supplier()
|
||||
p['supplier2'] # returns: Supplier(name='anonymous supplier')
|
||||
|
||||
p['variants'] # raises KeyError
|
||||
p['variants2'] # returns []
|
||||
|
||||
p['categories'] # raises KeyError
|
||||
p.get('categories') # returns None
|
||||
|
||||
p['numbers'] # returns []
|
||||
}}}
|
||||
|
||||
=== Accesing and changing nested item values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
p = Product(supplier=Supplier(name="some name", url="http://example.com"))
|
||||
p['supplier']['url'] # returns 'http://example.com'
|
||||
p['supplier']['url'] = "http://www.other.com" # works as expected
|
||||
p['supplier']['url'] = 123 # fails: wrong type for supplier url
|
||||
|
||||
p['variants'] = [v1, v2]
|
||||
p['variants'][0]['name'] # returns v1 name
|
||||
p['variants'][1]['name'] # returns v2 name
|
||||
|
||||
# XXX: decide what to do about these cases:
|
||||
p['variants'].append(v3) # works but doesn't check type of v3
|
||||
p['variants'].append(1) # works but shouldn't?
|
||||
}}}
|
||||
= SEP-003 - Nested items API (!ItemField) =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||3||
|
||||
||'''Title:'''||Nested items API (!ItemField) ||
|
||||
||'''Author:'''||Pablo Hoffman||
|
||||
||'''Created:'''||2009-07-19||
|
||||
||'''Status'''||Obsoleted by [wiki:SEP-008]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This page presents different usage scenarios for the new nested items field API called !ItemField.
|
||||
|
||||
== Prerequisites ==
|
||||
|
||||
This API proposal relies on the following API:
|
||||
|
||||
1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance)
|
||||
2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}}
|
||||
|
||||
== Proposed Implementation of !ItemField ==
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.fields import BaseField
|
||||
|
||||
class ItemField(BaseField):
|
||||
def __init__(self, item_type, default=None):
|
||||
self._item_type = item_type
|
||||
super(ItemField, self).__init__(default)
|
||||
|
||||
def to_python(self, value):
|
||||
return self._item_type(value) if not isinstance(value, self._item_type) else value
|
||||
|
||||
def get_default(self):
|
||||
# WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors
|
||||
# always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour
|
||||
return self._default
|
||||
}}}
|
||||
|
||||
== Usage Scenarios ==
|
||||
|
||||
=== Defining an item containing !ItemField's ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.models import Item
|
||||
from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField
|
||||
|
||||
class Supplier(Item):
|
||||
name = TextField(default="anonymous supplier")
|
||||
url = UrlField()
|
||||
|
||||
class Variant(Item):
|
||||
name = TextField(required=True)
|
||||
url = UrlField()
|
||||
price = DecimalField()
|
||||
|
||||
class Product(Variant):
|
||||
supplier = ItemField(Supplier, default=Supplier(name="default supplier")
|
||||
variants = ListField(ItemField(Variant))
|
||||
|
||||
# these ones are used for documenting default value examples
|
||||
supplier2 = ItemField(Supplier)
|
||||
variants2 = ListField(ItemField(Variant), default=[])
|
||||
}}}
|
||||
|
||||
It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant
|
||||
relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Product(Item):
|
||||
variants = ItemField(Product) # Fails to compile
|
||||
}}}
|
||||
|
||||
=== Assigning an item field ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
supplier = Supplier(name="Supplier 1", url="http://example.com")
|
||||
|
||||
p = Product()
|
||||
|
||||
# standard assignment
|
||||
p['supplier'] = supplier
|
||||
# this also works as it tries to instantiate a Supplier with the given dict
|
||||
p['supplier'] = {'name': 'Supplier 1' url='http://example.com'}
|
||||
# this fails because it can't instantiate a Supplier
|
||||
p['supplier'] = 'Supplier 1'
|
||||
# this fails because url doesn't have the valid type
|
||||
p['supplier'] = {'name': 'Supplier 1' url=123}
|
||||
|
||||
v1 = Variant()
|
||||
v1['name'] = "lala"
|
||||
v1['price'] = Decimal("100")
|
||||
|
||||
v2 = Variant()
|
||||
v2['name'] = "lolo"
|
||||
v2['price'] = Decimal("150")
|
||||
|
||||
# standard assignment
|
||||
p['variants'] = [v1, v2] # OK
|
||||
# can also instantiate at assignment time
|
||||
p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")]
|
||||
# this also works as it tries to instantiate a Variant with the given dict
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")]
|
||||
# this fails because it can't instantiate a Variant
|
||||
p['variants'] = [v1, 'test']
|
||||
# this fails because 'coco' is not a valid value for price
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': 'coco']
|
||||
}}}
|
||||
|
||||
=== Default values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
p = Product()
|
||||
|
||||
p['supplier'] # returns: Supplier(name='default supplier')
|
||||
p['supplier2'] # raises KeyError
|
||||
p['supplier2'] = Supplier()
|
||||
p['supplier2'] # returns: Supplier(name='anonymous supplier')
|
||||
|
||||
p['variants'] # raises KeyError
|
||||
p['variants2'] # returns []
|
||||
|
||||
p['categories'] # raises KeyError
|
||||
p.get('categories') # returns None
|
||||
|
||||
p['numbers'] # returns []
|
||||
}}}
|
||||
|
||||
=== Accesing and changing nested item values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
p = Product(supplier=Supplier(name="some name", url="http://example.com"))
|
||||
p['supplier']['url'] # returns 'http://example.com'
|
||||
p['supplier']['url'] = "http://www.other.com" # works as expected
|
||||
p['supplier']['url'] = 123 # fails: wrong type for supplier url
|
||||
|
||||
p['variants'] = [v1, v2]
|
||||
p['variants'][0]['name'] # returns v1 name
|
||||
p['variants'][1]['name'] # returns v2 name
|
||||
|
||||
# XXX: decide what to do about these cases:
|
||||
p['variants'].append(v3) # works but doesn't check type of v3
|
||||
p['variants'].append(1) # works but shouldn't?
|
||||
}}}
|
||||
|
|
|
|||
1224
sep/sep-014.trac
1224
sep/sep-014.trac
File diff suppressed because it is too large
Load Diff
528
sep/sep-016.trac
528
sep/sep-016.trac
|
|
@ -1,265 +1,265 @@
|
|||
= SEP-016: Leg Spider =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||16||
|
||||
||'''Title:'''||Leg Spider||
|
||||
||'''Author:'''||Insophia Team||
|
||||
||'''Created:'''||2010-06-03||
|
||||
||'''Status'''||Superseded by [wiki:SEP-018]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders.
|
||||
|
||||
== Rationale ==
|
||||
|
||||
The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy.
|
||||
|
||||
The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider".
|
||||
|
||||
== {{{LegSpider}}} API ==
|
||||
|
||||
A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods:
|
||||
|
||||
* {{{legs}}}
|
||||
* legs composing this spider
|
||||
* {{{process_response(response)}}}
|
||||
* Process a (downloaded) response and return a list of requests and items
|
||||
* {{{process_request(request)}}}
|
||||
* Process a request after it has been extracted and before returning it from the spider
|
||||
* {{{process_item(item)}}}
|
||||
* Process an item after it has been extracted and before returning it from the spider
|
||||
* {{{set_spider()}}}
|
||||
* Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior.
|
||||
|
||||
== How Leg Spiders work ==
|
||||
|
||||
1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output.
|
||||
2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself.
|
||||
|
||||
== Leg Spider examples ==
|
||||
|
||||
=== Regex (HTML) Link Extractor ===
|
||||
|
||||
A typical application of LegSpider's is to build Link Extractors. For example:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class RegexHtmlLinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if isinstance(response, HtmlResponse):
|
||||
allowed_regexes = self.spider.url_regexes_to_follow
|
||||
# extract urls to follow using allowed_regexes
|
||||
return [Request(x) for x in urls_to_follow]
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexHtmlLinkExtractor()]
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
def parse_response(self, response):
|
||||
# parse response and extract items
|
||||
return items
|
||||
}}}
|
||||
|
||||
=== RSS2 link extractor ===
|
||||
|
||||
This is a Leg Spider that can be used for following links from RSS2 feeds.
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Rss2LinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if response.headers.get('Content-type') == 'application/rss+xml':
|
||||
xs = XmlXPathSelector(response)
|
||||
urls = xs.select("//item/link/text()").extract()
|
||||
return [Request(x) for x in urls]
|
||||
}}}
|
||||
|
||||
=== Callback dispatcher based on rules ===
|
||||
|
||||
Another example could be to build a callback dispatcher based on rules:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CallbackRules(LegSpider):
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(CallbackRules, self).__init__(*a, **kw)
|
||||
for regex, method_name in self.spider.callback_rules.items():
|
||||
r = re.compile(regex)
|
||||
m = getattr(self.spider, method_name, None)
|
||||
if m:
|
||||
self._rules[r] = m
|
||||
|
||||
def process_response(self, response):
|
||||
for regex, method in self._rules.items():
|
||||
m = regex.search(response.url)
|
||||
if m:
|
||||
return method(response)
|
||||
return []
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CallbackRules()]
|
||||
callback_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
def parse_product(self, response):
|
||||
# parse reponse and populate item
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== URL Canonicalizers ===
|
||||
|
||||
Another example could be for building URL canonicalizers:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CanonializeUrl(LegSpider):
|
||||
|
||||
def process_request(self, request):
|
||||
curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
|
||||
return request.replace(url=curl)
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CanonicalizeUrl()]
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
# ...
|
||||
}}}
|
||||
|
||||
=== Setting item identifier ===
|
||||
|
||||
Another example could be for setting a unique identifier to items, based on certain fields:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class ItemIdSetter(LegSpider):
|
||||
|
||||
def process_item(self, item):
|
||||
id_field = self.spider.id_field
|
||||
id_fields_to_hash = self.spider.id_fields_to_hash
|
||||
item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash)
|
||||
return item
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [ItemIdSetter()]
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_response(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== Combining multiple leg spiders ===
|
||||
|
||||
Here's an example that combines functionality from multiple leg spiders:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()]
|
||||
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
parse_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_product(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
|
||||
def process_category(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
|
||||
|
||||
== Leg Spiders vs Spider middlewares ==
|
||||
|
||||
A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality.
|
||||
|
||||
== When not to use Leg Spiders ==
|
||||
|
||||
Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as:
|
||||
|
||||
* Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware.
|
||||
* Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit.
|
||||
|
||||
== {{{LegSpider}}} proof-of-concept implementation ==
|
||||
|
||||
Here's a proof-of-concept implementation of {{{LegSpider}}}:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
class LegSpider(BaseSpider):
|
||||
"""A spider made of legs"""
|
||||
|
||||
legs = []
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(LegSpider, self).__init__(*args, **kwargs)
|
||||
self._legs = [self] + self.legs[:]
|
||||
for l in self._legs:
|
||||
l.set_spider(self)
|
||||
|
||||
def parse(self, response):
|
||||
res = self._process_response(response)
|
||||
for r in res:
|
||||
if isinstance(r, BaseItem):
|
||||
yield self._process_item(r)
|
||||
else:
|
||||
yield self._process_request(r)
|
||||
|
||||
def process_response(self, response):
|
||||
return []
|
||||
|
||||
def process_request(self, request):
|
||||
return request
|
||||
|
||||
def process_item(self, item):
|
||||
return item
|
||||
|
||||
def set_spider(self, spider):
|
||||
self.spider = spider
|
||||
|
||||
def _process_response(self, response):
|
||||
res = []
|
||||
for l in self._legs:
|
||||
res.extend(iterate_spider_output(l.process_response(response)))
|
||||
return res
|
||||
|
||||
def _process_request(self, request):
|
||||
for l in self._legs:
|
||||
request = l.process_request(request)
|
||||
return request
|
||||
|
||||
def _process_item(self, item):
|
||||
for l in self._legs:
|
||||
item = l.process_item(item)
|
||||
return item
|
||||
= SEP-016: Leg Spider =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||16||
|
||||
||'''Title:'''||Leg Spider||
|
||||
||'''Author:'''||Insophia Team||
|
||||
||'''Created:'''||2010-06-03||
|
||||
||'''Status'''||Superseded by [wiki:SEP-018]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders.
|
||||
|
||||
== Rationale ==
|
||||
|
||||
The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy.
|
||||
|
||||
The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider".
|
||||
|
||||
== {{{LegSpider}}} API ==
|
||||
|
||||
A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods:
|
||||
|
||||
* {{{legs}}}
|
||||
* legs composing this spider
|
||||
* {{{process_response(response)}}}
|
||||
* Process a (downloaded) response and return a list of requests and items
|
||||
* {{{process_request(request)}}}
|
||||
* Process a request after it has been extracted and before returning it from the spider
|
||||
* {{{process_item(item)}}}
|
||||
* Process an item after it has been extracted and before returning it from the spider
|
||||
* {{{set_spider()}}}
|
||||
* Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior.
|
||||
|
||||
== How Leg Spiders work ==
|
||||
|
||||
1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output.
|
||||
2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself.
|
||||
|
||||
== Leg Spider examples ==
|
||||
|
||||
=== Regex (HTML) Link Extractor ===
|
||||
|
||||
A typical application of LegSpider's is to build Link Extractors. For example:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class RegexHtmlLinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if isinstance(response, HtmlResponse):
|
||||
allowed_regexes = self.spider.url_regexes_to_follow
|
||||
# extract urls to follow using allowed_regexes
|
||||
return [Request(x) for x in urls_to_follow]
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexHtmlLinkExtractor()]
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
def parse_response(self, response):
|
||||
# parse response and extract items
|
||||
return items
|
||||
}}}
|
||||
|
||||
=== RSS2 link extractor ===
|
||||
|
||||
This is a Leg Spider that can be used for following links from RSS2 feeds.
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Rss2LinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if response.headers.get('Content-type') == 'application/rss+xml':
|
||||
xs = XmlXPathSelector(response)
|
||||
urls = xs.select("//item/link/text()").extract()
|
||||
return [Request(x) for x in urls]
|
||||
}}}
|
||||
|
||||
=== Callback dispatcher based on rules ===
|
||||
|
||||
Another example could be to build a callback dispatcher based on rules:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CallbackRules(LegSpider):
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(CallbackRules, self).__init__(*a, **kw)
|
||||
for regex, method_name in self.spider.callback_rules.items():
|
||||
r = re.compile(regex)
|
||||
m = getattr(self.spider, method_name, None)
|
||||
if m:
|
||||
self._rules[r] = m
|
||||
|
||||
def process_response(self, response):
|
||||
for regex, method in self._rules.items():
|
||||
m = regex.search(response.url)
|
||||
if m:
|
||||
return method(response)
|
||||
return []
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CallbackRules()]
|
||||
callback_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
def parse_product(self, response):
|
||||
# parse response and populate item
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== URL Canonicalizers ===
|
||||
|
||||
Another example could be for building URL canonicalizers:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CanonializeUrl(LegSpider):
|
||||
|
||||
def process_request(self, request):
|
||||
curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
|
||||
return request.replace(url=curl)
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CanonicalizeUrl()]
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
# ...
|
||||
}}}
|
||||
|
||||
=== Setting item identifier ===
|
||||
|
||||
Another example could be for setting a unique identifier to items, based on certain fields:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class ItemIdSetter(LegSpider):
|
||||
|
||||
def process_item(self, item):
|
||||
id_field = self.spider.id_field
|
||||
id_fields_to_hash = self.spider.id_fields_to_hash
|
||||
item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash)
|
||||
return item
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [ItemIdSetter()]
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_response(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== Combining multiple leg spiders ===
|
||||
|
||||
Here's an example that combines functionality from multiple leg spiders:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()]
|
||||
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
parse_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_product(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
|
||||
def process_category(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
|
||||
|
||||
== Leg Spiders vs Spider middlewares ==
|
||||
|
||||
A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality.
|
||||
|
||||
== When not to use Leg Spiders ==
|
||||
|
||||
Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as:
|
||||
|
||||
* Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware.
|
||||
* Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit.
|
||||
|
||||
== {{{LegSpider}}} proof-of-concept implementation ==
|
||||
|
||||
Here's a proof-of-concept implementation of {{{LegSpider}}}:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
class LegSpider(BaseSpider):
|
||||
"""A spider made of legs"""
|
||||
|
||||
legs = []
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(LegSpider, self).__init__(*args, **kwargs)
|
||||
self._legs = [self] + self.legs[:]
|
||||
for l in self._legs:
|
||||
l.set_spider(self)
|
||||
|
||||
def parse(self, response):
|
||||
res = self._process_response(response)
|
||||
for r in res:
|
||||
if isinstance(r, BaseItem):
|
||||
yield self._process_item(r)
|
||||
else:
|
||||
yield self._process_request(r)
|
||||
|
||||
def process_response(self, response):
|
||||
return []
|
||||
|
||||
def process_request(self, request):
|
||||
return request
|
||||
|
||||
def process_item(self, item):
|
||||
return item
|
||||
|
||||
def set_spider(self, spider):
|
||||
self.spider = spider
|
||||
|
||||
def _process_response(self, response):
|
||||
res = []
|
||||
for l in self._legs:
|
||||
res.extend(iterate_spider_output(l.process_response(response)))
|
||||
return res
|
||||
|
||||
def _process_request(self, request):
|
||||
for l in self._legs:
|
||||
request = l.process_request(request)
|
||||
return request
|
||||
|
||||
def _process_item(self, item):
|
||||
for l in self._legs:
|
||||
item = l.process_item(item)
|
||||
return item
|
||||
}}}
|
||||
1102
sep/sep-018.trac
1102
sep/sep-018.trac
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue