mirror of https://github.com/scrapy/scrapy.git
Merge pull request #628 from barraponto/codespell
Using lucasdemarchi/codespell to fix typos
This commit is contained in:
Mikhail Korobov
commit
c0a72e9c24
|
|
@ -172,7 +172,7 @@ the :ref:`topics-signals-ref` to know which ones.
|
|||
What does the response status code 999 means?
|
||||
---------------------------------------------
|
||||
|
||||
999 is a custom reponse status code used by Yahoo sites to throttle requests.
|
||||
999 is a custom response status code used by Yahoo sites to throttle requests.
|
||||
Try slowing down the crawling speed by using a download delay of ``2`` (or
|
||||
higher) in your spider::
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ Enhancements
|
|||
- Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`)
|
||||
- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`)
|
||||
- Support defining http error handling using settings (:issue:`466`)
|
||||
- Use moderm python idioms wherever you find legacies (:issue:`497`)
|
||||
- Use modern python idioms wherever you find legacies (:issue:`497`)
|
||||
- Improve and correct documentation
|
||||
(:issue:`527`, :issue:`524`, :issue:`521`, :issue:`517`, :issue:`512`, :issue:`505`,
|
||||
:issue:`502`, :issue:`489`, :issue:`465`, :issue:`460`, :issue:`425`, :issue:`536`)
|
||||
|
|
@ -118,7 +118,7 @@ Enhancements
|
|||
- Remove multi spider support from multiple core components
|
||||
(:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
|
||||
- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages.
|
||||
- Add pypy 2.1 to continous integration tests (:commit:`ecfa7431`)
|
||||
- Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`)
|
||||
- Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`)
|
||||
- Use importlib for parametric imports (:issue:`445`)
|
||||
- Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`)
|
||||
|
|
@ -188,7 +188,7 @@ List of contributors sorted by number of commits::
|
|||
|
||||
- IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`)
|
||||
- Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`)
|
||||
- Fix start_requests lazyness and early hangs (:commit:`89faf52`)
|
||||
- Fix start_requests laziness and early hangs (:commit:`89faf52`)
|
||||
|
||||
0.18.3 (released 2013-10-03)
|
||||
----------------------------
|
||||
|
|
@ -330,7 +330,7 @@ contributors sorted by number of commits::
|
|||
|
||||
- fixes spelling errors in documentation (:commit:`6d2b3aa`)
|
||||
- add doc about disabling an extension. refs #132 (:commit:`c90de33`)
|
||||
- Fixed error message formatting. log.err() doesn't support cool formatting and when error occured, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`)
|
||||
- Fixed error message formatting. log.err() doesn't support cool formatting and when error occurred, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`)
|
||||
- lint and improve images pipeline error logging (:commit:`56b45fc`)
|
||||
- fixed doc typos (:commit:`243be84`)
|
||||
- add documentation topics: Broad Crawls & Common Practies (:commit:`1fbb715`)
|
||||
|
|
|
|||
|
|
@ -411,7 +411,7 @@ Response objects
|
|||
|
||||
.. attribute:: Response.body
|
||||
|
||||
A str containing the body of this Response. Keep in mind that Reponse.body
|
||||
A str containing the body of this Response. Keep in mind that Response.body
|
||||
is always a str. If you want the unicode version use
|
||||
:meth:`TextResponse.body_as_unicode` (only available in
|
||||
:class:`TextResponse` and subclasses).
|
||||
|
|
|
|||
|
|
@ -57,12 +57,12 @@ class ReturnsContract(Contract):
|
|||
self.max_bound = float('inf')
|
||||
|
||||
def post_process(self, output):
|
||||
occurences = 0
|
||||
occurrences = 0
|
||||
for x in output:
|
||||
if isinstance(x, self.obj_type):
|
||||
occurences += 1
|
||||
occurrences += 1
|
||||
|
||||
assertion = (self.min_bound <= occurences <= self.max_bound)
|
||||
assertion = (self.min_bound <= occurrences <= self.max_bound)
|
||||
|
||||
if not assertion:
|
||||
if self.min_bound == self.max_bound:
|
||||
|
|
@ -71,7 +71,7 @@ class ReturnsContract(Contract):
|
|||
expected = '%s..%s' % (self.min_bound, self.max_bound)
|
||||
|
||||
raise ContractFail("Returned %s %s, expected %s" % \
|
||||
(occurences, self.obj_name, expected))
|
||||
(occurrences, self.obj_name, expected))
|
||||
|
||||
|
||||
class ScrapesContract(Contract):
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
|
|||
def _process_links(self, links):
|
||||
""" Normalize and filter extracted links
|
||||
|
||||
The subclass should override it if neccessary
|
||||
The subclass should override it if necessary
|
||||
"""
|
||||
links = unique_list(links, key=lambda link: link.url) if self.unique else links
|
||||
return links
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ class MemoryUsage(object):
|
|||
if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
|
||||
raise NotConfigured
|
||||
try:
|
||||
# stdlib's resource module is only availabe on unix platforms.
|
||||
# stdlib's resource module is only available on unix platforms.
|
||||
self.resource = import_module('resource')
|
||||
except ImportError:
|
||||
raise NotConfigured
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ class Scraper(object):
|
|||
return dfd
|
||||
|
||||
def _scrape2(self, request_result, request, spider):
|
||||
"""Handle the diferent cases of request's result been a Response or a
|
||||
"""Handle the different cases of request's result been a Response or a
|
||||
Failure"""
|
||||
if not isinstance(request_result, Failure):
|
||||
return self.spidermw.scrape_response(self.call_spider, \
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""
|
||||
This module implements a class which returns the appropiate Response class
|
||||
based on different criterias.
|
||||
This module implements a class which returns the appropriate Response class
|
||||
based on different criteria.
|
||||
|
||||
"""
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ class ResponseTypes(object):
|
|||
self.classes[mimetype] = load_object(cls)
|
||||
|
||||
def from_mimetype(self, mimetype):
|
||||
"""Return the most appropiate Response class for the given mimetype"""
|
||||
"""Return the most appropriate Response class for the given mimetype"""
|
||||
if mimetype is None:
|
||||
return Response
|
||||
elif mimetype in self.classes:
|
||||
|
|
@ -48,7 +48,7 @@ class ResponseTypes(object):
|
|||
return self.classes.get(basetype, Response)
|
||||
|
||||
def from_content_type(self, content_type, content_encoding=None):
|
||||
"""Return the most appropiate Response class from an HTTP Content-Type
|
||||
"""Return the most appropriate Response class from an HTTP Content-Type
|
||||
header """
|
||||
if content_encoding:
|
||||
return Response
|
||||
|
|
@ -64,7 +64,7 @@ class ResponseTypes(object):
|
|||
return Response
|
||||
|
||||
def from_headers(self, headers):
|
||||
"""Return the most appropiate Response class by looking at the HTTP
|
||||
"""Return the most appropriate Response class by looking at the HTTP
|
||||
headers"""
|
||||
cls = Response
|
||||
if 'Content-Type' in headers:
|
||||
|
|
@ -75,7 +75,7 @@ class ResponseTypes(object):
|
|||
return cls
|
||||
|
||||
def from_filename(self, filename):
|
||||
"""Return the most appropiate Response class from a file name"""
|
||||
"""Return the most appropriate Response class from a file name"""
|
||||
mimetype, encoding = self.mimetypes.guess_type(filename)
|
||||
if mimetype and not encoding:
|
||||
return self.from_mimetype(mimetype)
|
||||
|
|
@ -83,7 +83,7 @@ class ResponseTypes(object):
|
|||
return Response
|
||||
|
||||
def from_body(self, body):
|
||||
"""Try to guess the appropiate response based on the body content.
|
||||
"""Try to guess the appropriate response based on the body content.
|
||||
This method is a bit magic and could be improved in the future, but
|
||||
it's not meant to be used except for special cases where response types
|
||||
cannot be guess using more straightforward methods."""
|
||||
|
|
@ -98,7 +98,7 @@ class ResponseTypes(object):
|
|||
return self.from_mimetype('text')
|
||||
|
||||
def from_args(self, headers=None, url=None, filename=None, body=None):
|
||||
"""Guess the most appropiate Response class based on the given arguments"""
|
||||
"""Guess the most appropriate Response class based on the given arguments"""
|
||||
cls = Response
|
||||
if headers is not None:
|
||||
cls = self.from_headers(headers)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ def function(receiver):
|
|||
# an instance-method...
|
||||
return receiver, receiver.im_func.func_code, 1
|
||||
elif not hasattr(receiver, 'func_code'):
|
||||
raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver)))
|
||||
raise ValueError('unknown receiver type %s %s'%(receiver, type(receiver)))
|
||||
|
||||
return receiver, receiver.func_code, 0
|
||||
|
||||
|
|
|
|||
|
|
@ -581,7 +581,7 @@ class Request:
|
|||
|
||||
# In the future, having the protocol version be a parameter to this
|
||||
# method would probably be good. It would be nice if this method
|
||||
# weren't limited to issueing HTTP/1.1 requests.
|
||||
# weren't limited to issuing HTTP/1.1 requests.
|
||||
requestLines = []
|
||||
requestLines.append(
|
||||
'%s %s HTTP/1.1\r\n' % (self.method, self.uri))
|
||||
|
|
|
|||
|
|
@ -849,7 +849,7 @@ class IReactorMulticast(Interface):
|
|||
UDP socket methods that support multicast.
|
||||
|
||||
IMPORTANT: This is an experimental new interface. It may change
|
||||
without backwards compatability. Suggestions are welcome.
|
||||
without backwards compatibility. Suggestions are welcome.
|
||||
"""
|
||||
|
||||
def listenMulticast(port, protocol, interface='', maxPacketSize=8192,
|
||||
|
|
|
|||
306
sep/sep-003.trac
306
sep/sep-003.trac
|
|
@ -1,153 +1,153 @@
|
|||
= SEP-003 - Nested items API (!ItemField) =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||3||
|
||||
||'''Title:'''||Nested items API (!ItemField) ||
|
||||
||'''Author:'''||Pablo Hoffman||
|
||||
||'''Created:'''||2009-07-19||
|
||||
||'''Status'''||Obsoleted by [wiki:SEP-008]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This page presents different usage scenarios for the new nested items field API called !ItemField.
|
||||
|
||||
== Prerequisites ==
|
||||
|
||||
This API proposal relies on the following API:
|
||||
|
||||
1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance)
|
||||
2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}}
|
||||
|
||||
== Proposed Implementation of !ItemField ==
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.fields import BaseField
|
||||
|
||||
class ItemField(BaseField):
|
||||
def __init__(self, item_type, default=None):
|
||||
self._item_type = item_type
|
||||
super(ItemField, self).__init__(default)
|
||||
|
||||
def to_python(self, value):
|
||||
return self._item_type(value) if not isinstance(value, self._item_type) else value
|
||||
|
||||
def get_default(self):
|
||||
# WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors
|
||||
# always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour
|
||||
return self._default
|
||||
}}}
|
||||
|
||||
== Usage Scenarios ==
|
||||
|
||||
=== Defining an item containing !ItemField's ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.models import Item
|
||||
from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField
|
||||
|
||||
class Supplier(Item):
|
||||
name = TextField(default="anonymous supplier")
|
||||
url = UrlField()
|
||||
|
||||
class Variant(Item):
|
||||
name = TextField(required=True)
|
||||
url = UrlField()
|
||||
price = DecimalField()
|
||||
|
||||
class Product(Variant):
|
||||
supplier = ItemField(Supplier, default=Supplier(name="default supplier")
|
||||
variants = ListField(ItemField(Variant))
|
||||
|
||||
# these ones are used for documenting default value examples
|
||||
supplier2 = ItemField(Supplier)
|
||||
variants2 = ListField(ItemField(Variant), default=[])
|
||||
}}}
|
||||
|
||||
It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant
|
||||
relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Product(Item):
|
||||
variants = ItemField(Product) # Fails to compile
|
||||
}}}
|
||||
|
||||
=== Assigning an item field ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
supplier = Supplier(name="Supplier 1", url="http://example.com")
|
||||
|
||||
p = Product()
|
||||
|
||||
# standard assignment
|
||||
p['supplier'] = supplier
|
||||
# this also works as it tries to instantiate a Supplier with the given dict
|
||||
p['supplier'] = {'name': 'Supplier 1' url='http://example.com'}
|
||||
# this fails because it can't instantiate a Supplier
|
||||
p['supplier'] = 'Supplier 1'
|
||||
# this fails because url doesn't have the valid type
|
||||
p['supplier'] = {'name': 'Supplier 1' url=123}
|
||||
|
||||
v1 = Variant()
|
||||
v1['name'] = "lala"
|
||||
v1['price'] = Decimal("100")
|
||||
|
||||
v2 = Variant()
|
||||
v2['name'] = "lolo"
|
||||
v2['price'] = Decimal("150")
|
||||
|
||||
# standard assignment
|
||||
p['variants'] = [v1, v2] # OK
|
||||
# can also instantiate at assignment time
|
||||
p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")]
|
||||
# this also works as it tries to instantiate a Variant with the given dict
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")]
|
||||
# this fails because it can't instantiate a Variant
|
||||
p['variants'] = [v1, 'test']
|
||||
# this fails beacuse 'coco' is not a valid value for price
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': 'coco']
|
||||
}}}
|
||||
|
||||
=== Default values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
p = Product()
|
||||
|
||||
p['supplier'] # returns: Supplier(name='default supplier')
|
||||
p['supplier2'] # raises KeyError
|
||||
p['supplier2'] = Supplier()
|
||||
p['supplier2'] # returns: Supplier(name='anonymous supplier')
|
||||
|
||||
p['variants'] # raises KeyError
|
||||
p['variants2'] # returns []
|
||||
|
||||
p['categories'] # raises KeyError
|
||||
p.get('categories') # returns None
|
||||
|
||||
p['numbers'] # returns []
|
||||
}}}
|
||||
|
||||
=== Accesing and changing nested item values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
p = Product(supplier=Supplier(name="some name", url="http://example.com"))
|
||||
p['supplier']['url'] # returns 'http://example.com'
|
||||
p['supplier']['url'] = "http://www.other.com" # works as expected
|
||||
p['supplier']['url'] = 123 # fails: wrong type for supplier url
|
||||
|
||||
p['variants'] = [v1, v2]
|
||||
p['variants'][0]['name'] # returns v1 name
|
||||
p['variants'][1]['name'] # returns v2 name
|
||||
|
||||
# XXX: decide what to do about these cases:
|
||||
p['variants'].append(v3) # works but doesn't check type of v3
|
||||
p['variants'].append(1) # works but shouldn't?
|
||||
}}}
|
||||
= SEP-003 - Nested items API (!ItemField) =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||3||
|
||||
||'''Title:'''||Nested items API (!ItemField) ||
|
||||
||'''Author:'''||Pablo Hoffman||
|
||||
||'''Created:'''||2009-07-19||
|
||||
||'''Status'''||Obsoleted by [wiki:SEP-008]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This page presents different usage scenarios for the new nested items field API called !ItemField.
|
||||
|
||||
== Prerequisites ==
|
||||
|
||||
This API proposal relies on the following API:
|
||||
|
||||
1. instantiating a item with an item instance as its first argument (ie. {{{item2 = MyItem(item1)}}}) must return a '''copy''' of the first item instance)
|
||||
2. items can be instantiated using this syntax: {{{item = Item(attr1=value1, attr2=value2)}}}
|
||||
|
||||
== Proposed Implementation of !ItemField ==
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.fields import BaseField
|
||||
|
||||
class ItemField(BaseField):
|
||||
def __init__(self, item_type, default=None):
|
||||
self._item_type = item_type
|
||||
super(ItemField, self).__init__(default)
|
||||
|
||||
def to_python(self, value):
|
||||
return self._item_type(value) if not isinstance(value, self._item_type) else value
|
||||
|
||||
def get_default(self):
|
||||
# WARNING: returns default item instead of a copy - this must be well documented, as Items are mutable objects and may lead to unexpected behaviors
|
||||
# always returning a copy may not be desirable either (see Supplier item, for example). this method can be overridden to change this behaviour
|
||||
return self._default
|
||||
}}}
|
||||
|
||||
== Usage Scenarios ==
|
||||
|
||||
=== Defining an item containing !ItemField's ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.item.models import Item
|
||||
from scrapy.item.fields import ListField, ItemField, TextField, UrlField, DecimalField
|
||||
|
||||
class Supplier(Item):
|
||||
name = TextField(default="anonymous supplier")
|
||||
url = UrlField()
|
||||
|
||||
class Variant(Item):
|
||||
name = TextField(required=True)
|
||||
url = UrlField()
|
||||
price = DecimalField()
|
||||
|
||||
class Product(Variant):
|
||||
supplier = ItemField(Supplier, default=Supplier(name="default supplier")
|
||||
variants = ListField(ItemField(Variant))
|
||||
|
||||
# these ones are used for documenting default value examples
|
||||
supplier2 = ItemField(Supplier)
|
||||
variants2 = ListField(ItemField(Variant), default=[])
|
||||
}}}
|
||||
|
||||
It's important to note here that the (perhaps most intuitive) way of defining a Product-Variant
|
||||
relationship (ie. defining a recursive !ItemField) doesn't work. For example, this fails to compile:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Product(Item):
|
||||
variants = ItemField(Product) # Fails to compile
|
||||
}}}
|
||||
|
||||
=== Assigning an item field ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
supplier = Supplier(name="Supplier 1", url="http://example.com")
|
||||
|
||||
p = Product()
|
||||
|
||||
# standard assignment
|
||||
p['supplier'] = supplier
|
||||
# this also works as it tries to instantiate a Supplier with the given dict
|
||||
p['supplier'] = {'name': 'Supplier 1' url='http://example.com'}
|
||||
# this fails because it can't instantiate a Supplier
|
||||
p['supplier'] = 'Supplier 1'
|
||||
# this fails because url doesn't have the valid type
|
||||
p['supplier'] = {'name': 'Supplier 1' url=123}
|
||||
|
||||
v1 = Variant()
|
||||
v1['name'] = "lala"
|
||||
v1['price'] = Decimal("100")
|
||||
|
||||
v2 = Variant()
|
||||
v2['name'] = "lolo"
|
||||
v2['price'] = Decimal("150")
|
||||
|
||||
# standard assignment
|
||||
p['variants'] = [v1, v2] # OK
|
||||
# can also instantiate at assignment time
|
||||
p['variants'] = [v1, Variant(name="lolo", price=Decimal("150")]
|
||||
# this also works as it tries to instantiate a Variant with the given dict
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': Decimal("150")]
|
||||
# this fails because it can't instantiate a Variant
|
||||
p['variants'] = [v1, 'test']
|
||||
# this fails because 'coco' is not a valid value for price
|
||||
p['variants'] = [v1, {'name': 'lolo', 'price': 'coco']
|
||||
}}}
|
||||
|
||||
=== Default values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
p = Product()
|
||||
|
||||
p['supplier'] # returns: Supplier(name='default supplier')
|
||||
p['supplier2'] # raises KeyError
|
||||
p['supplier2'] = Supplier()
|
||||
p['supplier2'] # returns: Supplier(name='anonymous supplier')
|
||||
|
||||
p['variants'] # raises KeyError
|
||||
p['variants2'] # returns []
|
||||
|
||||
p['categories'] # raises KeyError
|
||||
p.get('categories') # returns None
|
||||
|
||||
p['numbers'] # returns []
|
||||
}}}
|
||||
|
||||
=== Accesing and changing nested item values ===
|
||||
|
||||
{{{
|
||||
#!python
|
||||
|
||||
p = Product(supplier=Supplier(name="some name", url="http://example.com"))
|
||||
p['supplier']['url'] # returns 'http://example.com'
|
||||
p['supplier']['url'] = "http://www.other.com" # works as expected
|
||||
p['supplier']['url'] = 123 # fails: wrong type for supplier url
|
||||
|
||||
p['variants'] = [v1, v2]
|
||||
p['variants'][0]['name'] # returns v1 name
|
||||
p['variants'][1]['name'] # returns v2 name
|
||||
|
||||
# XXX: decide what to do about these cases:
|
||||
p['variants'].append(v3) # works but doesn't check type of v3
|
||||
p['variants'].append(1) # works but shouldn't?
|
||||
}}}
|
||||
|
|
|
|||
1224
sep/sep-014.trac
1224
sep/sep-014.trac
File diff suppressed because it is too large
Load Diff
528
sep/sep-016.trac
528
sep/sep-016.trac
|
|
@ -1,265 +1,265 @@
|
|||
= SEP-016: Leg Spider =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||16||
|
||||
||'''Title:'''||Leg Spider||
|
||||
||'''Author:'''||Insophia Team||
|
||||
||'''Created:'''||2010-06-03||
|
||||
||'''Status'''||Superseded by [wiki:SEP-018]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders.
|
||||
|
||||
== Rationale ==
|
||||
|
||||
The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy.
|
||||
|
||||
The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider".
|
||||
|
||||
== {{{LegSpider}}} API ==
|
||||
|
||||
A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods:
|
||||
|
||||
* {{{legs}}}
|
||||
* legs composing this spider
|
||||
* {{{process_response(response)}}}
|
||||
* Process a (downloaded) response and return a list of requests and items
|
||||
* {{{process_request(request)}}}
|
||||
* Process a request after it has been extracted and before returning it from the spider
|
||||
* {{{process_item(item)}}}
|
||||
* Process an item after it has been extracted and before returning it from the spider
|
||||
* {{{set_spider()}}}
|
||||
* Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior.
|
||||
|
||||
== How Leg Spiders work ==
|
||||
|
||||
1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output.
|
||||
2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself.
|
||||
|
||||
== Leg Spider examples ==
|
||||
|
||||
=== Regex (HTML) Link Extractor ===
|
||||
|
||||
A typical application of LegSpider's is to build Link Extractors. For example:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class RegexHtmlLinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if isinstance(response, HtmlResponse):
|
||||
allowed_regexes = self.spider.url_regexes_to_follow
|
||||
# extract urls to follow using allowed_regexes
|
||||
return [Request(x) for x in urls_to_follow]
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexHtmlLinkExtractor()]
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
def parse_response(self, response):
|
||||
# parse response and extract items
|
||||
return items
|
||||
}}}
|
||||
|
||||
=== RSS2 link extractor ===
|
||||
|
||||
This is a Leg Spider that can be used for following links from RSS2 feeds.
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Rss2LinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if response.headers.get('Content-type') == 'application/rss+xml':
|
||||
xs = XmlXPathSelector(response)
|
||||
urls = xs.select("//item/link/text()").extract()
|
||||
return [Request(x) for x in urls]
|
||||
}}}
|
||||
|
||||
=== Callback dispatcher based on rules ===
|
||||
|
||||
Another example could be to build a callback dispatcher based on rules:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CallbackRules(LegSpider):
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(CallbackRules, self).__init__(*a, **kw)
|
||||
for regex, method_name in self.spider.callback_rules.items():
|
||||
r = re.compile(regex)
|
||||
m = getattr(self.spider, method_name, None)
|
||||
if m:
|
||||
self._rules[r] = m
|
||||
|
||||
def process_response(self, response):
|
||||
for regex, method in self._rules.items():
|
||||
m = regex.search(response.url)
|
||||
if m:
|
||||
return method(response)
|
||||
return []
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CallbackRules()]
|
||||
callback_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
def parse_product(self, response):
|
||||
# parse reponse and populate item
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== URL Canonicalizers ===
|
||||
|
||||
Another example could be for building URL canonicalizers:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CanonializeUrl(LegSpider):
|
||||
|
||||
def process_request(self, request):
|
||||
curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
|
||||
return request.replace(url=curl)
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CanonicalizeUrl()]
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
# ...
|
||||
}}}
|
||||
|
||||
=== Setting item identifier ===
|
||||
|
||||
Another example could be for setting a unique identifier to items, based on certain fields:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class ItemIdSetter(LegSpider):
|
||||
|
||||
def process_item(self, item):
|
||||
id_field = self.spider.id_field
|
||||
id_fields_to_hash = self.spider.id_fields_to_hash
|
||||
item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash)
|
||||
return item
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [ItemIdSetter()]
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_response(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== Combining multiple leg spiders ===
|
||||
|
||||
Here's an example that combines functionality from multiple leg spiders:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()]
|
||||
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
parse_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_product(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
|
||||
def process_category(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
|
||||
|
||||
== Leg Spiders vs Spider middlewares ==
|
||||
|
||||
A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality.
|
||||
|
||||
== When not to use Leg Spiders ==
|
||||
|
||||
Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as:
|
||||
|
||||
* Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware.
|
||||
* Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit.
|
||||
|
||||
== {{{LegSpider}}} proof-of-concept implementation ==
|
||||
|
||||
Here's a proof-of-concept implementation of {{{LegSpider}}}:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
class LegSpider(BaseSpider):
|
||||
"""A spider made of legs"""
|
||||
|
||||
legs = []
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(LegSpider, self).__init__(*args, **kwargs)
|
||||
self._legs = [self] + self.legs[:]
|
||||
for l in self._legs:
|
||||
l.set_spider(self)
|
||||
|
||||
def parse(self, response):
|
||||
res = self._process_response(response)
|
||||
for r in res:
|
||||
if isinstance(r, BaseItem):
|
||||
yield self._process_item(r)
|
||||
else:
|
||||
yield self._process_request(r)
|
||||
|
||||
def process_response(self, response):
|
||||
return []
|
||||
|
||||
def process_request(self, request):
|
||||
return request
|
||||
|
||||
def process_item(self, item):
|
||||
return item
|
||||
|
||||
def set_spider(self, spider):
|
||||
self.spider = spider
|
||||
|
||||
def _process_response(self, response):
|
||||
res = []
|
||||
for l in self._legs:
|
||||
res.extend(iterate_spider_output(l.process_response(response)))
|
||||
return res
|
||||
|
||||
def _process_request(self, request):
|
||||
for l in self._legs:
|
||||
request = l.process_request(request)
|
||||
return request
|
||||
|
||||
def _process_item(self, item):
|
||||
for l in self._legs:
|
||||
item = l.process_item(item)
|
||||
return item
|
||||
= SEP-016: Leg Spider =
|
||||
|
||||
[[PageOutline(2-5,Contents)]]
|
||||
|
||||
||'''SEP:'''||16||
|
||||
||'''Title:'''||Leg Spider||
|
||||
||'''Author:'''||Insophia Team||
|
||||
||'''Created:'''||2010-06-03||
|
||||
||'''Status'''||Superseded by [wiki:SEP-018]||
|
||||
|
||||
== Introduction ==
|
||||
|
||||
This SEP introduces a new kind of Spider called {{{LegSpider}}} which provides modular functionality which can be plugged to different spiders.
|
||||
|
||||
== Rationale ==
|
||||
|
||||
The purpose of Leg Spiders is to define an architecture for building spiders based on smaller well-tested components (aka. Legs) that can be combined to achieve the desired functionality. These reusable components will benefit all Scrapy users by building a repository of well-tested components (legs) that can be shared among different spiders and projects. Some of them will come bundled with Scrapy.
|
||||
|
||||
The Legs themselves can be also combined with sub-legs, in a hierarchical fashion. Legs are also spiders themselves, hence the name "Leg Spider".
|
||||
|
||||
== {{{LegSpider}}} API ==
|
||||
|
||||
A {{{LegSpider}}} is a {{{BaseSpider}}} subclass that adds the following attributes and methods:
|
||||
|
||||
* {{{legs}}}
|
||||
* legs composing this spider
|
||||
* {{{process_response(response)}}}
|
||||
* Process a (downloaded) response and return a list of requests and items
|
||||
* {{{process_request(request)}}}
|
||||
* Process a request after it has been extracted and before returning it from the spider
|
||||
* {{{process_item(item)}}}
|
||||
* Process an item after it has been extracted and before returning it from the spider
|
||||
* {{{set_spider()}}}
|
||||
* Defines the main spider associated with this Leg Spider, which is often used to configure the Leg Spider behavior.
|
||||
|
||||
== How Leg Spiders work ==
|
||||
|
||||
1. Each Leg Spider has zero or many Leg Spiders associated with it. When a response arrives, the Leg Spider process it with its {{{process_response}}} method and also the {{{process_response}}} method of all its "sub leg spiders". Finally, the output of all of them is combined to produce the final aggregated output.
|
||||
2. Each element of the aggregated output of {{{process_response}}} is processed with either {{{process_item}}} or {{{process_request}}} before being returned from the spider. Similar to {{{process_response}}}, each item/request is processed with all {{{process_{request,item}}}} of the leg spiders composing the spider, and also with those of the spider itself.
|
||||
|
||||
== Leg Spider examples ==
|
||||
|
||||
=== Regex (HTML) Link Extractor ===
|
||||
|
||||
A typical application of LegSpider's is to build Link Extractors. For example:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class RegexHtmlLinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if isinstance(response, HtmlResponse):
|
||||
allowed_regexes = self.spider.url_regexes_to_follow
|
||||
# extract urls to follow using allowed_regexes
|
||||
return [Request(x) for x in urls_to_follow]
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexHtmlLinkExtractor()]
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
def parse_response(self, response):
|
||||
# parse response and extract items
|
||||
return items
|
||||
}}}
|
||||
|
||||
=== RSS2 link extractor ===
|
||||
|
||||
This is a Leg Spider that can be used for following links from RSS2 feeds.
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class Rss2LinkExtractor(LegSpider):
|
||||
|
||||
def process_response(self, response):
|
||||
if response.headers.get('Content-type') == 'application/rss+xml':
|
||||
xs = XmlXPathSelector(response)
|
||||
urls = xs.select("//item/link/text()").extract()
|
||||
return [Request(x) for x in urls]
|
||||
}}}
|
||||
|
||||
=== Callback dispatcher based on rules ===
|
||||
|
||||
Another example could be to build a callback dispatcher based on rules:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CallbackRules(LegSpider):
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(CallbackRules, self).__init__(*a, **kw)
|
||||
for regex, method_name in self.spider.callback_rules.items():
|
||||
r = re.compile(regex)
|
||||
m = getattr(self.spider, method_name, None)
|
||||
if m:
|
||||
self._rules[r] = m
|
||||
|
||||
def process_response(self, response):
|
||||
for regex, method in self._rules.items():
|
||||
m = regex.search(response.url)
|
||||
if m:
|
||||
return method(response)
|
||||
return []
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CallbackRules()]
|
||||
callback_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
def parse_product(self, response):
|
||||
# parse response and populate item
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== URL Canonicalizers ===
|
||||
|
||||
Another example could be for building URL canonicalizers:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class CanonializeUrl(LegSpider):
|
||||
|
||||
def process_request(self, request):
|
||||
curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
|
||||
return request.replace(url=curl)
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [CanonicalizeUrl()]
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
# ...
|
||||
}}}
|
||||
|
||||
=== Setting item identifier ===
|
||||
|
||||
Another example could be for setting a unique identifier to items, based on certain fields:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class ItemIdSetter(LegSpider):
|
||||
|
||||
def process_item(self, item):
|
||||
id_field = self.spider.id_field
|
||||
id_fields_to_hash = self.spider.id_fields_to_hash
|
||||
item[id_field] = make_hash_based_on_fields(item, id_fields_to_hash)
|
||||
return item
|
||||
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [ItemIdSetter()]
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_response(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
=== Combining multiple leg spiders ===
|
||||
|
||||
Here's an example that combines functionality from multiple leg spiders:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
class MySpider(LegSpider):
|
||||
|
||||
legs = [RegexLinkExtractor(), ParseRules(), CanonicalizeUrl(), ItemIdSetter()]
|
||||
|
||||
url_regexes_to_follow = ['/product.php?.*']
|
||||
|
||||
parse_rules = {
|
||||
'/product.php.*': 'parse_product',
|
||||
'/category.php.*': 'parse_category',
|
||||
}
|
||||
|
||||
canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...]
|
||||
|
||||
id_field = 'guid'
|
||||
id_fields_to_hash = ['supplier_name', 'supplier_id']
|
||||
|
||||
def process_product(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
|
||||
def process_category(self, item):
|
||||
# extract item from response
|
||||
return item
|
||||
}}}
|
||||
|
||||
|
||||
|
||||
== Leg Spiders vs Spider middlewares ==
|
||||
|
||||
A common question that would arise is when one should use Leg Spiders and when to use Spider middlewares. Leg Spiders functionality is meant to implement spider-specific functionality, like link extraction which has custom rules per spider. Spider middlewares, on the other hand, are meant to implement global functionality.
|
||||
|
||||
== When not to use Leg Spiders ==
|
||||
|
||||
Leg Spiders are not a silver bullet to implement all kinds of spiders, so it's important to keep in mind their scope and limitations, such as:
|
||||
|
||||
* Leg Spiders can't filter duplicate requests, since they don't have access to all requests at the same time. This functionality should be done in a spider or scheduler middleware.
|
||||
* Leg Spiders are meant to be used for spiders whose behavior (requests & items to extract) depends only on the current page and not previously crawled pages (aka. "context-free spiders"). If your spider has some custom logic with chained downloads (for example, multi-page items) then Leg Spiders may not be a good fit.
|
||||
|
||||
== {{{LegSpider}}} proof-of-concept implementation ==
|
||||
|
||||
Here's a proof-of-concept implementation of {{{LegSpider}}}:
|
||||
|
||||
{{{
|
||||
#!python
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
class LegSpider(BaseSpider):
|
||||
"""A spider made of legs"""
|
||||
|
||||
legs = []
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(LegSpider, self).__init__(*args, **kwargs)
|
||||
self._legs = [self] + self.legs[:]
|
||||
for l in self._legs:
|
||||
l.set_spider(self)
|
||||
|
||||
def parse(self, response):
|
||||
res = self._process_response(response)
|
||||
for r in res:
|
||||
if isinstance(r, BaseItem):
|
||||
yield self._process_item(r)
|
||||
else:
|
||||
yield self._process_request(r)
|
||||
|
||||
def process_response(self, response):
|
||||
return []
|
||||
|
||||
def process_request(self, request):
|
||||
return request
|
||||
|
||||
def process_item(self, item):
|
||||
return item
|
||||
|
||||
def set_spider(self, spider):
|
||||
self.spider = spider
|
||||
|
||||
def _process_response(self, response):
|
||||
res = []
|
||||
for l in self._legs:
|
||||
res.extend(iterate_spider_output(l.process_response(response)))
|
||||
return res
|
||||
|
||||
def _process_request(self, request):
|
||||
for l in self._legs:
|
||||
request = l.process_request(request)
|
||||
return request
|
||||
|
||||
def _process_item(self, item):
|
||||
for l in self._legs:
|
||||
item = l.process_item(item)
|
||||
return item
|
||||
}}}
|
||||
1102
sep/sep-018.trac
1102
sep/sep-018.trac
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue