From 5fa1009712b6b79561d64ad1bc26a2e554158d81 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Fri, 9 Jan 2009 13:45:11 +0000 Subject: [PATCH] Improved adaptors documentation --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40695 --- scrapy/trunk/docs/topics/adaptors.rst | 115 +++++++++++++++++++++----- 1 file changed, 96 insertions(+), 19 deletions(-) diff --git a/scrapy/trunk/docs/topics/adaptors.rst b/scrapy/trunk/docs/topics/adaptors.rst index 5e0b69e51..5523283ed 100644 --- a/scrapy/trunk/docs/topics/adaptors.rst +++ b/scrapy/trunk/docs/topics/adaptors.rst @@ -10,7 +10,8 @@ Quick overview | Adaptors are basically functions that receive one value (advanced adaptors may receive more, but we'll see that later), modify it, and return a new value. | In order to adapt our scraped data we can use an adaptor pipeline for each of the item's attributes. -| Adaptor pipelines are nothing else but a list of adaptors which will be iterated, calling each adaptor and passing the values from one to each other. +| Adaptor pipelines are nothing else but a list of adaptors which will be iterated at the moment of assigning an attribute, calling each adaptor + and passing the values from one to each other. Example use =========== @@ -91,25 +92,31 @@ The weight and the price could get a better parsing though; at least to convert | In this case it wasn't necessary to use the extract adaptor, because applying the ``re`` method over a selector already extracts the resulting content. | Now, the price was quite easy, but the weight is a bit tricky, for the simple fact that it has more than one weighing unit. In this case, we'll have to write our own little adaptor that takes care of parsing a string and returning a Decimal according to its unit. -| Something like this could work, although it's very primitive: - >>> def parse_weight(string): - '''This adaptor receives a string and tries to parse it - as weight, converting it to grams and returning a Decimal object''' - conversions = { - 'kg': Decimal('0.001'), - 'gr': Decimal('1'), - 'mg': Decimal('1000'), - } - quantity = re.search('(\d+)', string) - if not quantity: - return Decimal('0') - quantity = Decimal(quantity.group(1)) - for unit in conversions: - if unit in string: - quantity = quantity / conversions[unit] - break - return quantity +Something like this could work, although it's very primitive:: + + def parse_weight(string): + '''This adaptor receives a string and tries to parse it + as weight, converting it to grams and returning a Decimal object''' + + conversions = { + 'kg': Decimal('0.001'), + 'gr': Decimal('1'), + 'mg': Decimal('1000'), + } + + quantity = re.search('(\d+)', string) + if not quantity: + return Decimal('0') + quantity = Decimal(quantity.group(1)) + + for unit in conversions: + if unit in string: + quantity = quantity / conversions[unit] + break + return quantity + +And now some testing:: >>> parse_weight('4kg') Decimal('4000') @@ -195,3 +202,73 @@ Scraping the sample page with this code would give us these items:: There could be more parsing done here through adaptors, like parsing different price currencies, or more advanced weight parsing (this one was very, very, simple and buggy). Nevertheless, I hope that this was useful as an example use of adaptors. + + +More complex adaptors +===================== + +| Okay, what happens when you have adaptors that receive extra parameters in order to modify their behaviour? +| For example, going back to money, what if you have a function capable of converting from one currency to another one by receiving three parameters: the value, its currency, + and the currency to convert to? +| Well, don't worry, Scrapy can handle this situation and use this adaptor too, and in fact, its something quite easy to do. +| +| Basically the difference between a regular adaptor and a "complex" one (I call them complex just for the fact that the others are *very* simple), is that the first one + always receives and returns a single value, while the others may receive more, passed by you to the item's ``attribute`` method. +| Whenever this method (``attribute``) receives any extra parameters (apart from the value to set, ``override``, and ``add``), it automatically passes them to any adaptor + that receives a parameter called ``adaptor_args`` in a dictionary. +| This adaptor should read the dictionary and check if it has received any parameters there. +| +| Notice that you should be careful with the naming of these parameters in order to not generate confusion between adaptors, because every adaptor (that has the ``adaptor_args`` + parameter, of course) will receive the same dict of keywords, containing keywords that are destined for itself, and some that aren't. +| For example, it's not very wise to use a parameter called 'encoding' unless you'd like all the adaptors to receive it; otherwise it'd be better to use the adaptor's name + as a prefix to the parameter, just to make it clearer. +| +| Let's now see the currency example, now written in code. + +First, we'll define a class called Currency and a dictionary, in order to store some currencies and their exchange rate:: + + class Currency(object): + def __init__(self, from_dollars, to_dollars): + self._from_dollars = from_dollars + self._to_dollars = to_dollars + + def is_decimal(self, value): + if not isinstance(value, Decimal): + raise Exception('Value must be a Decimal object') + return True + + def from_dollars(self, value): + if self.is_decimal(value): + return value * self._from_dollars + + def to_dollars(self, value): + if self.is_decimal(value): + return value * self._to_dollars + + currencies = { + 'dollar': Currency(from_dollars=Decimal('1'), to_dollars=Decimal('1')), + 'euro': Currency(from_dollars=Decimal('0.7294'), to_dollars=Decimal('1.3710')), + 'yen': Currency(from_dollars=Decimal('90.82'), to_dollars=Decimal('0.0110')), + 'pound': Currency(from_dollars=Decimal('0.6547'), to_dollars=Decimal('1.5274')), + } + +Ok, now that we've got some exchange information, the only thing missing is the adaptor that makes use of it:: + + def exchange_adaptor(value, adaptor_args): + currency_from = adaptor_args.get('currency_from', 'dollar') + currency_to = adaptor_args.get('currency_to', 'dollar') + + if currency_from in currencies and currency_to in currencies: + dollars = currencies[currency_from].to_dollars(value) + return currencies[currency_to].from_dollars(dollars) + else: + raise Exception('Unsupported currencies were specified') + +And finally, testing it!:: + + >>> item = ScrapedItem() + >>> item.set_attrib_adaptors('price', [ Decimal, exchange_adaptor ]) + >>> item.attribute('price', '20.5', currency_from='euro', currency_to='dollar') + >>> item.price + Decimal('28.10550') +