From 2d3803672bbb4c710b00a89c49801b274414b6d0 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 9 Apr 2014 05:34:14 +0600 Subject: [PATCH] DOC use top-level shortcuts in docs --- docs/intro/overview.rst | 16 +- docs/intro/tutorial.rst | 205 +++++++++++++------------- docs/topics/commands.rst | 4 +- docs/topics/debug.rst | 11 +- docs/topics/djangoitem.rst | 69 ++++----- docs/topics/downloader-middleware.rst | 4 +- docs/topics/exporters.rst | 24 +-- docs/topics/firebug.rst | 8 +- docs/topics/images.rst | 14 +- docs/topics/items.rst | 24 +-- docs/topics/jobs.rst | 4 +- docs/topics/loaders.rst | 11 +- docs/topics/practices.rst | 14 +- docs/topics/request-response.rst | 21 ++- docs/topics/selectors.rst | 12 +- docs/topics/shell.rst | 4 +- docs/topics/spider-middleware.rst | 2 +- docs/topics/spiders.rst | 63 ++++---- 18 files changed, 259 insertions(+), 251 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 5803df4e3..50d571309 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -43,13 +43,13 @@ done through :ref:`Scrapy Items ` (Torrent files, in this case). This would be our Item:: - from scrapy.item import Item, Field + import scrapy - class TorrentItem(Item): - url = Field() - name = Field() - description = Field() - size = Field() + class TorrentItem(scrapy.Item): + url = scrapy.Field() + name = scrapy.Field() + description = scrapy.Field() + size = scrapy.Field() Write a Spider to extract the data ================================== @@ -129,9 +129,9 @@ For more information about XPath see the `XPath reference`_. Finally, here's the spider code:: + import scrapy from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor - from scrapy.selector import Selector class MininovaSpider(CrawlSpider): @@ -141,7 +141,7 @@ Finally, here's the spider code:: rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] def parse_torrent(self, response): - sel = Selector(response) + sel = scrapy.Selector(response) torrent = TorrentItem() torrent['url'] = response.url torrent['name'] = sel.xpath("//h1/text()").extract() diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index b48898f0b..8d1693b28 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -36,20 +36,20 @@ Creating a project Before you start scraping, you will have set up a new Scrapy project. Enter a directory where you'd like to store your code and then run:: - scrapy startproject tutorial + scrapy startproject tutorial This will create a ``tutorial`` directory with the following contents:: - tutorial/ - scrapy.cfg - tutorial/ - __init__.py - items.py - pipelines.py - settings.py - spiders/ - __init__.py - ... + tutorial/ + scrapy.cfg + tutorial/ + __init__.py + items.py + pipelines.py + settings.py + spiders/ + __init__.py + ... These are basically: @@ -68,8 +68,8 @@ Defining our Item like simple python dicts but provide additional protection against populating undeclared fields, to prevent typos. -They are declared by creating a :class:`scrapy.item.Item` class and defining -its attributes as :class:`scrapy.item.Field` objects, like you will in an ORM +They are declared by creating a :class:`scrapy.Item ` class and defining +its attributes as :class:`scrapy.Field ` objects, like you will in an ORM (don't worry if you're not familiar with ORMs, you will see that this is an easy task). @@ -78,12 +78,12 @@ from dmoz.org, as we want to capture the name, url and description of the sites, we define fields for each of these three attributes. To do that, we edit ``items.py``, found in the ``tutorial`` directory. Our Item class looks like this:: - from scrapy.item import Item, Field + import scrapy - class DmozItem(Item): - title = Field() - link = Field() - desc = Field() + class DmozItem(scrapy.Item): + title = scrapy.Field() + link = scrapy.Field() + desc = scrapy.Field() This may seem complicated at first, but defining the item allows you to use other handy components of Scrapy that need to know how your item looks. @@ -97,7 +97,7 @@ of domains). They define an initial list of URLs to download, how to follow links, and how to parse the contents of those pages to extract :ref:`items `. -To create a Spider, you must subclass :class:`scrapy.spider.Spider` and +To create a Spider, you must subclass :class:`scrapy.Spider ` and define the three main mandatory attributes: * :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be @@ -123,19 +123,20 @@ define the three main mandatory attributes: This is the code for our first Spider; save it in a file named ``dmoz_spider.py`` under the ``tutorial/spiders`` directory:: - from scrapy.spider import Spider + import scrapy - class DmozSpider(Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] + class DmozSpider(scrapy.Spider): + name = "dmoz" + allowed_domains = ["dmoz.org"] + start_urls = [ + "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", + "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + ] - def parse(self, response): - filename = response.url.split("/")[-2] - open(filename, 'wb').write(response.body) + def parse(self, response): + filename = response.url.split("/")[-2] + with open(filename, 'wb') as f: + f.write(response.body) Crawling -------- @@ -170,9 +171,9 @@ created: *Books* and *Resources*, with the content of both URLs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy creates :class:`scrapy.http.Request` objects for each URL in the -``start_urls`` attribute of the Spider, and assigns them the ``parse`` method of -the spider as their callback function. +Scrapy creates :class:`scrapy.Request ` objects +for each URL in the ``start_urls`` attribute of the Spider, and assigns +them the ``parse`` method of the spider as their callback function. These Requests are scheduled, then executed, and :class:`scrapy.http.Response` objects are returned and then fed back to the @@ -243,7 +244,7 @@ installed on your system. To start a shell, you must go to the project's top level directory and run:: - scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/" + scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/" .. note:: @@ -281,20 +282,20 @@ on response's type. So let's try it:: - In [1]: sel.xpath('//title') - Out[1]: [Open Directory - Computers: Progr'>] + In [1]: sel.xpath('//title') + Out[1]: [Open Directory - Computers: Progr'>] - In [2]: sel.xpath('//title').extract() - Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] + In [2]: sel.xpath('//title').extract() + Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [3]: sel.xpath('//title/text()') - Out[3]: [] + In [3]: sel.xpath('//title/text()') + Out[3]: [] - In [4]: sel.xpath('//title/text()').extract() - Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] + In [4]: sel.xpath('//title/text()').extract() + Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [5]: sel.xpath('//title/text()').re('(\w+):') - Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] + In [5]: sel.xpath('//title/text()').re('(\w+):') + Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] Extracting the data ^^^^^^^^^^^^^^^^^^^ @@ -313,30 +314,30 @@ is inside a ``
    `` element, in fact the *second* ``
      `` element. So we can select each ``
    • `` element belonging to the sites list with this code:: - sel.xpath('//ul/li') + sel.xpath('//ul/li') And from them, the sites descriptions:: - sel.xpath('//ul/li/text()').extract() + sel.xpath('//ul/li/text()').extract() The sites titles:: - sel.xpath('//ul/li/a/text()').extract() + sel.xpath('//ul/li/a/text()').extract() And the sites links:: - sel.xpath('//ul/li/a/@href').extract() + sel.xpath('//ul/li/a/@href').extract() As we've said before, each ``.xpath()`` call returns a list of selectors, so we can concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use that property here, so:: - sites = sel.xpath('//ul/li') - for site in sites: - title = site.xpath('a/text()').extract() - link = site.xpath('a/@href').extract() - desc = site.xpath('text()').extract() - print title, link, desc + sites = sel.xpath('//ul/li') + for site in sites: + title = site.xpath('a/text()').extract() + link = site.xpath('a/@href').extract() + desc = site.xpath('text()').extract() + print title, link, desc .. note:: @@ -347,55 +348,9 @@ that property here, so:: Let's add this code to our spider:: - from scrapy.spider import Spider - from scrapy.selector import Selector + import scrapy - class DmozSpider(Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] - - def parse(self, response): - sel = Selector(response) - sites = sel.xpath('//ul/li') - for site in sites: - title = site.xpath('a/text()').extract() - link = site.xpath('a/@href').extract() - desc = site.xpath('text()').extract() - print title, link, desc - -Notice we import our Selector class from scrapy.selector and instantiate a -new Selector object. We can now specify our XPaths just as we did in the shell. -Now try crawling the dmoz.org domain again and you'll see sites being printed -in your output, run:: - - scrapy crawl dmoz - -Using our item --------------- - -:class:`~scrapy.item.Item` objects are custom python dicts; you can access the -values of their fields (attributes of the class we defined earlier) using the -standard dict syntax like:: - - >>> item = DmozItem() - >>> item['title'] = 'Example title' - >>> item['title'] - 'Example title' - -Spiders are expected to return their scraped data inside -:class:`~scrapy.item.Item` objects. So, in order to return the data we've -scraped so far, the final code for our Spider would be like this:: - - from scrapy.spider import Spider - from scrapy.selector import Selector - - from tutorial.items import DmozItem - - class DmozSpider(Spider): + class DmozSpider(scrapy.Spider): name = "dmoz" allowed_domains = ["dmoz.org"] start_urls = [ @@ -404,7 +359,51 @@ scraped so far, the final code for our Spider would be like this:: ] def parse(self, response): - sel = Selector(response) + sel = scrapy.Selector(response) + sites = sel.xpath('//ul/li') + for site in sites: + title = site.xpath('a/text()').extract() + link = site.xpath('a/@href').extract() + desc = site.xpath('text()').extract() + print title, link, desc + +Notice we import our Selector class from scrapy and instantiate a new +Selector object. We can now specify our XPaths just as we did in the shell. +Now try crawling the dmoz.org domain again and you'll see sites being printed +in your output, run:: + + scrapy crawl dmoz + +Using our item +-------------- + +:class:`~scrapy.item.Item` objects are custom python dicts; you can access the +values of their fields (attributes of the class we defined earlier) using the +standard dict syntax like:: + + >>> item = DmozItem() + >>> item['title'] = 'Example title' + >>> item['title'] + 'Example title' + +Spiders are expected to return their scraped data inside +:class:`~scrapy.item.Item` objects. So, in order to return the data we've +scraped so far, the final code for our Spider would be like this:: + + import scrapy + + from tutorial.items import DmozItem + + class DmozSpider(scrapy.Spider): + name = "dmoz" + allowed_domains = ["dmoz.org"] + start_urls = [ + "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", + "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + ] + + def parse(self, response): + sel = scrapy.Selector(response) sites = sel.xpath('//ul/li') items = [] for site in sites: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 672e42e74..41ad3158f 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -190,9 +190,9 @@ Usage example:: xmlfeed $ scrapy genspider -d basic - from scrapy.spider import Spider + import scrapy - class $classname(Spider): + class $classname(scrapy.Spider): name = "$name" allowed_domains = ["$domain"] start_urls = ( diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 47a901a6d..5739d9e1b 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -7,7 +7,10 @@ Debugging Spiders This document explains the most common techniques for debugging spiders. Consider the following scrapy spider below:: - class MySpider(Spider): + import scrapy + from myproject.items import MyItem + + class MySpider(scrapy.Spider): name = 'myspider' start_urls = ( 'http://example.com/page1', @@ -17,13 +20,13 @@ Consider the following scrapy spider below:: def parse(self, response): # collect `item_urls` for item_url in item_urls: - yield Request(url=item_url, callback=self.parse_item) + yield scrapy.Request(item_url, self.parse_item) def parse_item(self, response): item = MyItem() # populate `item` fields - yield Request(url=item_details_url, meta={'item': item}, - callback=self.parse_details) + # and extract item_details_url + yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item}) def parse_details(self, response): item = response.meta['item'] diff --git a/docs/topics/djangoitem.rst b/docs/topics/djangoitem.rst index 5be75c9e1..84417dfab 100644 --- a/docs/topics/djangoitem.rst +++ b/docs/topics/djangoitem.rst @@ -20,59 +20,62 @@ and define its ``django_model`` attribute to be a valid Django model. With this you will get an item with a field for each Django model field. In addition, you can define fields that aren't present in the model and even -override fields that are present in the model defining them in the item. +override fields that are present in the model defining them in the item. Let's see some examples: Creating a Django model for the examples:: - - from django.db import models - class Person(models.Model): - name = models.CharField(max_length=255) - age = models.IntegerField() + from django.db import models + + class Person(models.Model): + name = models.CharField(max_length=255) + age = models.IntegerField() Defining a basic :class:`DjangoItem`:: - from scrapy.contrib.djangoitem import DjangoItem + from scrapy.contrib.djangoitem import DjangoItem + + class PersonItem(DjangoItem): + django_model = Person - class PersonItem(DjangoItem): - django_model = Person - :class:`DjangoItem` work just like :class:`~scrapy.item.Item`:: - >>> p = PersonItem() - >>> p['name'] = 'John' - >>> p['age'] = '22' + >>> p = PersonItem() + >>> p['name'] = 'John' + >>> p['age'] = '22' To obtain the Django model from the item, we call the extra method :meth:`~DjangoItem.save` of the :class:`DjangoItem`:: - >>> person = p.save() - >>> person.name - 'John' - >>> person.age - '22' - >>> person.id - 1 + >>> person = p.save() + >>> person.name + 'John' + >>> person.age + '22' + >>> person.id + 1 The model is already saved when we call :meth:`~DjangoItem.save`, we can prevent this by calling it with ``commit=False``. We can use ``commit=False`` in :meth:`~DjangoItem.save` method to obtain an unsaved model:: - >>> person = p.save(commit=False) - >>> person.name - 'John' - >>> person.age - '22' - >>> person.id - None + >>> person = p.save(commit=False) + >>> person.name + 'John' + >>> person.age + '22' + >>> person.id + None As said before, we can add other fields to the item:: - class PersonItem(DjangoItem): - django_model = Person - sex = Field() + import scrapy + from scrapy.contrib.djangoitem import DjangoItem + + class PersonItem(DjangoItem): + django_model = Person + sex = scrapy.Field() :: @@ -85,9 +88,9 @@ As said before, we can add other fields to the item:: And we can override the fields of the model with your own:: - class PersonItem(DjangoItem): - django_model = Person - name = Field(default='No Name') + class PersonItem(DjangoItem): + django_model = Person + name = scrapy.Field(default='No Name') This is useful to provide properties to the field, like a default or any other property that your project uses. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 30075fa7b..da5a85655 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -198,7 +198,7 @@ There is support for keeping multiple cookie sessions per spider by using the For example:: for i, url in enumerate(urls): - yield Request("http://www.example.com", meta={'cookiejar': i}, + yield scrapy.Request("http://www.example.com", meta={'cookiejar': i}, callback=self.parse_page) Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep @@ -206,7 +206,7 @@ passing it along on subsequent requests. For example:: def parse_page(self, response): # do some processing - return Request("http://www.example.com/otherpage", + return scrapy.Request("http://www.example.com/otherpage", meta={'cookiejar': response.meta['cookiejar']}, callback=self.parse_other_page) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 9353448f2..c472f5b96 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -96,14 +96,14 @@ value and returns its serialized form. Example:: - from scrapy.item import Item, Field + import scrapy - def serialize_price(value): - return '$ %s' % str(value) + def serialize_price(value): + return '$ %s' % str(value) - class Product(Item): - name = Field() - price = Field(serializer=serialize_price) + class Product(scrapy.Item): + name = scrapy.Field() + price = scrapy.Field(serializer=serialize_price) 2. Overriding the serialize_field() method @@ -113,7 +113,7 @@ You can also override the :meth:`~BaseItemExporter.serialize_field()` method to customize how your field value will be exported. Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method -after your custom code. +after your custom code. Example:: @@ -125,7 +125,7 @@ Example:: if field == 'price': return '$ %s' % str(value) return super(Product, self).serialize_field(field, name, value) - + .. _topics-exporters-reference: Built-in Item Exporters reference @@ -146,7 +146,7 @@ BaseItemExporter support for common features used by all (concrete) Item Exporters, such as defining what fields to export, whether to export empty fields, or which encoding to use. - + These features can be configured through the constructor arguments which populate their respective instance attributes: :attr:`fields_to_export`, :attr:`export_empty_fields`, :attr:`encoding`. @@ -278,7 +278,7 @@ CsvItemExporter :param file: the file-like object to use for exporting the data. :param include_headers_line: If enabled, makes the exporter output a header - line with the field names taken from + line with the field names taken from :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. :type include_headers_line: boolean @@ -296,7 +296,7 @@ CsvItemExporter product,price Color TV,1200 DVD player,200 - + .. _csv.writer: http://docs.python.org/library/csv.html#csv.writer PickleItemExporter @@ -304,7 +304,7 @@ PickleItemExporter .. class:: PickleItemExporter(file, protocol=0, \**kwargs) - Exports Items in pickle format to the given file-like object. + Exports Items in pickle format to the given file-like object. :param file: the file-like object to use for exporting the data. diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 5aa03a65e..8b0d5a14c 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -61,7 +61,7 @@ Getting links to follow By looking at the category URLs we can see they share a pattern: http://directory.google.com/Category/Subcategory/Another_Subcategory - + Once we know that, we are able to construct a regular expression to follow those links. For example, the following one:: @@ -103,7 +103,7 @@ This is how the spider would look so far:: Extracting the data =================== -Now we're going to write the code to extract data from those pages. +Now we're going to write the code to extract data from those pages. With the help of Firebug, we'll take a look at some page containing links to websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can @@ -146,7 +146,7 @@ that have that grey colour of the links, Finally, we can write our ``parse_category()`` method:: def parse_category(self, response): - sel = Selector(response) + sel = scrapy.Selector(response) # The path to website links in directory page links = sel.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') @@ -164,6 +164,6 @@ not in the original HTML, such as the typical case of ```` elements. or tags which Therefer in page HTML -sources may on Firebug inspects the live DOM +sources may on Firebug inspects the live DOM .. _has been shut down by Google: http://searchenginewatch.com/article/2096661/Google-Directory-Has-Been-Shut-Down diff --git a/docs/topics/images.rst b/docs/topics/images.rst index 28be43e03..91e59d2a9 100644 --- a/docs/topics/images.rst +++ b/docs/topics/images.rst @@ -67,13 +67,13 @@ In order to use the image pipeline you just need to :ref:`enable it ` and define an item with the ``image_urls`` and ``images`` fields:: - from scrapy.item import Item + import scrapy - class MyItem(Item): + class MyItem(scrapy.Item): # ... other item fields ... - image_urls = Field() - images = Field() + image_urls = scrapy.Field() + images = scrapy.Field() If you need something more complex and want to override the custom images pipeline behaviour, see :ref:`topics-images-override`. @@ -228,7 +228,7 @@ Here are the methods that you should override in your custom Images Pipeline: def get_media_requests(self, item, info): for image_url in item['image_urls']: - yield Request(image_url) + yield scrapy.Request(image_url) Those requests will be processed by the pipeline and, when they have finished downloading, the results will be sent to the @@ -302,15 +302,15 @@ Custom Images pipeline example Here is a full example of the Images Pipeline whose methods are examplified above:: + import scrapy from scrapy.contrib.pipeline.images import ImagesPipeline from scrapy.exceptions import DropItem - from scrapy.http import Request class MyImagesPipeline(ImagesPipeline): def get_media_requests(self, item, info): for image_url in item['image_urls']: - yield Request(image_url) + yield scrapy.Request(image_url) def item_completed(self, results, item, info): image_paths = [x['path'] for ok, x in results if ok] diff --git a/docs/topics/items.rst b/docs/topics/items.rst index bf7ca74d1..ee604a7f1 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -25,13 +25,13 @@ Declaring Items Items are declared using a simple class definition syntax and :class:`Field` objects. Here is an example:: - from scrapy.item import Item, Field + import scrapy - class Product(Item): - name = Field() - price = Field() - stock = Field() - last_updated = Field(serializer=str) + class Product(scrapy.Item): + name = scrapy.Field() + price = scrapy.Field() + stock = scrapy.Field() + last_updated = scrapy.Field(serializer=str) .. note:: Those familiar with `Django`_ will notice that Scrapy Items are declared similar to `Django Models`_, except that Scrapy Items are much @@ -185,14 +185,14 @@ fields) by declaring a subclass of your original Item. For example:: class DiscountedProduct(Product): - discount_percent = Field(serializer=str) - discount_expiration_date = Field() + discount_percent = scrapy.Field(serializer=str) + discount_expiration_date = scrapy.Field() You can also extend field metadata by using the previous field metadata and appending more values, or changing existing values, like this:: class SpecificProduct(Product): - name = Field(Product.fields['name'], serializer=my_serializer) + name = scrapy.Field(Product.fields['name'], serializer=my_serializer) That adds (or replaces) the ``serializer`` metadata key for the ``name`` field, keeping all the previously existing metadata values. @@ -202,11 +202,11 @@ Item objects .. class:: Item([arg]) - Return a new Item optionally initialized from the given argument. - + Return a new Item optionally initialized from the given argument. + Items replicate the standard `dict API`_, including its constructor. The only additional attribute provided by Items is: - + .. attribute:: fields A dictionary containing *all declared fields* for this Item, not only diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 56ea6a8c3..4e09000d2 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -81,7 +81,7 @@ So, for example, this won't work:: def some_callback(self, response): somearg = 'test' - return Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg)) + return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg)) def other_callback(self, response, somearg): print "the argument passed is:", somearg @@ -90,7 +90,7 @@ But this will:: def some_callback(self, response): somearg = 'test' - return Request('http://www.example.com', meta={'somearg': somearg}) + return scrapy.Request('http://www.example.com', meta={'somearg': somearg}) def other_callback(self, response): somearg = response.meta['somearg'] diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index b9db5daa8..4ac514be6 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -181,18 +181,17 @@ this way. However, there is one more place where you can specify the input and output processors to use: in the :ref:`Item Field ` metadata. Here is an example:: - from scrapy.item import Item, Field + import scrapy from scrapy.contrib.loader.processor import MapCompose, Join, TakeFirst - - from scrapy.utils.markup import remove_entities + from w3lib.html import remove_entities from myproject.utils import filter_prices - class Product(Item): - name = Field( + class Product(scrapy.Item): + name = scrapy.Field( input_processor=MapCompose(remove_entities), output_processor=Join(), ) - price = Field( + price = scrapy.Field( default=0, input_processor=MapCompose(remove_entities, filter_prices), output_processor=TakeFirst(), diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index c5427c899..64b3a2da7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -149,18 +149,18 @@ If you are still unable to prevent your bot getting banned, consider contacting Dynamic Creation of Item Classes ================================ -For applications in which the structure of item class is to be determined by +For applications in which the structure of item class is to be determined by user input, or other changing conditions, you can dynamically create item classes instead of manually coding them. :: - from scrapy.item import DictItem, Field + from scrapy.item import DictItem, Field - def create_item_class(class_name, field_list): - field_dict = {} - for field_name in field_list: - field_dict[field_name] = Field() + def create_item_class(class_name, field_list): + field_dict = {} + for field_name in field_list: + field_dict[field_name] = Field() - return type(class_name, (DictItem,), field_dict) + return type(class_name, (DictItem,), field_dict) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b53c21ce7..cda074a71 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -183,8 +183,8 @@ downloaded :class:`Response` object as its first argument. Example:: def parse_page1(self, response): - return Request("http://www.example.com/some_page.html", - callback=self.parse_page2) + return scrapy.Request("http://www.example.com/some_page.html", + callback=self.parse_page2) def parse_page2(self, response): # this would log http://www.example.com/some_page.html @@ -200,8 +200,8 @@ different fields from different pages:: def parse_page1(self, response): item = MyItem() item['main_url'] = response.url - request = Request("http://www.example.com/some_page.html", - callback=self.parse_page2) + request = scrapy.Request("http://www.example.com/some_page.html", + callback=self.parse_page2) request.meta['item'] = item return request @@ -349,14 +349,19 @@ automatically pre-populated and only override a couple of them, such as the user name and password. You can use the :meth:`FormRequest.from_response` method for this job. Here's an example spider which uses it:: - class LoginSpider(Spider): + + import scrapy + + class LoginSpider(scrapy.Spider): name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] def parse(self, response): - return [FormRequest.from_response(response, - formdata={'username': 'john', 'password': 'secret'}, - callback=self.after_login)] + return scrapy.FormRequest.from_response( + response, + formdata={'username': 'john', 'password': 'secret'}, + callback=self.after_login + ) def after_login(self, response): # check login succeed before going on diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8d55fe7cb..0a96331d4 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -53,16 +53,15 @@ Constructing selectors .. highlight:: python Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class -constructed by passing a `Response` object as first argument, the response's -body is what they're going to be "selecting":: +constructed by passing a :class:`~scrapy.http.Response` object as first +argument, the response's body is what they're going to be "selecting":: - from scrapy.spider import Spider - from scrapy.selector import Selector + import scrapy - class MySpider(Spider): + class MySpider(scrapy.Spider): # ... def parse(self, response): - sel = Selector(response) + sel = scrapy.Selector(response) # Using XPath query print sel.xpath('//p') # Using CSS query @@ -262,6 +261,7 @@ The ``test()`` function for example can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit:: + >>> from scrapy import Selector >>> doc = """ ...
      ...
        diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 990dedc6c..8827f7fb6 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -171,10 +171,10 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function. Here's an example of how you would call it from your spider:: - from scrapy.spider import Spider + import scrapy - class MySpider(Spider): + class MySpider(scrapy.Spider): name = "myspider" start_urls = [ "http://example.com", diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index fc7cf734a..3df59998b 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -129,7 +129,7 @@ single Python class that defines one or more of the following methods: :type exception: `Exception`_ object :param spider: the spider which raised the exception - :type spider: :class:`scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spider.Spider` object .. method:: process_start_requests(start_requests, spider) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 399aa7e6a..929e21e10 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -58,7 +58,9 @@ Spider arguments are passed through the :command:`crawl` command using the Spiders receive arguments in their constructors:: - class MySpider(Spider): + import scrapy + + class MySpider(scrapy.Spider): name = 'myspider' def __init__(self, category=None, *args, **kwargs): @@ -82,12 +84,12 @@ rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed. For the examples used in the following spiders, we'll assume you have a project with a ``TestItem`` declared in a ``myproject.items`` module:: - from scrapy.item import Item + import scrapy - class TestItem(Item): - id = Field() - name = Field() - description = Field() + class TestItem(scrapy.Item): + id = scrapy.Field() + name = scrapy.Field() + description = scrapy.Field() .. module:: scrapy.spider @@ -150,9 +152,9 @@ Spider a POST request, you could do:: def start_requests(self): - return [FormRequest("http://www.example.com/login", - formdata={'user': 'john', 'pass': 'secret'}, - callback=self.logged_in)] + return [scrapy.FormRequest("http://www.example.com/login", + formdata={'user': 'john', 'pass': 'secret'}, + callback=self.logged_in)] def logged_in(self, response): # here you would extract links to follow and return Requests for @@ -199,10 +201,10 @@ Spider example Let's see an example:: - from scrapy import log # This module is useful for printing out debug information - from scrapy.spider import Spider + import scrapy - class MySpider(Spider): + + class MySpider(scrapy.Spider): name = 'example.com' allowed_domains = ['example.com'] start_urls = [ @@ -216,12 +218,10 @@ Let's see an example:: Another example returning multiple Requests and Items from a single callback:: - from scrapy.selector import Selector - from scrapy.spider import Spider - from scrapy.http import Request + import scrapy from myproject.items import MyItem - class MySpider(Spider): + class MySpider(scrapy.Spider): name = 'example.com' allowed_domains = ['example.com'] start_urls = [ @@ -231,12 +231,12 @@ Another example returning multiple Requests and Items from a single callback:: ] def parse(self, response): - sel = Selector(response) + sel = scrapy.Selector(response) for h3 in sel.xpath('//h3').extract(): yield MyItem(title=h3) for url in sel.xpath('//a/@href').extract(): - yield Request(url, callback=self.parse) + yield scrapy.Request(url, callback=self.parse) .. module:: scrapy.contrib.spiders :synopsis: Collection of generic spiders @@ -312,10 +312,9 @@ CrawlSpider example Let's now take a look at an example CrawlSpider with rules:: + import scrapy from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor - from scrapy.selector import Selector - from scrapy.item import Item class MySpider(CrawlSpider): name = 'example.com' @@ -323,7 +322,7 @@ Let's now take a look at an example CrawlSpider with rules:: start_urls = ['http://www.example.com'] rules = ( - # Extract links matching 'category.php' (but not matching 'subsection.php') + # Extract links matching 'category.php' (but not matching 'subsection.php') # and follow links from them (since no callback means follow=True by default). Rule(SgmlLinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))), @@ -334,8 +333,8 @@ Let's now take a look at an example CrawlSpider with rules:: def parse_item(self, response): self.log('Hi, this is an item page! %s' % response.url) - sel = Selector(response) - item = Item() + sel = scrapy.Selector(response) + item = scrapy.Item() item['id'] = sel.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') item['name'] = sel.xpath('//td[@id="item_name"]/text()').extract() item['description'] = sel.xpath('//td[@id="item_description"]/text()').extract() @@ -414,7 +413,7 @@ XMLFeedSpider also returns a response (it could be the same or another one). .. method:: parse_node(response, selector) - + This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an :class:`~scrapy.selector.Selector` for each node. Overriding this @@ -424,7 +423,7 @@ XMLFeedSpider them. .. method:: process_results(response, results) - + This method is called for each result (item or request) returned by the spider, and it's intended to perform any last time processing required before returning the results to the framework core, for example setting the @@ -445,13 +444,13 @@ These spiders are pretty easy to use, let's have a look at one example:: name = 'example.com' allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.xml'] - iterator = 'iternodes' # This is actually unnecessary, since it's the default value + iterator = 'iternodes' # This is actually unnecessary, since it's the default value itertag = 'item' def parse_node(self, response, node): log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract()))) - item = Item() + item = TestItem() item['id'] = node.xpath('@id').extract() item['name'] = node.xpath('name').extract() item['description'] = node.xpath('description').extract() @@ -476,12 +475,12 @@ CSVFeedSpider Defaults to ``','`` (comma). .. attribute:: headers - + A list of the rows contained in the file CSV feed which will be used to extract fields from it. .. method:: parse_row(response, row) - + Receives a response and a dict (representing each row) with a key for each provided (or detected) header of the CSV file. This spider also gives the opportunity to override ``adapt_response`` and ``process_results`` methods @@ -566,9 +565,9 @@ SitemapSpider Specifies if alternate links for one ``url`` should be followed. These are links for the same website in another language passed within the same ``url`` block. - + For example:: - + http://example.com/ @@ -642,7 +641,7 @@ Combine SitemapSpider with other sources of urls:: def start_requests(self): requests = list(super(MySpider, self).start_requests()) - requests += [Request(x, callback=self.parse_other) for x in self.other_urls] + requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls] return requests def parse_shop(self, response):