From 9a643d689c9659c03de3630d2f379f441d2c5791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 22 Jan 2014 20:48:21 -0200
Subject: [PATCH 1/6] cleanup toplevel namespace

---
 scrapy/__init__.py                            | 29 +++++++++++--------
 ...rse_monkeypatches.py => _monkeypatches.py} |  0
 scrapy/tests/test_toplevel.py                 | 16 ++++++++++
 3 files changed, 33 insertions(+), 12 deletions(-)
 rename scrapy/{xlib/urlparse_monkeypatches.py => _monkeypatches.py} (100%)
 create mode 100644 scrapy/tests/test_toplevel.py

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 019179cfd..9917f70e8 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -1,40 +1,45 @@
 """
 Scrapy - a screen scraping framework written in Python
 """
-from __future__ import print_function
+
+__all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version']
+
+# Scrapy version
 import pkgutil
-__version__ = pkgutil.get_data(__package__, 'VERSION').strip()
-if not isinstance(__version__, str):
-    __version__ = __version__.decode('ascii')
+__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
 version_info = tuple(int(v) for v in __version__.split('.')[:3])
+del pkgutil
 
-import sys, os, warnings
-
+# Check minimum required Python version
+import sys
 if sys.version_info < (2, 7):
     print("Scrapy %s requires Python 2.7" % __version__)
     sys.exit(1)
+del sys
 
-# ignore noisy twisted deprecation warnings
+# Ignore noisy twisted deprecation warnings
+import warnings
 warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
+del warnings
 
-# monkey patches to fix external library issues
-from scrapy.xlib import urlparse_monkeypatches
+# Apply monkey patches to fix issues in external libraries
+import _monkeypatches
+del _monkeypatches
 
 # WARNING: optional_features set is deprecated and will be removed soon. Do not use.
 optional_features = set()
-
 # TODO: backwards compatibility, remove for Scrapy 0.20
 optional_features.add('ssl')
-
 try:
     import boto
+    del boto
 except ImportError:
     pass
 else:
     optional_features.add('boto')
-
 try:
     import django
+    del django
 except ImportError:
     pass
 else:
diff --git a/scrapy/xlib/urlparse_monkeypatches.py b/scrapy/_monkeypatches.py
similarity index 100%
rename from scrapy/xlib/urlparse_monkeypatches.py
rename to scrapy/_monkeypatches.py
diff --git a/scrapy/tests/test_toplevel.py b/scrapy/tests/test_toplevel.py
new file mode 100644
index 000000000..5d014ce6f
--- /dev/null
+++ b/scrapy/tests/test_toplevel.py
@@ -0,0 +1,16 @@
+from unittest import TestCase
+import six
+import scrapy
+
+
+class ToplevelTestCase(TestCase):
+
+    def test_version(self):
+        self.assertIs(type(scrapy.__version__), six.text_type)
+
+    def test_version_info(self):
+        self.assertIs(type(scrapy.version_info), tuple)
+
+    def test_optional_features(self):
+        self.assertIs(type(scrapy.optional_features), set)
+        self.assertIn('ssl', scrapy.optional_features)

From 62c7daf785ac0fc0ee75f46059655d2006b96ec0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Wed, 22 Jan 2014 20:58:53 -0200
Subject: [PATCH 2/6] Add basic top-level shortcuts

---
 scrapy/__init__.py            |  9 ++++++++-
 scrapy/tests/test_toplevel.py | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 9917f70e8..d3fd1d3ba 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -2,7 +2,8 @@
 Scrapy - a screen scraping framework written in Python
 """
 
-__all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version']
+__all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version',
+           'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
 
 # Scrapy version
 import pkgutil
@@ -49,3 +50,9 @@ from twisted import version as _txv
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
 if twisted_version >= (11, 1, 0):
     optional_features.add('http11')
+
+# Declare top-level shortcuts
+from scrapy.spider import Spider
+from scrapy.http import Request, FormRequest
+from scrapy.selector import Selector
+from scrapy.item import Item, Field
diff --git a/scrapy/tests/test_toplevel.py b/scrapy/tests/test_toplevel.py
index 5d014ce6f..17cf82213 100644
--- a/scrapy/tests/test_toplevel.py
+++ b/scrapy/tests/test_toplevel.py
@@ -14,3 +14,21 @@ class ToplevelTestCase(TestCase):
     def test_optional_features(self):
         self.assertIs(type(scrapy.optional_features), set)
         self.assertIn('ssl', scrapy.optional_features)
+
+    def test_request_shortcut(self):
+        from scrapy.http import Request, FormRequest
+        self.assertIs(scrapy.Request, Request)
+        self.assertIs(scrapy.FormRequest, FormRequest)
+
+    def test_spider_shortcut(self):
+        from scrapy.spider import Spider
+        self.assertIs(scrapy.Spider, Spider)
+
+    def test_selector_shortcut(self):
+        from scrapy.selector import Selector
+        self.assertIs(scrapy.Selector, Selector)
+
+    def test_item_shortcut(self):
+        from scrapy.item import Item, Field
+        self.assertIs(scrapy.Item, Item)
+        self.assertIs(scrapy.Field, Field)

From 4ddad88d6f8d305a220a295cd57e7d7546af07a3 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 9 Apr 2014 04:22:35 +0600
Subject: [PATCH 3/6] use "import scrapy" in templates

---
 scrapy/templates/project/module/items.py.tmpl | 6 +++---
 scrapy/templates/spiders/basic.tmpl           | 6 +++---
 scrapy/templates/spiders/crawl.tmpl           | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl
index aab8ca9b3..826c385e8 100644
--- a/scrapy/templates/project/module/items.py.tmpl
+++ b/scrapy/templates/project/module/items.py.tmpl
@@ -3,9 +3,9 @@
 # See documentation in:
 # http://doc.scrapy.org/en/latest/topics/items.html
 
-from scrapy.item import Item, Field
+import scrapy
 
-class ${ProjectName}Item(Item):
+class ${ProjectName}Item(scrapy.Item):
     # define the fields for your item here like:
-    # name = Field()
+    # name = scrapy.Field()
     pass
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 0a1fbd7fd..da5c57a39 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -1,6 +1,6 @@
-from scrapy.spider import Spider
+import scrapy
 
-class $classname(Spider):
+class $classname(scrapy.Spider):
     name = "$name"
     allowed_domains = ["$domain"]
     start_urls = (
@@ -8,4 +8,4 @@ class $classname(Spider):
         )
 
     def parse(self, response):
-        pass 
+        pass
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 82c27d7f4..25e84f940 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -1,4 +1,4 @@
-from scrapy.selector import Selector
+import scrapy
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
 from scrapy.contrib.spiders import CrawlSpider, Rule
 from $project_name.items import ${ProjectName}Item
@@ -13,7 +13,7 @@ class $classname(CrawlSpider):
     )
 
     def parse_item(self, response):
-        sel = Selector(response)
+        sel = scrapy.Selector(response)
         i = ${ProjectName}Item()
         #i['domain_id'] = sel.xpath('//input[@id="sid"]/@value').extract()
         #i['name'] = sel.xpath('//div[@id="name"]').extract()

From 2d3803672bbb4c710b00a89c49801b274414b6d0 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 9 Apr 2014 05:34:14 +0600
Subject: [PATCH 4/6] DOC use top-level shortcuts in docs

---
 docs/intro/overview.rst               |  16 +-
 docs/intro/tutorial.rst               | 205 +++++++++++++-------------
 docs/topics/commands.rst              |   4 +-
 docs/topics/debug.rst                 |  11 +-
 docs/topics/djangoitem.rst            |  69 ++++-----
 docs/topics/downloader-middleware.rst |   4 +-
 docs/topics/exporters.rst             |  24 +--
 docs/topics/firebug.rst               |   8 +-
 docs/topics/images.rst                |  14 +-
 docs/topics/items.rst                 |  24 +--
 docs/topics/jobs.rst                  |   4 +-
 docs/topics/loaders.rst               |  11 +-
 docs/topics/practices.rst             |  14 +-
 docs/topics/request-response.rst      |  21 ++-
 docs/topics/selectors.rst             |  12 +-
 docs/topics/shell.rst                 |   4 +-
 docs/topics/spider-middleware.rst     |   2 +-
 docs/topics/spiders.rst               |  63 ++++----
 18 files changed, 259 insertions(+), 251 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 5803df4e3..50d571309 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -43,13 +43,13 @@ done through :ref:`Scrapy Items <topics-items>` (Torrent files, in this case).
 
 This would be our Item::
 
-    from scrapy.item import Item, Field
+    import scrapy
 
-    class TorrentItem(Item):
-        url = Field()
-        name = Field()
-        description = Field()
-        size = Field()
+    class TorrentItem(scrapy.Item):
+        url = scrapy.Field()
+        name = scrapy.Field()
+        description = scrapy.Field()
+        size = scrapy.Field()
 
 Write a Spider to extract the data
 ==================================
@@ -129,9 +129,9 @@ For more information about XPath see the `XPath reference`_.
 
 Finally, here's the spider code::
 
+    import scrapy
     from scrapy.contrib.spiders import CrawlSpider, Rule
     from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-    from scrapy.selector import Selector
 
     class MininovaSpider(CrawlSpider):
 
@@ -141,7 +141,7 @@ Finally, here's the spider code::
         rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
 
         def parse_torrent(self, response):
-            sel = Selector(response)
+            sel = scrapy.Selector(response)
             torrent = TorrentItem()
             torrent['url'] = response.url
             torrent['name'] = sel.xpath("//h1/text()").extract()
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index b48898f0b..8d1693b28 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -36,20 +36,20 @@ Creating a project
 Before you start scraping, you will have set up a new Scrapy project. Enter a
 directory where you'd like to store your code and then run::
 
-   scrapy startproject tutorial
+    scrapy startproject tutorial
 
 This will create a ``tutorial`` directory with the following contents::
 
-   tutorial/
-       scrapy.cfg
-       tutorial/
-           __init__.py
-           items.py
-           pipelines.py
-           settings.py
-           spiders/
-               __init__.py
-               ...
+    tutorial/
+        scrapy.cfg
+        tutorial/
+            __init__.py
+            items.py
+            pipelines.py
+            settings.py
+            spiders/
+                __init__.py
+                ...
 
 These are basically:
 
@@ -68,8 +68,8 @@ Defining our Item
 like simple python dicts but provide additional protection against populating
 undeclared fields, to prevent typos.
 
-They are declared by creating a :class:`scrapy.item.Item` class and defining
-its attributes as :class:`scrapy.item.Field` objects, like you will in an ORM
+They are declared by creating a :class:`scrapy.Item <scrapy.item.Item>` class and defining
+its attributes as :class:`scrapy.Field <scrapy.item.Field>` objects, like you will in an ORM
 (don't worry if you're not familiar with ORMs, you will see that this is an
 easy task).
 
@@ -78,12 +78,12 @@ from dmoz.org, as we want to capture the name, url and description of the
 sites, we define fields for each of these three attributes. To do that, we edit
 ``items.py``, found in the ``tutorial`` directory. Our Item class looks like this::
 
-    from scrapy.item import Item, Field
+    import scrapy
 
-    class DmozItem(Item):
-        title = Field()
-        link = Field()
-        desc = Field()
+    class DmozItem(scrapy.Item):
+        title = scrapy.Field()
+        link = scrapy.Field()
+        desc = scrapy.Field()
 
 This may seem complicated at first, but defining the item allows you to use other handy
 components of Scrapy that need to know how your item looks.
@@ -97,7 +97,7 @@ of domains).
 They define an initial list of URLs to download, how to follow links, and how
 to parse the contents of those pages to extract :ref:`items <topics-items>`.
 
-To create a Spider, you must subclass :class:`scrapy.spider.Spider` and
+To create a Spider, you must subclass :class:`scrapy.Spider <scrapy.spider.Spider>` and
 define the three main mandatory attributes:
 
 * :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be
@@ -123,19 +123,20 @@ define the three main mandatory attributes:
 This is the code for our first Spider; save it in a file named
 ``dmoz_spider.py`` under the ``tutorial/spiders`` directory::
 
-   from scrapy.spider import Spider
+    import scrapy
 
-   class DmozSpider(Spider):
-       name = "dmoz"
-       allowed_domains = ["dmoz.org"]
-       start_urls = [
-           "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
-           "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
-       ]
+    class DmozSpider(scrapy.Spider):
+        name = "dmoz"
+        allowed_domains = ["dmoz.org"]
+        start_urls = [
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
+        ]
 
-       def parse(self, response):
-           filename = response.url.split("/")[-2]
-           open(filename, 'wb').write(response.body)
+        def parse(self, response):
+            filename = response.url.split("/")[-2]
+            with open(filename, 'wb') as f:
+                f.write(response.body)
 
 Crawling
 --------
@@ -170,9 +171,9 @@ created: *Books* and *Resources*, with the content of both URLs.
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Scrapy creates :class:`scrapy.http.Request` objects for each URL in the
-``start_urls`` attribute of the Spider, and assigns them the ``parse`` method of
-the spider as their callback function.
+Scrapy creates :class:`scrapy.Request <scrapy.http.Request>` objects
+for each URL in the ``start_urls`` attribute of the Spider, and assigns
+them the ``parse`` method of the spider as their callback function.
 
 These Requests are scheduled, then executed, and
 :class:`scrapy.http.Response` objects are returned and then fed back to the
@@ -243,7 +244,7 @@ installed on your system.
 
 To start a shell, you must go to the project's top level directory and run::
 
-   scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/"
+    scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/"
 
 .. note::
 
@@ -281,20 +282,20 @@ on response's type.
 
 So let's try it::
 
-   In [1]: sel.xpath('//title')
-   Out[1]: [<Selector xpath='//title' data=u'<title>Open Directory - Computers: Progr'>]
+    In [1]: sel.xpath('//title')
+    Out[1]: [<Selector xpath='//title' data=u'<title>Open Directory - Computers: Progr'>]
 
-   In [2]: sel.xpath('//title').extract()
-   Out[2]: [u'<title>Open Directory - Computers: Programming: Languages: Python: Books</title>']
+    In [2]: sel.xpath('//title').extract()
+    Out[2]: [u'<title>Open Directory - Computers: Programming: Languages: Python: Books</title>']
 
-   In [3]: sel.xpath('//title/text()')
-   Out[3]: [<Selector xpath='//title/text()' data=u'Open Directory - Computers: Programming:'>]
+    In [3]: sel.xpath('//title/text()')
+    Out[3]: [<Selector xpath='//title/text()' data=u'Open Directory - Computers: Programming:'>]
 
-   In [4]: sel.xpath('//title/text()').extract()
-   Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books']
+    In [4]: sel.xpath('//title/text()').extract()
+    Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books']
 
-   In [5]: sel.xpath('//title/text()').re('(\w+):')
-   Out[5]: [u'Computers', u'Programming', u'Languages', u'Python']
+    In [5]: sel.xpath('//title/text()').re('(\w+):')
+    Out[5]: [u'Computers', u'Programming', u'Languages', u'Python']
 
 Extracting the data
 ^^^^^^^^^^^^^^^^^^^
@@ -313,30 +314,30 @@ is inside a ``<ul>`` element, in fact the *second* ``<ul>`` element.
 So we can select each ``<li>`` element belonging to the sites list with this
 code::
 
-   sel.xpath('//ul/li')
+    sel.xpath('//ul/li')
 
 And from them, the sites descriptions::
 
-   sel.xpath('//ul/li/text()').extract()
+    sel.xpath('//ul/li/text()').extract()
 
 The sites titles::
 
-   sel.xpath('//ul/li/a/text()').extract()
+    sel.xpath('//ul/li/a/text()').extract()
 
 And the sites links::
 
-   sel.xpath('//ul/li/a/@href').extract()
+    sel.xpath('//ul/li/a/@href').extract()
 
 As we've said before, each ``.xpath()`` call returns a list of selectors, so we can
 concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use
 that property here, so::
 
-   sites = sel.xpath('//ul/li')
-   for site in sites:
-       title = site.xpath('a/text()').extract()
-       link = site.xpath('a/@href').extract()
-       desc = site.xpath('text()').extract()
-       print title, link, desc
+    sites = sel.xpath('//ul/li')
+    for site in sites:
+        title = site.xpath('a/text()').extract()
+        link = site.xpath('a/@href').extract()
+        desc = site.xpath('text()').extract()
+        print title, link, desc
 
 .. note::
 
@@ -347,55 +348,9 @@ that property here, so::
 
 Let's add this code to our spider::
 
-   from scrapy.spider import Spider
-   from scrapy.selector import Selector
+    import scrapy
 
-   class DmozSpider(Spider):
-       name = "dmoz"
-       allowed_domains = ["dmoz.org"]
-       start_urls = [
-           "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
-           "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
-       ]
-
-       def parse(self, response):
-           sel = Selector(response)
-           sites = sel.xpath('//ul/li')
-           for site in sites:
-               title = site.xpath('a/text()').extract()
-               link = site.xpath('a/@href').extract()
-               desc = site.xpath('text()').extract()
-               print title, link, desc
-
-Notice we import our Selector class from scrapy.selector and instantiate a
-new Selector object.  We can now specify our XPaths just as we did in the shell.
-Now try crawling the dmoz.org domain again and you'll see sites being printed
-in your output, run::
-
-   scrapy crawl dmoz
-
-Using our item
---------------
-
-:class:`~scrapy.item.Item` objects are custom python dicts; you can access the
-values of their fields (attributes of the class we defined earlier) using the
-standard dict syntax like::
-
-   >>> item = DmozItem()
-   >>> item['title'] = 'Example title'
-   >>> item['title']
-   'Example title'
-
-Spiders are expected to return their scraped data inside
-:class:`~scrapy.item.Item` objects. So, in order to return the data we've
-scraped so far, the final code for our Spider would be like this::
-
-    from scrapy.spider import Spider
-    from scrapy.selector import Selector
-
-    from tutorial.items import DmozItem
-
-    class DmozSpider(Spider):
+    class DmozSpider(scrapy.Spider):
         name = "dmoz"
         allowed_domains = ["dmoz.org"]
         start_urls = [
@@ -404,7 +359,51 @@ scraped so far, the final code for our Spider would be like this::
         ]
 
         def parse(self, response):
-            sel = Selector(response)
+            sel = scrapy.Selector(response)
+            sites = sel.xpath('//ul/li')
+            for site in sites:
+                title = site.xpath('a/text()').extract()
+                link = site.xpath('a/@href').extract()
+                desc = site.xpath('text()').extract()
+                print title, link, desc
+
+Notice we import our Selector class from scrapy and instantiate a new
+Selector object.  We can now specify our XPaths just as we did in the shell.
+Now try crawling the dmoz.org domain again and you'll see sites being printed
+in your output, run::
+
+    scrapy crawl dmoz
+
+Using our item
+--------------
+
+:class:`~scrapy.item.Item` objects are custom python dicts; you can access the
+values of their fields (attributes of the class we defined earlier) using the
+standard dict syntax like::
+
+    >>> item = DmozItem()
+    >>> item['title'] = 'Example title'
+    >>> item['title']
+    'Example title'
+
+Spiders are expected to return their scraped data inside
+:class:`~scrapy.item.Item` objects. So, in order to return the data we've
+scraped so far, the final code for our Spider would be like this::
+
+    import scrapy
+
+    from tutorial.items import DmozItem
+
+    class DmozSpider(scrapy.Spider):
+        name = "dmoz"
+        allowed_domains = ["dmoz.org"]
+        start_urls = [
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
+            "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
+        ]
+
+        def parse(self, response):
+            sel = scrapy.Selector(response)
             sites = sel.xpath('//ul/li')
             items = []
             for site in sites:
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 672e42e74..41ad3158f 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -190,9 +190,9 @@ Usage example::
       xmlfeed
 
     $ scrapy genspider -d basic
-    from scrapy.spider import Spider
+    import scrapy
 
-    class $classname(Spider):
+    class $classname(scrapy.Spider):
         name = "$name"
         allowed_domains = ["$domain"]
         start_urls = (
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index 47a901a6d..5739d9e1b 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -7,7 +7,10 @@ Debugging Spiders
 This document explains the most common techniques for debugging spiders.
 Consider the following scrapy spider below::
 
-    class MySpider(Spider):
+    import scrapy
+    from myproject.items import MyItem
+
+    class MySpider(scrapy.Spider):
         name = 'myspider'
         start_urls = (
             'http://example.com/page1',
@@ -17,13 +20,13 @@ Consider the following scrapy spider below::
         def parse(self, response):
             # collect `item_urls`
             for item_url in item_urls:
-                yield Request(url=item_url, callback=self.parse_item)
+                yield scrapy.Request(item_url, self.parse_item)
 
         def parse_item(self, response):
             item = MyItem()
             # populate `item` fields
-            yield Request(url=item_details_url, meta={'item': item},
-                callback=self.parse_details)
+            # and extract item_details_url
+            yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
 
         def parse_details(self, response):
             item = response.meta['item']
diff --git a/docs/topics/djangoitem.rst b/docs/topics/djangoitem.rst
index 5be75c9e1..84417dfab 100644
--- a/docs/topics/djangoitem.rst
+++ b/docs/topics/djangoitem.rst
@@ -20,59 +20,62 @@ and define its ``django_model`` attribute to be a valid Django model. With this
 you will get an item with a field for each Django model field.
 
 In addition, you can define fields that aren't present in the model and even
-override fields that are present in the model defining them in the item. 
+override fields that are present in the model defining them in the item.
 
 Let's see some examples:
 
 Creating a Django model for the examples::
-   
-   from django.db import models
 
-   class Person(models.Model):
-       name = models.CharField(max_length=255)
-       age = models.IntegerField()
+    from django.db import models
+
+    class Person(models.Model):
+        name = models.CharField(max_length=255)
+        age = models.IntegerField()
 
 Defining a basic :class:`DjangoItem`::
 
-   from scrapy.contrib.djangoitem import DjangoItem 
+    from scrapy.contrib.djangoitem import DjangoItem
+
+    class PersonItem(DjangoItem):
+        django_model = Person
 
-   class PersonItem(DjangoItem):
-       django_model = Person
-       
 :class:`DjangoItem` work just like :class:`~scrapy.item.Item`::
 
-   >>> p = PersonItem()
-   >>> p['name'] = 'John'
-   >>> p['age'] = '22'
+    >>> p = PersonItem()
+    >>> p['name'] = 'John'
+    >>> p['age'] = '22'
 
 To obtain the Django model from the item, we call the extra method
 :meth:`~DjangoItem.save` of the :class:`DjangoItem`::
 
-   >>> person = p.save()
-   >>> person.name
-   'John'
-   >>> person.age
-   '22'
-   >>> person.id
-   1
+    >>> person = p.save()
+    >>> person.name
+    'John'
+    >>> person.age
+    '22'
+    >>> person.id
+    1
 
 The model is already saved when we call :meth:`~DjangoItem.save`, we
 can prevent this by calling it with ``commit=False``. We can use
 ``commit=False`` in :meth:`~DjangoItem.save` method to obtain an unsaved model::
 
-   >>> person = p.save(commit=False)
-   >>> person.name
-   'John'
-   >>> person.age
-   '22'
-   >>> person.id
-   None
+    >>> person = p.save(commit=False)
+    >>> person.name
+    'John'
+    >>> person.age
+    '22'
+    >>> person.id
+    None
 
 As said before, we can add other fields to the item::
 
-   class PersonItem(DjangoItem):
-       django_model = Person
-       sex = Field()
+    import scrapy
+    from scrapy.contrib.djangoitem import DjangoItem
+
+    class PersonItem(DjangoItem):
+        django_model = Person
+        sex = scrapy.Field()
 
 ::
 
@@ -85,9 +88,9 @@ As said before, we can add other fields to the item::
 
 And we can override the fields of the model with your own::
 
-   class PersonItem(DjangoItem):
-       django_model = Person
-       name = Field(default='No Name')
+    class PersonItem(DjangoItem):
+        django_model = Person
+        name = scrapy.Field(default='No Name')
 
 This is useful to provide properties to the field, like a default or any other
 property that your project uses.
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 30075fa7b..da5a85655 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -198,7 +198,7 @@ There is support for keeping multiple cookie sessions per spider by using the
 For example::
 
     for i, url in enumerate(urls):
-        yield Request("http://www.example.com", meta={'cookiejar': i},
+        yield scrapy.Request("http://www.example.com", meta={'cookiejar': i},
             callback=self.parse_page)
 
 Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep
@@ -206,7 +206,7 @@ passing it along on subsequent requests. For example::
 
     def parse_page(self, response):
         # do some processing
-        return Request("http://www.example.com/otherpage",
+        return scrapy.Request("http://www.example.com/otherpage",
             meta={'cookiejar': response.meta['cookiejar']},
             callback=self.parse_other_page)
 
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 9353448f2..c472f5b96 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -96,14 +96,14 @@ value and returns its serialized form.
 
 Example::
 
-      from scrapy.item import Item, Field
+    import scrapy
 
-      def serialize_price(value):
-         return '$ %s' % str(value)
+    def serialize_price(value):
+        return '$ %s' % str(value)
 
-      class Product(Item):
-          name = Field()
-          price = Field(serializer=serialize_price)
+    class Product(scrapy.Item):
+        name = scrapy.Field()
+        price = scrapy.Field(serializer=serialize_price)
 
 
 2. Overriding the serialize_field() method
@@ -113,7 +113,7 @@ You can also override the :meth:`~BaseItemExporter.serialize_field()` method to
 customize how your field value will be exported.
 
 Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method
-after your custom code. 
+after your custom code.
 
 Example::
 
@@ -125,7 +125,7 @@ Example::
               if field == 'price':
                   return '$ %s' % str(value)
               return super(Product, self).serialize_field(field, name, value)
-             
+
 .. _topics-exporters-reference:
 
 Built-in Item Exporters reference
@@ -146,7 +146,7 @@ BaseItemExporter
    support for common features used by all (concrete) Item Exporters, such as
    defining what fields to export, whether to export empty fields, or which
    encoding to use.
-   
+
    These features can be configured through the constructor arguments which
    populate their respective instance attributes: :attr:`fields_to_export`,
    :attr:`export_empty_fields`, :attr:`encoding`.
@@ -278,7 +278,7 @@ CsvItemExporter
    :param file: the file-like object to use for exporting the data.
 
    :param include_headers_line: If enabled, makes the exporter output a header
-      line with the field names taken from 
+      line with the field names taken from
       :attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
    :type include_headers_line: boolean
 
@@ -296,7 +296,7 @@ CsvItemExporter
       product,price
       Color TV,1200
       DVD player,200
-      
+
 .. _csv.writer: http://docs.python.org/library/csv.html#csv.writer
 
 PickleItemExporter
@@ -304,7 +304,7 @@ PickleItemExporter
 
 .. class:: PickleItemExporter(file, protocol=0, \**kwargs)
 
-   Exports Items in pickle format to the given file-like object. 
+   Exports Items in pickle format to the given file-like object.
 
    :param file: the file-like object to use for exporting the data.
 
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index 5aa03a65e..8b0d5a14c 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -61,7 +61,7 @@ Getting links to follow
 By looking at the category URLs we can see they share a pattern:
 
     http://directory.google.com/Category/Subcategory/Another_Subcategory
-    
+
 Once we know that, we are able to construct a regular expression to follow
 those links. For example, the following one::
 
@@ -103,7 +103,7 @@ This is how the spider would look so far::
 Extracting the data
 ===================
 
-Now we're going to write the code to extract data from those pages. 
+Now we're going to write the code to extract data from those pages.
 
 With the help of Firebug, we'll take a look at some page containing links to
 websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can
@@ -146,7 +146,7 @@ that have that grey colour of the links,
 Finally, we can write our ``parse_category()`` method::
 
     def parse_category(self, response):
-        sel = Selector(response)
+        sel = scrapy.Selector(response)
 
         # The path to website links in directory page
         links = sel.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
@@ -164,6 +164,6 @@ not in the original HTML, such as the typical case of ``<tbody>``
 elements.
 
 or tags which Therefer   in page HTML
-sources may on Firebug inspects the live DOM 
+sources may on Firebug inspects the live DOM
 
 .. _has been shut down by Google: http://searchenginewatch.com/article/2096661/Google-Directory-Has-Been-Shut-Down
diff --git a/docs/topics/images.rst b/docs/topics/images.rst
index 28be43e03..91e59d2a9 100644
--- a/docs/topics/images.rst
+++ b/docs/topics/images.rst
@@ -67,13 +67,13 @@ In order to use the image pipeline you just need to :ref:`enable it
 <topics-images-enabling>` and define an item with the ``image_urls`` and
 ``images`` fields::
 
-    from scrapy.item import Item
+    import scrapy
 
-    class MyItem(Item):
+    class MyItem(scrapy.Item):
 
         # ... other item fields ...
-        image_urls = Field()
-        images = Field()
+        image_urls = scrapy.Field()
+        images = scrapy.Field()
 
 If you need something more complex and want to override the custom images
 pipeline behaviour, see :ref:`topics-images-override`.
@@ -228,7 +228,7 @@ Here are the methods that you should override in your custom Images Pipeline:
 
          def get_media_requests(self, item, info):
              for image_url in item['image_urls']:
-                 yield Request(image_url)
+                 yield scrapy.Request(image_url)
 
       Those requests will be processed by the pipeline and, when they have finished
       downloading, the results will be sent to the
@@ -302,15 +302,15 @@ Custom Images pipeline example
 Here is a full example of the Images Pipeline whose methods are examplified
 above::
 
+    import scrapy
     from scrapy.contrib.pipeline.images import ImagesPipeline
     from scrapy.exceptions import DropItem
-    from scrapy.http import Request
 
     class MyImagesPipeline(ImagesPipeline):
 
         def get_media_requests(self, item, info):
             for image_url in item['image_urls']:
-                yield Request(image_url)
+                yield scrapy.Request(image_url)
 
         def item_completed(self, results, item, info):
             image_paths = [x['path'] for ok, x in results if ok]
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index bf7ca74d1..ee604a7f1 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -25,13 +25,13 @@ Declaring Items
 Items are declared using a simple class definition syntax and :class:`Field`
 objects. Here is an example::
 
-    from scrapy.item import Item, Field
+    import scrapy
 
-    class Product(Item):
-        name = Field()
-        price = Field()
-        stock = Field()
-        last_updated = Field(serializer=str)
+    class Product(scrapy.Item):
+        name = scrapy.Field()
+        price = scrapy.Field()
+        stock = scrapy.Field()
+        last_updated = scrapy.Field(serializer=str)
 
 .. note:: Those familiar with `Django`_ will notice that Scrapy Items are
    declared similar to `Django Models`_, except that Scrapy Items are much
@@ -185,14 +185,14 @@ fields) by declaring a subclass of your original Item.
 For example::
 
     class DiscountedProduct(Product):
-        discount_percent = Field(serializer=str)
-        discount_expiration_date = Field()
+        discount_percent = scrapy.Field(serializer=str)
+        discount_expiration_date = scrapy.Field()
 
 You can also extend field metadata by using the previous field metadata and
 appending more values, or changing existing values, like this::
 
     class SpecificProduct(Product):
-        name = Field(Product.fields['name'], serializer=my_serializer)
+        name = scrapy.Field(Product.fields['name'], serializer=my_serializer)
 
 That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
 keeping all the previously existing metadata values.
@@ -202,11 +202,11 @@ Item objects
 
 .. class:: Item([arg])
 
-    Return a new Item optionally initialized from the given argument. 
-    
+    Return a new Item optionally initialized from the given argument.
+
     Items replicate the standard `dict API`_, including its constructor. The
     only additional attribute provided by Items is:
-    
+
     .. attribute:: fields
 
         A dictionary containing *all declared fields* for this Item, not only
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 56ea6a8c3..4e09000d2 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -81,7 +81,7 @@ So, for example, this won't work::
 
     def some_callback(self, response):
         somearg = 'test'
-        return Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
+        return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
         print "the argument passed is:", somearg
@@ -90,7 +90,7 @@ But this will::
 
     def some_callback(self, response):
         somearg = 'test'
-        return Request('http://www.example.com', meta={'somearg': somearg})
+        return scrapy.Request('http://www.example.com', meta={'somearg': somearg})
 
     def other_callback(self, response):
         somearg = response.meta['somearg']
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index b9db5daa8..4ac514be6 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -181,18 +181,17 @@ this way. However, there is one more place where you can specify the input and
 output processors to use: in the :ref:`Item Field <topics-items-fields>`
 metadata. Here is an example::
 
-    from scrapy.item import Item, Field
+    import scrapy
     from scrapy.contrib.loader.processor import MapCompose, Join, TakeFirst
-
-    from scrapy.utils.markup import remove_entities
+    from w3lib.html import remove_entities
     from myproject.utils import filter_prices
 
-    class Product(Item):
-        name = Field(
+    class Product(scrapy.Item):
+        name = scrapy.Field(
             input_processor=MapCompose(remove_entities),
             output_processor=Join(),
         )
-        price = Field(
+        price = scrapy.Field(
             default=0,
             input_processor=MapCompose(remove_entities, filter_prices),
             output_processor=TakeFirst(),
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index c5427c899..64b3a2da7 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -149,18 +149,18 @@ If you are still unable to prevent your bot getting banned, consider contacting
 Dynamic Creation of Item Classes
 ================================
 
-For applications in which the structure of item class is to be determined by 
+For applications in which the structure of item class is to be determined by
 user input, or other changing conditions, you can dynamically create item
 classes instead of manually coding them.
 
 ::
 
 
-	from scrapy.item import DictItem, Field
+    from scrapy.item import DictItem, Field
 
-	def create_item_class(class_name, field_list):
-	    field_dict = {}
-	    for field_name in field_list:
-	        field_dict[field_name] = Field()
+    def create_item_class(class_name, field_list):
+        field_dict = {}
+        for field_name in field_list:
+            field_dict[field_name] = Field()
 
-	    return type(class_name, (DictItem,), field_dict)
+        return type(class_name, (DictItem,), field_dict)
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b53c21ce7..cda074a71 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -183,8 +183,8 @@ downloaded :class:`Response` object as its first argument.
 Example::
 
     def parse_page1(self, response):
-        return Request("http://www.example.com/some_page.html",
-                          callback=self.parse_page2)
+        return scrapy.Request("http://www.example.com/some_page.html",
+                              callback=self.parse_page2)
 
     def parse_page2(self, response):
         # this would log http://www.example.com/some_page.html
@@ -200,8 +200,8 @@ different fields from different pages::
     def parse_page1(self, response):
         item = MyItem()
         item['main_url'] = response.url
-        request = Request("http://www.example.com/some_page.html",
-                          callback=self.parse_page2)
+        request = scrapy.Request("http://www.example.com/some_page.html",
+                                 callback=self.parse_page2)
         request.meta['item'] = item
         return request
 
@@ -349,14 +349,19 @@ automatically pre-populated and only override a couple of them, such as the
 user name and password. You can use the :meth:`FormRequest.from_response`
 method for this job. Here's an example spider which uses it::
 
-    class LoginSpider(Spider):
+
+    import scrapy
+
+    class LoginSpider(scrapy.Spider):
         name = 'example.com'
         start_urls = ['http://www.example.com/users/login.php']
 
         def parse(self, response):
-            return [FormRequest.from_response(response,
-                        formdata={'username': 'john', 'password': 'secret'},
-                        callback=self.after_login)]
+            return scrapy.FormRequest.from_response(
+                response,
+                formdata={'username': 'john', 'password': 'secret'},
+                callback=self.after_login
+            )
 
         def after_login(self, response):
             # check login succeed before going on
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8d55fe7cb..0a96331d4 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -53,16 +53,15 @@ Constructing selectors
 .. highlight:: python
 
 Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
-constructed by passing a `Response` object as first argument, the response's
-body is what they're going to be "selecting"::
+constructed by passing a :class:`~scrapy.http.Response` object as first
+argument, the response's body is what they're going to be "selecting"::
 
-    from scrapy.spider import Spider
-    from scrapy.selector import Selector
+    import scrapy
 
-    class MySpider(Spider):
+    class MySpider(scrapy.Spider):
         # ...
         def parse(self, response):
-            sel = Selector(response)
+            sel = scrapy.Selector(response)
             # Using XPath query
             print sel.xpath('//p')
             # Using CSS query
@@ -262,6 +261,7 @@ The ``test()`` function for example can prove quite useful when XPath's
 
 Example selecting links in list item with a "class" attribute ending with a digit::
 
+    >>> from scrapy import Selector
     >>> doc = """
     ... <div>
     ...     <ul>
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 990dedc6c..8827f7fb6 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -171,10 +171,10 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function.
 
 Here's an example of how you would call it from your spider::
 
-    from scrapy.spider import Spider
+    import scrapy
 
 
-    class MySpider(Spider):
+    class MySpider(scrapy.Spider):
         name = "myspider"
         start_urls = [
             "http://example.com",
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index fc7cf734a..3df59998b 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -129,7 +129,7 @@ single Python class that defines one or more of the following methods:
         :type exception: `Exception`_ object
 
         :param spider: the spider which raised the exception
-        :type spider: :class:`scrapy.spider.Spider` object
+        :type spider: :class:`~scrapy.spider.Spider` object
 
     .. method:: process_start_requests(start_requests, spider)
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 399aa7e6a..929e21e10 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -58,7 +58,9 @@ Spider arguments are passed through the :command:`crawl` command using the
 
 Spiders receive arguments in their constructors::
 
-    class MySpider(Spider):
+    import scrapy
+
+    class MySpider(scrapy.Spider):
         name = 'myspider'
 
         def __init__(self, category=None, *args, **kwargs):
@@ -82,12 +84,12 @@ rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed.
 For the examples used in the following spiders, we'll assume you have a project
 with a ``TestItem`` declared in a ``myproject.items`` module::
 
-    from scrapy.item import Item
+    import scrapy
 
-    class TestItem(Item):
-        id = Field()
-        name = Field()
-        description = Field()
+    class TestItem(scrapy.Item):
+        id = scrapy.Field()
+        name = scrapy.Field()
+        description = scrapy.Field()
 
 
 .. module:: scrapy.spider
@@ -150,9 +152,9 @@ Spider
        a POST request, you could do::
 
            def start_requests(self):
-               return [FormRequest("http://www.example.com/login",
-                                   formdata={'user': 'john', 'pass': 'secret'},
-                                   callback=self.logged_in)]
+               return [scrapy.FormRequest("http://www.example.com/login",
+                                          formdata={'user': 'john', 'pass': 'secret'},
+                                          callback=self.logged_in)]
 
            def logged_in(self, response):
                # here you would extract links to follow and return Requests for
@@ -199,10 +201,10 @@ Spider example
 
 Let's see an example::
 
-    from scrapy import log # This module is useful for printing out debug information
-    from scrapy.spider import Spider
+    import scrapy
 
-    class MySpider(Spider):
+
+    class MySpider(scrapy.Spider):
         name = 'example.com'
         allowed_domains = ['example.com']
         start_urls = [
@@ -216,12 +218,10 @@ Let's see an example::
 
 Another example returning multiple Requests and Items from a single callback::
 
-    from scrapy.selector import Selector
-    from scrapy.spider import Spider
-    from scrapy.http import Request
+    import scrapy
     from myproject.items import MyItem
 
-    class MySpider(Spider):
+    class MySpider(scrapy.Spider):
         name = 'example.com'
         allowed_domains = ['example.com']
         start_urls = [
@@ -231,12 +231,12 @@ Another example returning multiple Requests and Items from a single callback::
         ]
 
         def parse(self, response):
-            sel = Selector(response)
+            sel = scrapy.Selector(response)
             for h3 in sel.xpath('//h3').extract():
                 yield MyItem(title=h3)
 
             for url in sel.xpath('//a/@href').extract():
-                yield Request(url, callback=self.parse)
+                yield scrapy.Request(url, callback=self.parse)
 
 .. module:: scrapy.contrib.spiders
    :synopsis: Collection of generic spiders
@@ -312,10 +312,9 @@ CrawlSpider example
 
 Let's now take a look at an example CrawlSpider with rules::
 
+    import scrapy
     from scrapy.contrib.spiders import CrawlSpider, Rule
     from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-    from scrapy.selector import Selector
-    from scrapy.item import Item
 
     class MySpider(CrawlSpider):
         name = 'example.com'
@@ -323,7 +322,7 @@ Let's now take a look at an example CrawlSpider with rules::
         start_urls = ['http://www.example.com']
 
         rules = (
-            # Extract links matching 'category.php' (but not matching 'subsection.php') 
+            # Extract links matching 'category.php' (but not matching 'subsection.php')
             # and follow links from them (since no callback means follow=True by default).
             Rule(SgmlLinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
 
@@ -334,8 +333,8 @@ Let's now take a look at an example CrawlSpider with rules::
         def parse_item(self, response):
             self.log('Hi, this is an item page! %s' % response.url)
 
-            sel = Selector(response)
-            item = Item()
+            sel = scrapy.Selector(response)
+            item = scrapy.Item()
             item['id'] = sel.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
             item['name'] = sel.xpath('//td[@id="item_name"]/text()').extract()
             item['description'] = sel.xpath('//td[@id="item_description"]/text()').extract()
@@ -414,7 +413,7 @@ XMLFeedSpider
         also returns a response (it could be the same or another one).
 
     .. method:: parse_node(response, selector)
-       
+
         This method is called for the nodes matching the provided tag name
         (``itertag``).  Receives the response and an
         :class:`~scrapy.selector.Selector` for each node.  Overriding this
@@ -424,7 +423,7 @@ XMLFeedSpider
         them.
 
     .. method:: process_results(response, results)
-       
+
         This method is called for each result (item or request) returned by the
         spider, and it's intended to perform any last time processing required
         before returning the results to the framework core, for example setting the
@@ -445,13 +444,13 @@ These spiders are pretty easy to use, let's have a look at one example::
         name = 'example.com'
         allowed_domains = ['example.com']
         start_urls = ['http://www.example.com/feed.xml']
-        iterator = 'iternodes' # This is actually unnecessary, since it's the default value
+        iterator = 'iternodes'  # This is actually unnecessary, since it's the default value
         itertag = 'item'
 
         def parse_node(self, response, node):
             log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract())))
 
-            item = Item()
+            item = TestItem()
             item['id'] = node.xpath('@id').extract()
             item['name'] = node.xpath('name').extract()
             item['description'] = node.xpath('description').extract()
@@ -476,12 +475,12 @@ CSVFeedSpider
        Defaults to ``','`` (comma).
 
    .. attribute:: headers
-      
+
        A list of the rows contained in the file CSV feed which will be used to
        extract fields from it.
 
    .. method:: parse_row(response, row)
-      
+
        Receives a response and a dict (representing each row) with a key for each
        provided (or detected) header of the CSV file.  This spider also gives the
        opportunity to override ``adapt_response`` and ``process_results`` methods
@@ -566,9 +565,9 @@ SitemapSpider
         Specifies if alternate links for one ``url`` should be followed. These
         are links for the same website in another language passed within
         the same ``url`` block.
-        
+
         For example::
-       
+
             <url>
                 <loc>http://example.com/</loc>
                 <xhtml:link rel="alternate" hreflang="de" href="http://example.com/de"/>
@@ -642,7 +641,7 @@ Combine SitemapSpider with other sources of urls::
 
         def start_requests(self):
             requests = list(super(MySpider, self).start_requests())
-            requests += [Request(x, callback=self.parse_other) for x in self.other_urls]
+            requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls]
             return requests
 
         def parse_shop(self, response):

From 11e62117b745d4e489b0533873be48d09d19136a Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 10 Apr 2014 03:32:00 +0600
Subject: [PATCH 5/6] suggest scrapy.Selector in deprecation warnings

---
 scrapy/selector/lxmlsel.py    |  8 ++++----
 scrapy/tests/test_selector.py | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py
index 0234703cb..070cb2306 100644
--- a/scrapy/selector/lxmlsel.py
+++ b/scrapy/selector/lxmlsel.py
@@ -10,7 +10,7 @@ __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector',
 
 def _xpathselector_css(self, *a, **kw):
     raise RuntimeError('.css() method not available for %s, '
-                        'instantiate scrapy.selector.Selector '
+                        'instantiate scrapy.Selector '
                         'instead' % type(self).__name__)
 
 XPathSelector = create_deprecated_class(
@@ -21,7 +21,7 @@ XPathSelector = create_deprecated_class(
         '_default_type': 'html',
         'css': _xpathselector_css,
     },
-    new_class_path='scrapy.selector.Selector',
+    new_class_path='scrapy.Selector',
     old_class_path='scrapy.selector.XPathSelector',
 )
 
@@ -32,7 +32,7 @@ XmlXPathSelector = create_deprecated_class(
         '__slots__': (),
         '_default_type': 'xml',
     },
-    new_class_path='scrapy.selector.Selector',
+    new_class_path='scrapy.Selector',
     old_class_path='scrapy.selector.XmlXPathSelector',
 )
 
@@ -43,7 +43,7 @@ HtmlXPathSelector = create_deprecated_class(
         '__slots__': (),
         '_default_type': 'html',
     },
-    new_class_path='scrapy.selector.Selector',
+    new_class_path='scrapy.Selector',
     old_class_path='scrapy.selector.HtmlXPathSelector',
 )
 
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index 077ba2720..d60521cbc 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -354,7 +354,7 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
 
             # subclassing must issue a warning
             self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.selector.Selector', str(w[0].message))
+            self.assertIn('scrapy.Selector', str(w[0].message))
 
             # subclass instance doesn't issue a warning
             usel = UserClass(text=self.text)
@@ -363,7 +363,7 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             # class instance must issue a warning
             sel = cls(text=self.text)
             self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.selector.Selector', str(w[1].message))
+            self.assertIn('scrapy.Selector', str(w[1].message))
 
             # subclass and instance checks
             self.assertTrue(issubclass(cls, Selector))
@@ -378,7 +378,7 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
 
             # subclassing must issue a warning
             self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.selector.Selector', str(w[0].message))
+            self.assertIn('scrapy.Selector', str(w[0].message))
 
             # subclass instance doesn't issue a warning
             usel = UserClass(text=self.text)
@@ -387,7 +387,7 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             # class instance must issue a warning
             sel = cls(text=self.text)
             self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.selector.Selector', str(w[1].message))
+            self.assertIn('scrapy.Selector', str(w[1].message))
 
             # subclass and instance checks
             self.assertTrue(issubclass(cls, Selector))
@@ -405,7 +405,7 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
 
             # subclassing must issue a warning
             self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.selector.Selector', str(w[0].message))
+            self.assertIn('scrapy.Selector', str(w[0].message))
 
             # subclass instance doesn't issue a warning
             usel = UserClass(text=self.text)
@@ -414,7 +414,7 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             # class instance must issue a warning
             sel = cls(text=self.text)
             self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.selector.Selector', str(w[1].message))
+            self.assertIn('scrapy.Selector', str(w[1].message))
 
             # subclass and instance checks
             self.assertTrue(issubclass(cls, Selector))

From c3426e952bb72f715ee9d7c8aaf4aa8198b3f624 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 15 Apr 2014 06:54:14 +0600
Subject: [PATCH 6/6] TST fix tests that became broken after adding top-level
 imports and switching to py.test.

py.test replaces AssertionError exception with its own subclass in an import hook.
For some reason when top-level shortcuts were added "except AssertionError" in
contracts.__init__ meant "except py.test AssertionError" while scrapy.exceptions.ContractFail
was still a subclass of a builtin AssertionError, so Contracts reported errors
instead of failures and this made tests in test_contracts.py fail.

For now - just disable fancy py.test asserts.
---
 pytest.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index 8d3bc91a5..f034bf233 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -2,4 +2,4 @@
 [pytest]
 usefixtures = chdir setlog
 python_files=test_*.py __init__.py
-addopts = --doctest-modules
+addopts = --doctest-modules --assert=plain