From 6b1760d7a1d8949f14a43aa7ce00726e084b2abe Mon Sep 17 00:00:00 2001
From: Rolando Espinoza La fuente <darkrho@gmail.com>
Date: Sun, 27 Oct 2013 18:28:23 -0400
Subject: [PATCH 1/7] replaced __import__ by importlib.import_module.

Since python 2.7, importlib.import_module is the recommended way to
import modules programmatically.

From __import__'s doc:

    Import a module. Because this function is meant for use by the
Python
    interpreter and not for general use it is better to use
    importlib.import_module() to programmatically import a module.
---
 scrapy/commands/genspider.py      | 4 +++-
 scrapy/commands/runspider.py      | 3 ++-
 scrapy/contrib/httpcache.py       | 3 ++-
 scrapy/tests/test_dependencies.py | 2 +-
 scrapy/utils/misc.py              | 8 +++++---
 scrapy/utils/project.py           | 8 +++++---
 6 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index f8a5e2310..21b5ba3ea 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -2,6 +2,8 @@ from __future__ import print_function
 import os
 import shutil
 import string
+
+from importlib import import_module
 from os.path import join, dirname, abspath, exists, splitext
 
 import scrapy
@@ -90,7 +92,7 @@ class Command(ScrapyCommand):
             'classname': '%sSpider' % ''.join([s.capitalize() \
                 for s in module.split('_')])
         }
-        spiders_module = __import__(self.settings['NEWSPIDER_MODULE'], {}, {}, [''])
+        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
         spiders_dir = abspath(dirname(spiders_module.__file__))
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 3fe439b11..14d67e24b 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,5 +1,6 @@
 import sys
 import os
+from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.command import ScrapyCommand
@@ -15,7 +16,7 @@ def _import_file(filepath):
     if dirname:
         sys.path = [dirname] + sys.path
     try:
-        module = __import__(fname, {}, {}, [''])
+        module = import_module(fname)
     finally:
         if dirname:
             sys.path.pop(0)
diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py
index b473c2dc3..5dee898f0 100644
--- a/scrapy/contrib/httpcache.py
+++ b/scrapy/contrib/httpcache.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import cPickle as pickle
+from importlib import import_module
 from time import time
 from weakref import WeakKeyDictionary
 from email.utils import mktime_tz, parsedate_tz
@@ -164,7 +165,7 @@ class DbmCacheStorage(object):
     def __init__(self, settings):
         self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], {}, {}, [''])
+        self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
         self.db = None
 
     def open_spider(self, spider):
diff --git a/scrapy/tests/test_dependencies.py b/scrapy/tests/test_dependencies.py
index 794db0fe6..51b9c3af3 100644
--- a/scrapy/tests/test_dependencies.py
+++ b/scrapy/tests/test_dependencies.py
@@ -3,7 +3,7 @@ from twisted.trial import unittest
 class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
-            module = __import__('OpenSSL', {}, {}, [''])
+            module = __import__('OpenSSL')
         except ImportError as ex:
             raise unittest.SkipTest("OpenSSL is not available")
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 018fe031f..a16d42eae 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -2,6 +2,8 @@
 
 import re
 import hashlib
+
+from importlib import import_module
 from pkgutil import iter_modules
 
 from w3lib.html import remove_entities
@@ -35,7 +37,7 @@ def load_object(path):
 
     module, name = path[:dot], path[dot+1:]
     try:
-        mod = __import__(module, {}, {}, [''])
+        mod = import_module(module)
     except ImportError as e:
         raise ImportError("Error loading object '%s': %s" % (path, e))
 
@@ -55,7 +57,7 @@ def walk_modules(path, load=False):
     """
 
     mods = []
-    mod = __import__(path, {}, {}, [''])
+    mod = import_module(path)
     mods.append(mod)
     if hasattr(mod, '__path__'):
         for _, subpath, ispkg in iter_modules(mod.__path__):
@@ -63,7 +65,7 @@ def walk_modules(path, load=False):
             if ispkg:
                 mods += walk_modules(fullpath)
             else:
-                submod = __import__(fullpath, {}, {}, [''])
+                submod = import_module(fullpath)
                 mods.append(submod)
     return mods
 
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index a2edb7b08..332fc18b8 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,8 +1,10 @@
 import os
-from os.path import join, dirname, abspath, isabs, exists
 import cPickle as pickle
 import warnings
 
+from importlib import import_module
+from os.path import join, dirname, abspath, isabs, exists
+
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import CrawlerSettings
 from scrapy.exceptions import NotConfigured
@@ -14,7 +16,7 @@ def inside_project():
     scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
     if scrapy_module is not None:
         try:
-            __import__(scrapy_module)
+            import_module(scrapy_module)
         except ImportError as exc:
             warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc))
         else:
@@ -53,7 +55,7 @@ def get_project_settings():
         init_env(project)
     settings_module_path = os.environ.get(ENVVAR)
     if settings_module_path:
-        settings_module = __import__(settings_module_path, {}, {}, [''])
+        settings_module = import_module(settings_module_path)
     else:
         settings_module = None
     settings = CrawlerSettings(settings_module)

From 6f5423aebd1d985712200593cca2f2eb134e144a Mon Sep 17 00:00:00 2001
From: Rolando Espinoza La fuente <darkrho@gmail.com>
Date: Sun, 27 Oct 2013 19:10:25 -0400
Subject: [PATCH 2/7] Replaced remaning __import__(module) calls.

This commit replaces the statements __import__(module) as the previous
replaced the statements __import__(module, {}, {}, ['']).

At first I thought leaving the single-argument calls, but perhaps it's
better to be strict rather than having exceptions to the rule in this
case.
---
 scrapy/contrib/memusage.py          | 4 +++-
 scrapy/settings/default_settings.py | 3 ++-
 scrapy/tests/test_dependencies.py   | 3 ++-
 scrapy/utils/test.py                | 3 ++-
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
index 4d4aae1c9..98dd15027 100644
--- a/scrapy/contrib/memusage.py
+++ b/scrapy/contrib/memusage.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
 
 import socket
 from pprint import pformat
+from importlib import import_module
 
 from twisted.internet import task
 
@@ -20,7 +21,8 @@ class MemoryUsage(object):
         if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
             raise NotConfigured
         try:
-            self.resource = __import__('resource')
+            # stdlib's resource module is only availabe on unix platforms.
+            self.resource = import_module('resource')
         except ImportError:
             raise NotConfigured
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index e7475c3d5..91b8d8b48 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -15,6 +15,7 @@ Scrapy developers, if you add a setting here remember to:
 
 import os
 import sys
+from importlib import import_module
 from os.path import join, abspath, dirname
 
 BOT_NAME = 'scrapybot'
@@ -229,7 +230,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
 
 URLLENGTH_LIMIT = 2083
 
-USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % __import__('scrapy').__version__
+USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__
 
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]
diff --git a/scrapy/tests/test_dependencies.py b/scrapy/tests/test_dependencies.py
index 51b9c3af3..03bf2ffcf 100644
--- a/scrapy/tests/test_dependencies.py
+++ b/scrapy/tests/test_dependencies.py
@@ -1,9 +1,10 @@
+from importlib import import_module
 from twisted.trial import unittest
 
 class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
-            module = __import__('OpenSSL')
+            module = import_module('OpenSSL')
         except ImportError as ex:
             raise unittest.SkipTest("OpenSSL is not available")
 
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index c2d473e6f..ec9ca1c51 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -4,6 +4,7 @@ This module contains some assorted functions used in tests
 
 import os
 
+from importlib import import_module
 from twisted.trial.unittest import SkipTest
 
 
@@ -39,7 +40,7 @@ def get_crawler(settings_dict=None):
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this
     installation of Scrapy"""
-    scrapy_path = __import__('scrapy').__path__[0]
+    scrapy_path = import_module('scrapy').__path__[0]
     return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
 
 def get_testenv():

From bc7fa61136cfe51792c093416e4f18e7c7def86c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 7 Nov 2013 02:32:00 -0200
Subject: [PATCH 3/7] Django 1.6 form validation errors now include
 ValidationError exception instances instead of just strings

---
 requirements.txt                         | 1 +
 scrapy/tests/test_djangoitem/__init__.py | 8 +-------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 5a767c91b..e79a05122 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ w3lib>=1.2
 queuelib
 lxml
 pyOpenSSL
+cssselect>=0.9
diff --git a/scrapy/tests/test_djangoitem/__init__.py b/scrapy/tests/test_djangoitem/__init__.py
index 4f6c820be..cff97fc98 100644
--- a/scrapy/tests/test_djangoitem/__init__.py
+++ b/scrapy/tests/test_djangoitem/__init__.py
@@ -75,13 +75,7 @@ class DjangoItemTest(unittest.TestCase):
         long_name = 'z' * 300
         i = BasePersonItem(name=long_name)
         self.assertFalse(i.is_valid())
-        self.assertEqual(
-            {
-                'age': [u'This field cannot be null.'],
-                'name': [u'Ensure this value has at most 255 characters (it has 300).']
-            },
-            i.errors)
-
+        self.assertEqual(set(i.errors), set(['age', 'name']))
         i = BasePersonItem(name='John')
         self.assertTrue(i.is_valid(exclude=['age']))
         self.assertEqual({}, i.errors)

From fabb351097a028c3517040529a654e34f7269db3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 7 Nov 2013 04:02:46 -0200
Subject: [PATCH 4/7] map travis-ci matrix to tox environments

---
 .travis.yml                      | 29 ++++++++++++-----------------
 .travis/requirements-latest.txt  |  9 ---------
 .travis/requirements-precise.txt | 10 ----------
 .travis/requirements-trunk.txt   |  3 ---
 requirements.txt                 |  4 ++--
 tox.ini                          | 29 ++++++++++++++++++++++-------
 6 files changed, 36 insertions(+), 48 deletions(-)
 delete mode 100644 .travis/requirements-latest.txt
 delete mode 100644 .travis/requirements-precise.txt
 delete mode 100644 .travis/requirements-trunk.txt

diff --git a/.travis.yml b/.travis.yml
index 58db457d8..84250a75f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,26 +1,21 @@
 language: python
+python: 2.7
 env:
-    TRAVISBUG="#1027"
+    - TOX_ENV=py27
+    - TOX_ENV=precise
+    - TOX_ENV=trunk
+    - TOX_ENV=pypy
+
 matrix:
-    exclude:
-        - env: TRAVISBUG="#1027"
-    include:
-        - python: "2.7"
-          env: BUILDENV=precise
-        - python: "2.7"
-          env: BUILDENV=latest
-        - python: "2.7"
-          env: BUILDENV=trunk
-        - python: "pypy"
-          env: BUILDENV=latest
     allow_failures:
-        - python: "pypy"
-          env: BUILDENV=latest
+        - env: TOX_ENV=pypy
+
 install:
-    - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt
-    - pip install --use-mirrors .
+    - pip install -M tox
+
 script:
-    - trial scrapy
+    - tox -e $TOX_ENV
+
 notifications:
     irc:
         channels:
diff --git a/.travis/requirements-latest.txt b/.travis/requirements-latest.txt
deleted file mode 100644
index 58bd132ef..000000000
--- a/.travis/requirements-latest.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-pyOpenSSL
-lxml
-twisted
-boto
-Pillow
-django
-
-# testing requirements
-mock
diff --git a/.travis/requirements-precise.txt b/.travis/requirements-precise.txt
deleted file mode 100644
index 918dfdc45..000000000
--- a/.travis/requirements-precise.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# travis workers uses openssl 1.0 that fails for pyOpenSSL < 0.13
-pyOpenSSL==0.13
-lxml==2.3.2
-twisted==11.1.0
-boto==2.2.2
-Pillow<2.0
-django==1.3.1
-
-# testing requirements
-mock==1.0.1
diff --git a/.travis/requirements-trunk.txt b/.travis/requirements-trunk.txt
deleted file mode 100644
index 482cd91a3..000000000
--- a/.travis/requirements-trunk.txt
+++ /dev/null
@@ -1,3 +0,0 @@
--r requirements-latest.txt
-git+https://github.com/scrapy/w3lib#egg=w3lib
-git+https://github.com/scrapy/queuelib#egg=queuelib
diff --git a/requirements.txt b/requirements.txt
index e79a05122..e070a183e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 Twisted>=10.0.0
-w3lib>=1.2
-queuelib
 lxml
 pyOpenSSL
 cssselect>=0.9
+w3lib>=1.2
+queuelib
diff --git a/tox.ini b/tox.ini
index fc2327142..56c992aac 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,25 +4,40 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, precise, trunk
+envlist = py27, pypy, precise, trunk
 
 [testenv]
 deps =
-    -r{toxinidir}/.travis/requirements-latest.txt
+    -rrequirements.txt
+    # Extras
+    boto
+    Pillow
+    django
+    # Only required to run tests
+    mock
 commands =
-    {toxinidir}/bin/runtests.sh []
+    trial scrapy
 
 [testenv:precise]
 basepython = python2.7
 deps =
-    -r{toxinidir}/.travis/requirements-precise.txt
+    pyOpenSSL==0.13
+    lxml==2.3.2
+    twisted==11.1.0
+    boto==2.2.2
+    Pillow<2.0
+    django==1.3.1
+    cssselect==0.9.1
+    mock==1.0.1
 
 [testenv:trunk]
 basepython = python2.7
-deps =
-    -r{toxinidir}/.travis/requirements-trunk.txt
+commands =
+    pip install https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
+    pip install https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
+    trial scrapy
 
 [testenv:windows]
 commands =
-    {toxinidir}/bin/runtests.bat []
+    bin/runtests.bat []
 sitepackages = False

From ecfa743105120538d6bab498b2651b3cc8c31633 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 7 Nov 2013 04:26:25 -0200
Subject: [PATCH 5/7] install updated pypy from ppa

---
 .travis-install.sh | 14 ++++++++++++++
 .travis.yml        |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100755 .travis-install.sh

diff --git a/.travis-install.sh b/.travis-install.sh
new file mode 100755
index 000000000..357de55ef
--- /dev/null
+++ b/.travis-install.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+set -x
+
+if [[ "${TOX_ENV}" == "pypy" ]]; then
+    sudo add-apt-repository -y ppa:pypy/ppa
+    sudo apt-get -qy update
+    sudo apt-get install -y pypy
+    # This is required because we need to get rid of the Travis installed PyPy
+    # or it'll take precedence over the PPA installed one.
+    sudo rm -rf /usr/local/pypy/bin
+fi
+
+pip install tox coveralls
diff --git a/.travis.yml b/.travis.yml
index 84250a75f..f823d1923 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,7 @@ matrix:
         - env: TOX_ENV=pypy
 
 install:
-    - pip install -M tox
+    - ./.travis-install.sh
 
 script:
     - tox -e $TOX_ENV

From 971f60d7961bd96d0e7bd62e8f7e94500ce281f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 7 Nov 2013 05:01:32 -0200
Subject: [PATCH 6/7] TOXENV is tox supported env

---
 .travis-install.sh => .travis-workarounds.sh |  4 +---
 .travis.yml                                  | 16 ++++++++--------
 2 files changed, 9 insertions(+), 11 deletions(-)
 rename .travis-install.sh => .travis-workarounds.sh (82%)

diff --git a/.travis-install.sh b/.travis-workarounds.sh
similarity index 82%
rename from .travis-install.sh
rename to .travis-workarounds.sh
index 357de55ef..0d9ea2033 100755
--- a/.travis-install.sh
+++ b/.travis-workarounds.sh
@@ -2,7 +2,7 @@
 set -e
 set -x
 
-if [[ "${TOX_ENV}" == "pypy" ]]; then
+if [[ "${TOXENV}" == "pypy" ]]; then
     sudo add-apt-repository -y ppa:pypy/ppa
     sudo apt-get -qy update
     sudo apt-get install -y pypy
@@ -10,5 +10,3 @@ if [[ "${TOX_ENV}" == "pypy" ]]; then
     # or it'll take precedence over the PPA installed one.
     sudo rm -rf /usr/local/pypy/bin
 fi
-
-pip install tox coveralls
diff --git a/.travis.yml b/.travis.yml
index f823d1923..f2f9c4eb2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,20 +1,20 @@
 language: python
 python: 2.7
 env:
-    - TOX_ENV=py27
-    - TOX_ENV=precise
-    - TOX_ENV=trunk
-    - TOX_ENV=pypy
+    - TOXENV=py27
+    - TOXENV=precise
+    - TOXENV=trunk
+    - TOXENV=pypy
 
 matrix:
     allow_failures:
-        - env: TOX_ENV=pypy
+        - env: TOXENV=pypy
 
 install:
-    - ./.travis-install.sh
+    - ./.travis-workarounds.sh
+    - pip install -M tox
 
-script:
-    - tox -e $TOX_ENV
+script: tox
 
 notifications:
     irc:

From d29791d7ab1d083be50ff48087b940eb3e082e37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 7 Nov 2013 09:54:04 -0200
Subject: [PATCH 7/7] building Pillow with pypy requries dev headers

---
 .travis-workarounds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis-workarounds.sh b/.travis-workarounds.sh
index 0d9ea2033..3f7727376 100755
--- a/.travis-workarounds.sh
+++ b/.travis-workarounds.sh
@@ -5,7 +5,7 @@ set -x
 if [[ "${TOXENV}" == "pypy" ]]; then
     sudo add-apt-repository -y ppa:pypy/ppa
     sudo apt-get -qy update
-    sudo apt-get install -y pypy
+    sudo apt-get install -y pypy pypy-dev
     # This is required because we need to get rid of the Travis installed PyPy
     # or it'll take precedence over the PPA installed one.
     sudo rm -rf /usr/local/pypy/bin