move tests under root dir

MANIFEST.in

@@ -6,9 +6,7 @@ include MANIFEST.in
 include scrapy/VERSION
 include scrapy/mime.types
 recursive-include scrapy/templates *
-recursive-include scrapy/tests/sample_data *
 recursive-include scrapy license.txt
-recursive-include scrapy/tests *.egg
 recursive-include docs *
 prune docs/build
 recursive-include extras *

conftest.py

@@ -4,7 +4,7 @@ from twisted.python import log
 
 from scrapy import optional_features
 
-collect_ignore = ["stats.py"]
+collect_ignore = ["scrapy/stats.py"]
 if 'django' not in optional_features:
     collect_ignore.append("tests/test_djangoitem/models.py")
 

docs/contributing.rst

@@ -135,23 +135,20 @@ Documentation policies
 Tests
 =====
 
-Tests are implemented using the `Twisted unit-testing framework`_ called
+Tests are implemented using the `Twisted unit-testing framework`_, running
-``trial``.
+tests requires `tox`_.
 
 Running tests
 -------------
 
 To run all tests go to the root directory of Scrapy source code and run:
 
-    ``bin/runtests.sh`` (on unix)
+    ``tox``
 
-    ``bin\runtests.bat`` (on windows)
+To run a specific test (say ``tests/test_contrib_loader.py``) use:
 
-To run a specific test (say ``scrapy.tests.test_contrib_loader``) use:
+    ``tox tests/test_contrib_loader.py``
 
-    ``bin/runtests.sh scrapy.tests.test_contrib_loader`` (on unix)
-
-    ``bin\runtests.bat scrapy.tests.test_contrib_loader`` (on windows)
 
 Writing tests
 -------------
@@ -160,20 +157,21 @@ All functionality (including new features and bug fixes) must include a test
 case to check that it works as expected, so please include tests for your
 patches if you want them to get accepted sooner.
 
-Scrapy uses unit-tests, which are located in the ``scrapy.tests`` package
+Scrapy uses unit-tests, which are located in the `tests/`_ directory.
-(`scrapy/tests`_ directory). Their module name typically resembles the full
+Their module name typically resembles the full path of the module they're
-path of the module they're testing. For example, the item loaders code is in::
+testing. For example, the item loaders code is in::
 
     scrapy.contrib.loader
 
 And their unit-tests are in::
 
-    scrapy.tests.test_contrib_loader
+    tests/test_contrib_loader.py
 
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: http://groups.google.com/group/scrapy-users
 .. _Twisted unit-testing framework: http://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
-.. _scrapy/tests: https://github.com/scrapy/scrapy/tree/master/scrapy/tests
+.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
 .. _pull request: http://help.github.com/send-pull-requests/
+.. _tox: https://pypi.python.org/pypi/tox

extras/coverage-report.sh

@@ -2,6 +2,6 @@
 #
 # Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage
 
-coverage run --branch $(which trial) --reporter=text scrapy.tests
+coverage run --branch $(which trial) --reporter=text tests
 coverage html -i
 python -m webbrowser htmlcov/index.html

scrapy/commands/bench.py

@@ -1,6 +1,6 @@
 from scrapy.command import ScrapyCommand
-from scrapy.tests.spiders import FollowAllSpider
+from tests.spiders import FollowAllSpider
-from scrapy.tests.mockserver import MockServer
+from tests.mockserver import MockServer
 
 class Command(ScrapyCommand):
 

scrapy/tests/test_cmdline/settings.py

@@ -1,5 +0,0 @@
-EXTENSIONS = [
-    'scrapy.tests.test_cmdline.extensions.TestExtension'
-]
-
-TEST1 = 'default'

scrapy/utils/url.py

@@ -50,7 +50,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
     The url passed can be a str or unicode, while the url returned is always a
     str.
 
-    For examples see the tests in scrapy.tests.test_utils_url
+    For examples see the tests in tests/test_utils_url.py
     """
 
     scheme, netloc, path, params, query, fragment = parse_url(url)

tests/__init__.py

@@ -1,5 +1,5 @@
 """
-scrapy.tests: this package contains all Scrapy unittests
+tests: this package contains all Scrapy unittests
 
 To run all Scrapy unittests go to Scrapy main dir and type: 
 

tests/mockserver.py

@@ -179,7 +179,7 @@ class MockServer():
 
     def __enter__(self):
         from scrapy.utils.test import get_testenv
-        self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'],
+        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
                           stdout=PIPE, env=get_testenv())
         self.proc.stdout.readline()
 

tests/test_closespider.py

@@ -1,8 +1,8 @@
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import docrawl
-from scrapy.tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider
+from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider
-from scrapy.tests.mockserver import MockServer
+from tests.mockserver import MockServer
 
 
 class TestCloseSpider(TestCase):

tests/test_cmdline/__init__.py

@@ -8,7 +8,7 @@ class CmdlineTest(unittest.TestCase):
 
     def setUp(self):
         self.env = get_testenv()
-        self.env['SCRAPY_SETTINGS_MODULE'] = 'scrapy.tests.test_cmdline.settings'
+        self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings'
 
     def _execute(self, *new_args, **kwargs):
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args

tests/test_cmdline/settings.py

@@ -0,0 +1,5 @@
+EXTENSIONS = [
+    'tests.test_cmdline.extensions.TestExtension'
+]
+
+TEST1 = 'default'

tests/test_contrib_linkextractors.py

@@ -6,7 +6,7 @@ from scrapy.link import Link
 from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
 from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
-from scrapy.tests import get_testdata
+from tests import get_testdata
 
 
 class LinkExtractorTestCase(unittest.TestCase):

tests/test_crawl.py

@@ -4,9 +4,9 @@ import mock
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import docrawl, get_testlog
-from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
+from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
-from scrapy.tests.mockserver import MockServer
+from tests.mockserver import MockServer
 from scrapy.http import Request
 
 

tests/test_djangoitem/__init__.py

@@ -4,7 +4,7 @@ from twisted.trial import unittest
 from scrapy.contrib.djangoitem import DjangoItem, Field
 from scrapy import optional_features
 
-os.environ['DJANGO_SETTINGS_MODULE'] = 'scrapy.tests.test_djangoitem.settings'
+os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_djangoitem.settings'
 
 if 'django' in optional_features:
     from .models import Person, IdentifiedPerson

tests/test_downloader_handlers.py

@@ -46,13 +46,13 @@ class OffDH(object):
 class LoadTestCase(unittest.TestCase):
 
     def test_enabled_handler(self):
-        handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.DummyDH'}
+        handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'}
         dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
         self.assertIn('scheme', dh._handlers)
         self.assertNotIn('scheme', dh._notconfigured)
 
     def test_not_configured_handler(self):
-        handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.OffDH'}
+        handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'}
         dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers}))
         self.assertNotIn('scheme', dh._handlers)
         self.assertIn('scheme', dh._notconfigured)

tests/test_downloadermiddleware_decompression.py

@@ -2,7 +2,7 @@ from unittest import TestCase, main
 from scrapy.http import Response, XmlResponse
 from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware
 from scrapy.spider import Spider
-from scrapy.tests import get_testdata
+from tests import get_testdata
 from scrapy.utils.test import assert_samelines
 
 

tests/test_downloadermiddleware_httpcache.py

@@ -120,7 +120,7 @@ class DbmStorageTest(DefaultStorageTest):
 
 class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
 
-    dbm_module = 'scrapy.tests.mocks.dummydbm'
+    dbm_module = 'tests.mocks.dummydbm'
 
     def _get_settings(self, **new_settings):
         new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)

tests/test_downloadermiddleware_httpcompression.py

@@ -6,7 +6,7 @@ from gzip import GzipFile
 from scrapy.spider import Spider
 from scrapy.http import Response, Request, HtmlResponse
 from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware
-from scrapy.tests import tests_datadir
+from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
 

tests/test_engine.py

@@ -21,7 +21,7 @@ from twisted.trial import unittest
 from scrapy import signals
 from scrapy.utils.test import get_crawler
 from scrapy.xlib.pydispatch import dispatcher
-from scrapy.tests import tests_datadir
+from tests import tests_datadir
 from scrapy.spider import Spider
 from scrapy.item import Item, Field
 from scrapy.contrib.linkextractors import LinkExtractor

tests/test_middleware.py

@@ -47,7 +47,7 @@ class TestMiddlewareManager(MiddlewareManager):
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
-        return ['scrapy.tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
+        return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
 
     def _add_middleware(self, mw):
         super(TestMiddlewareManager, self)._add_middleware(mw)

tests/test_proxy_connect.py

@@ -9,8 +9,8 @@ from netlib import http_auth
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import get_testlog, docrawl
-from scrapy.tests.spiders import SimpleSpider
+from tests.spiders import SimpleSpider
-from scrapy.tests.mockserver import MockServer
+from tests.mockserver import MockServer
 
 
 

tests/test_settings/__init__.py

@@ -127,7 +127,7 @@ class SettingsTest(unittest.TestCase):
 
         self.settings.attributes = {}
         self.settings.setmodule(
-            'scrapy.tests.test_settings.default_settings', 10)
+            'tests.test_settings.default_settings', 10)
 
         self.assertItemsEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))

tests/test_spidermanager/__init__.py

@@ -59,11 +59,11 @@ class SpiderManagerTest(unittest.TestCase):
             ['spider3'])
 
     def test_load_spider_module(self):
-        self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider1'])
+        self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider1'])
         assert len(self.spiderman._spiders) == 1
 
     def test_load_base_spider(self):
-        self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0'])
+        self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider0'])
         assert len(self.spiderman._spiders) == 0
 
     def test_load_from_crawler(self):