diff --git a/MANIFEST.in b/MANIFEST.in index c9b608a43..0561cc74c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,9 +6,7 @@ include MANIFEST.in include scrapy/VERSION include scrapy/mime.types recursive-include scrapy/templates * -recursive-include scrapy/tests/sample_data * recursive-include scrapy license.txt -recursive-include scrapy/tests *.egg recursive-include docs * prune docs/build recursive-include extras * diff --git a/bin/runtests.bat b/bin/runtests.bat deleted file mode 100644 index 6c882fa3d..000000000 --- a/bin/runtests.bat +++ /dev/null @@ -1,6 +0,0 @@ -@ECHO off - -SET test=scrapy -SET PYTHONPATH=%CD% -IF NOT "%1" == "" SET test="%1" -trial --reporter=text %test% diff --git a/bin/runtests.sh b/bin/runtests.sh deleted file mode 100755 index 0e7a318b9..000000000 --- a/bin/runtests.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -# look for twisted trial command -if type trial >/dev/null 2>&1; then - trial="trial" -elif [ -x /usr/lib/twisted/bin/trial ]; then - trial="/usr/lib/twisted/bin/trial" -elif [ -x /usr/lib64/twisted/bin/trial ]; then - trial="/usr/lib64/twisted/bin/trial" -else - echo "Unable to run tests: trial command (included with Twisted) not found" - exit 1 -fi - -# use vsftpd (if available) for testing ftp feed storage -if type vsftpd >/dev/null 2>&1; then - vsftpd_conf=$(mktemp /tmp/vsftpd-XXXX) - cat >$vsftpd_conf <") + args = request.args.copy() + for nl in nlist: + args['n'] = nl + argstr = urlencode(args, doseq=True) + request.write("follow {1}
" + .format(argstr, nl)) + request.write("") + return '' + + +def _getarg(request, name, default=None, type=str): + return type(request.args[name][0]) \ + if name in request.args else default + + +if __name__ == '__main__': + root = Root() + factory = Site(root) + httpPort = reactor.listenTCP(8998, Site(root)) + + def _print_listening(): + httpHost = httpPort.getHost() + print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port)) + reactor.callWhenRunning(_print_listening) + reactor.run() diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 50107b434..ab4d75f87 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -50,7 +50,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, The url passed can be a str or unicode, while the url returned is always a str. - For examples see the tests in scrapy.tests.test_utils_url + For examples see the tests in tests/test_utils_url.py """ scheme, netloc, path, params, query, fragment = parse_url(url) diff --git a/scrapy/tests/__init__.py b/tests/__init__.py similarity index 86% rename from scrapy/tests/__init__.py rename to tests/__init__.py index afd463740..f7d11c780 100644 --- a/scrapy/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,5 @@ """ -scrapy.tests: this package contains all Scrapy unittests +tests: this package contains all Scrapy unittests To run all Scrapy unittests go to Scrapy main dir and type: diff --git a/scrapy/tests/keys/cert.pem b/tests/keys/cert.pem similarity index 100% rename from scrapy/tests/keys/cert.pem rename to tests/keys/cert.pem diff --git a/scrapy/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem similarity index 100% rename from scrapy/tests/keys/mitmproxy-ca.pem rename to tests/keys/mitmproxy-ca.pem diff --git a/scrapy/tests/mocks/__init__.py b/tests/mocks/__init__.py similarity index 100% rename from scrapy/tests/mocks/__init__.py rename to tests/mocks/__init__.py diff --git a/scrapy/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py similarity index 100% rename from scrapy/tests/mocks/dummydbm.py rename to tests/mocks/dummydbm.py diff --git a/scrapy/tests/mockserver.py b/tests/mockserver.py similarity index 98% rename from scrapy/tests/mockserver.py rename to tests/mockserver.py index 0f26e848d..6910532b6 100644 --- a/scrapy/tests/mockserver.py +++ b/tests/mockserver.py @@ -179,7 +179,7 @@ class MockServer(): def __enter__(self): from scrapy.utils.test import get_testenv - self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], stdout=PIPE, env=get_testenv()) self.proc.stdout.readline() diff --git a/tests-requirements.txt b/tests/requirements.txt similarity index 100% rename from tests-requirements.txt rename to tests/requirements.txt diff --git a/scrapy/tests/sample_data/compressed/feed-sample1.tar b/tests/sample_data/compressed/feed-sample1.tar similarity index 100% rename from scrapy/tests/sample_data/compressed/feed-sample1.tar rename to tests/sample_data/compressed/feed-sample1.tar diff --git a/scrapy/tests/sample_data/compressed/feed-sample1.xml b/tests/sample_data/compressed/feed-sample1.xml similarity index 100% rename from scrapy/tests/sample_data/compressed/feed-sample1.xml rename to tests/sample_data/compressed/feed-sample1.xml diff --git a/scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 b/tests/sample_data/compressed/feed-sample1.xml.bz2 similarity index 100% rename from scrapy/tests/sample_data/compressed/feed-sample1.xml.bz2 rename to tests/sample_data/compressed/feed-sample1.xml.bz2 diff --git a/scrapy/tests/sample_data/compressed/feed-sample1.xml.gz b/tests/sample_data/compressed/feed-sample1.xml.gz similarity index 100% rename from scrapy/tests/sample_data/compressed/feed-sample1.xml.gz rename to tests/sample_data/compressed/feed-sample1.xml.gz diff --git a/scrapy/tests/sample_data/compressed/feed-sample1.zip b/tests/sample_data/compressed/feed-sample1.zip similarity index 100% rename from scrapy/tests/sample_data/compressed/feed-sample1.zip rename to tests/sample_data/compressed/feed-sample1.zip diff --git a/scrapy/tests/sample_data/compressed/html-gzip.bin b/tests/sample_data/compressed/html-gzip.bin similarity index 100% rename from scrapy/tests/sample_data/compressed/html-gzip.bin rename to tests/sample_data/compressed/html-gzip.bin diff --git a/scrapy/tests/sample_data/compressed/html-rawdeflate.bin b/tests/sample_data/compressed/html-rawdeflate.bin similarity index 100% rename from scrapy/tests/sample_data/compressed/html-rawdeflate.bin rename to tests/sample_data/compressed/html-rawdeflate.bin diff --git a/scrapy/tests/sample_data/compressed/html-zlibdeflate.bin b/tests/sample_data/compressed/html-zlibdeflate.bin similarity index 100% rename from scrapy/tests/sample_data/compressed/html-zlibdeflate.bin rename to tests/sample_data/compressed/html-zlibdeflate.bin diff --git a/scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz b/tests/sample_data/compressed/truncated-crc-error-short.gz similarity index 100% rename from scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz rename to tests/sample_data/compressed/truncated-crc-error-short.gz diff --git a/scrapy/tests/sample_data/compressed/truncated-crc-error.gz b/tests/sample_data/compressed/truncated-crc-error.gz similarity index 100% rename from scrapy/tests/sample_data/compressed/truncated-crc-error.gz rename to tests/sample_data/compressed/truncated-crc-error.gz diff --git a/scrapy/tests/sample_data/feeds/feed-sample1.xml b/tests/sample_data/feeds/feed-sample1.xml similarity index 100% rename from scrapy/tests/sample_data/feeds/feed-sample1.xml rename to tests/sample_data/feeds/feed-sample1.xml diff --git a/scrapy/tests/sample_data/feeds/feed-sample2.xml b/tests/sample_data/feeds/feed-sample2.xml similarity index 100% rename from scrapy/tests/sample_data/feeds/feed-sample2.xml rename to tests/sample_data/feeds/feed-sample2.xml diff --git a/scrapy/tests/sample_data/feeds/feed-sample3.csv b/tests/sample_data/feeds/feed-sample3.csv similarity index 100% rename from scrapy/tests/sample_data/feeds/feed-sample3.csv rename to tests/sample_data/feeds/feed-sample3.csv diff --git a/scrapy/tests/sample_data/feeds/feed-sample4.csv b/tests/sample_data/feeds/feed-sample4.csv similarity index 100% rename from scrapy/tests/sample_data/feeds/feed-sample4.csv rename to tests/sample_data/feeds/feed-sample4.csv diff --git a/scrapy/tests/sample_data/feeds/feed-sample5.csv b/tests/sample_data/feeds/feed-sample5.csv similarity index 100% rename from scrapy/tests/sample_data/feeds/feed-sample5.csv rename to tests/sample_data/feeds/feed-sample5.csv diff --git a/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html similarity index 100% rename from scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html rename to tests/sample_data/link_extractor/linkextractor_latin1.html diff --git a/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html similarity index 100% rename from scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html rename to tests/sample_data/link_extractor/linkextractor_noenc.html diff --git a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html similarity index 100% rename from scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html rename to tests/sample_data/link_extractor/sgml_linkextractor.html diff --git a/scrapy/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html similarity index 100% rename from scrapy/tests/sample_data/test_site/index.html rename to tests/sample_data/test_site/index.html diff --git a/scrapy/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html similarity index 100% rename from scrapy/tests/sample_data/test_site/item1.html rename to tests/sample_data/test_site/item1.html diff --git a/scrapy/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html similarity index 100% rename from scrapy/tests/sample_data/test_site/item2.html rename to tests/sample_data/test_site/item2.html diff --git a/scrapy/tests/spiders.py b/tests/spiders.py similarity index 100% rename from scrapy/tests/spiders.py rename to tests/spiders.py diff --git a/scrapy/tests/test_closespider.py b/tests/test_closespider.py similarity index 94% rename from scrapy/tests/test_closespider.py rename to tests/test_closespider.py index c6622d486..8d30a4643 100644 --- a/scrapy/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,8 +1,8 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.utils.test import docrawl -from scrapy.tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider -from scrapy.tests.mockserver import MockServer +from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider +from tests.mockserver import MockServer class TestCloseSpider(TestCase): diff --git a/scrapy/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py similarity index 92% rename from scrapy/tests/test_cmdline/__init__.py rename to tests/test_cmdline/__init__.py index 579da0e61..00fce2fbc 100644 --- a/scrapy/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -8,7 +8,7 @@ class CmdlineTest(unittest.TestCase): def setUp(self): self.env = get_testenv() - self.env['SCRAPY_SETTINGS_MODULE'] = 'scrapy.tests.test_cmdline.settings' + self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings' def _execute(self, *new_args, **kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args diff --git a/scrapy/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py similarity index 100% rename from scrapy/tests/test_cmdline/extensions.py rename to tests/test_cmdline/extensions.py diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py new file mode 100644 index 000000000..9aceffb0d --- /dev/null +++ b/tests/test_cmdline/settings.py @@ -0,0 +1,5 @@ +EXTENSIONS = [ + 'tests.test_cmdline.extensions.TestExtension' +] + +TEST1 = 'default' diff --git a/scrapy/tests/test_command_fetch.py b/tests/test_command_fetch.py similarity index 100% rename from scrapy/tests/test_command_fetch.py rename to tests/test_command_fetch.py diff --git a/scrapy/tests/test_command_shell.py b/tests/test_command_shell.py similarity index 100% rename from scrapy/tests/test_command_shell.py rename to tests/test_command_shell.py diff --git a/scrapy/tests/test_command_version.py b/tests/test_command_version.py similarity index 100% rename from scrapy/tests/test_command_version.py rename to tests/test_command_version.py diff --git a/scrapy/tests/test_commands.py b/tests/test_commands.py similarity index 100% rename from scrapy/tests/test_commands.py rename to tests/test_commands.py diff --git a/scrapy/tests/test_contracts.py b/tests/test_contracts.py similarity index 100% rename from scrapy/tests/test_contracts.py rename to tests/test_contracts.py diff --git a/scrapy/tests/test_contrib_exporter.py b/tests/test_contrib_exporter.py similarity index 100% rename from scrapy/tests/test_contrib_exporter.py rename to tests/test_contrib_exporter.py diff --git a/scrapy/tests/test_contrib_feedexport.py b/tests/test_contrib_feedexport.py similarity index 100% rename from scrapy/tests/test_contrib_feedexport.py rename to tests/test_contrib_feedexport.py diff --git a/scrapy/tests/test_contrib_linkextractors.py b/tests/test_contrib_linkextractors.py similarity index 99% rename from scrapy/tests/test_contrib_linkextractors.py rename to tests/test_contrib_linkextractors.py index 3609e4b80..3617cb810 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/tests/test_contrib_linkextractors.py @@ -6,7 +6,7 @@ from scrapy.link import Link from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor -from scrapy.tests import get_testdata +from tests import get_testdata class LinkExtractorTestCase(unittest.TestCase): diff --git a/scrapy/tests/test_contrib_loader.py b/tests/test_contrib_loader.py similarity index 100% rename from scrapy/tests/test_contrib_loader.py rename to tests/test_contrib_loader.py diff --git a/scrapy/tests/test_contrib_spiderstate.py b/tests/test_contrib_spiderstate.py similarity index 100% rename from scrapy/tests/test_contrib_spiderstate.py rename to tests/test_contrib_spiderstate.py diff --git a/scrapy/tests/test_crawl.py b/tests/test_crawl.py similarity index 98% rename from scrapy/tests/test_crawl.py rename to tests/test_crawl.py index 81666930d..9401bd0c9 100644 --- a/scrapy/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -4,9 +4,9 @@ import mock from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.utils.test import docrawl, get_testlog -from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ +from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider -from scrapy.tests.mockserver import MockServer +from tests.mockserver import MockServer from scrapy.http import Request diff --git a/scrapy/tests/test_dependencies.py b/tests/test_dependencies.py similarity index 100% rename from scrapy/tests/test_dependencies.py rename to tests/test_dependencies.py diff --git a/scrapy/tests/test_djangoitem/__init__.py b/tests/test_djangoitem/__init__.py similarity index 97% rename from scrapy/tests/test_djangoitem/__init__.py rename to tests/test_djangoitem/__init__.py index d50428d0b..45fd7c470 100644 --- a/scrapy/tests/test_djangoitem/__init__.py +++ b/tests/test_djangoitem/__init__.py @@ -4,7 +4,7 @@ from twisted.trial import unittest from scrapy.contrib.djangoitem import DjangoItem, Field from scrapy import optional_features -os.environ['DJANGO_SETTINGS_MODULE'] = 'scrapy.tests.test_djangoitem.settings' +os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_djangoitem.settings' if 'django' in optional_features: from .models import Person, IdentifiedPerson diff --git a/scrapy/tests/test_djangoitem/models.py b/tests/test_djangoitem/models.py similarity index 100% rename from scrapy/tests/test_djangoitem/models.py rename to tests/test_djangoitem/models.py diff --git a/scrapy/tests/test_djangoitem/settings.py b/tests/test_djangoitem/settings.py similarity index 100% rename from scrapy/tests/test_djangoitem/settings.py rename to tests/test_djangoitem/settings.py diff --git a/scrapy/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py similarity index 99% rename from scrapy/tests/test_downloader_handlers.py rename to tests/test_downloader_handlers.py index a4e54ed76..8c1ad1435 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -46,13 +46,13 @@ class OffDH(object): class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): - handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.DummyDH'} + handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) self.assertIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): - handlers = {'scheme': 'scrapy.tests.test_downloader_handlers.OffDH'} + handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) diff --git a/scrapy/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware.py rename to tests/test_downloadermiddleware.py diff --git a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py rename to tests/test_downloadermiddleware_ajaxcrawlable.py diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_cookies.py rename to tests/test_downloadermiddleware_cookies.py diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py similarity index 97% rename from scrapy/tests/test_downloadermiddleware_decompression.py rename to tests/test_downloadermiddleware_decompression.py index 33208028a..81e12b4f9 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -2,7 +2,7 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware from scrapy.spider import Spider -from scrapy.tests import get_testdata +from tests import get_testdata from scrapy.utils.test import assert_samelines diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_defaultheaders.py rename to tests/test_downloadermiddleware_defaultheaders.py diff --git a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_downloadtimeout.py rename to tests/test_downloadermiddleware_downloadtimeout.py diff --git a/scrapy/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_httpauth.py rename to tests/test_downloadermiddleware_httpauth.py diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py similarity index 99% rename from scrapy/tests/test_downloadermiddleware_httpcache.py rename to tests/test_downloadermiddleware_httpcache.py index efd9a9b7a..0eb5e7144 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -120,7 +120,7 @@ class DbmStorageTest(DefaultStorageTest): class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): - dbm_module = 'scrapy.tests.mocks.dummydbm' + dbm_module = 'tests.mocks.dummydbm' def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py similarity index 99% rename from scrapy/tests/test_downloadermiddleware_httpcompression.py rename to tests/test_downloadermiddleware_httpcompression.py index 8a0e75d90..1cc6f44c1 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -6,7 +6,7 @@ from gzip import GzipFile from scrapy.spider import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware -from scrapy.tests import tests_datadir +from tests import tests_datadir from w3lib.encoding import resolve_encoding diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_httpproxy.py rename to tests/test_downloadermiddleware_httpproxy.py diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_redirect.py rename to tests/test_downloadermiddleware_redirect.py diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_retry.py rename to tests/test_downloadermiddleware_retry.py diff --git a/scrapy/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_robotstxt.py rename to tests/test_downloadermiddleware_robotstxt.py diff --git a/scrapy/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_stats.py rename to tests/test_downloadermiddleware_stats.py diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py similarity index 100% rename from scrapy/tests/test_downloadermiddleware_useragent.py rename to tests/test_downloadermiddleware_useragent.py diff --git a/scrapy/tests/test_dupefilter.py b/tests/test_dupefilter.py similarity index 100% rename from scrapy/tests/test_dupefilter.py rename to tests/test_dupefilter.py diff --git a/scrapy/tests/test_engine.py b/tests/test_engine.py similarity index 99% rename from scrapy/tests/test_engine.py rename to tests/test_engine.py index 2881b9168..6a0314a02 100644 --- a/scrapy/tests/test_engine.py +++ b/tests/test_engine.py @@ -21,7 +21,7 @@ from twisted.trial import unittest from scrapy import signals from scrapy.utils.test import get_crawler from scrapy.xlib.pydispatch import dispatcher -from scrapy.tests import tests_datadir +from tests import tests_datadir from scrapy.spider import Spider from scrapy.item import Item, Field from scrapy.contrib.linkextractors import LinkExtractor diff --git a/scrapy/tests/test_http_cookies.py b/tests/test_http_cookies.py similarity index 100% rename from scrapy/tests/test_http_cookies.py rename to tests/test_http_cookies.py diff --git a/scrapy/tests/test_http_headers.py b/tests/test_http_headers.py similarity index 100% rename from scrapy/tests/test_http_headers.py rename to tests/test_http_headers.py diff --git a/scrapy/tests/test_http_request.py b/tests/test_http_request.py similarity index 100% rename from scrapy/tests/test_http_request.py rename to tests/test_http_request.py diff --git a/scrapy/tests/test_http_response.py b/tests/test_http_response.py similarity index 100% rename from scrapy/tests/test_http_response.py rename to tests/test_http_response.py diff --git a/scrapy/tests/test_item.py b/tests/test_item.py similarity index 100% rename from scrapy/tests/test_item.py rename to tests/test_item.py diff --git a/scrapy/tests/test_link.py b/tests/test_link.py similarity index 100% rename from scrapy/tests/test_link.py rename to tests/test_link.py diff --git a/scrapy/tests/test_log.py b/tests/test_log.py similarity index 100% rename from scrapy/tests/test_log.py rename to tests/test_log.py diff --git a/scrapy/tests/test_logformatter.py b/tests/test_logformatter.py similarity index 100% rename from scrapy/tests/test_logformatter.py rename to tests/test_logformatter.py diff --git a/scrapy/tests/test_mail.py b/tests/test_mail.py similarity index 100% rename from scrapy/tests/test_mail.py rename to tests/test_mail.py diff --git a/scrapy/tests/test_middleware.py b/tests/test_middleware.py similarity index 96% rename from scrapy/tests/test_middleware.py rename to tests/test_middleware.py index 145b86b02..6834ba855 100644 --- a/scrapy/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -47,7 +47,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['scrapy.tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): super(TestMiddlewareManager, self)._add_middleware(mw) diff --git a/scrapy/tests/test_pipeline_files.py b/tests/test_pipeline_files.py similarity index 100% rename from scrapy/tests/test_pipeline_files.py rename to tests/test_pipeline_files.py diff --git a/scrapy/tests/test_pipeline_images.py b/tests/test_pipeline_images.py similarity index 100% rename from scrapy/tests/test_pipeline_images.py rename to tests/test_pipeline_images.py diff --git a/scrapy/tests/test_pipeline_media.py b/tests/test_pipeline_media.py similarity index 100% rename from scrapy/tests/test_pipeline_media.py rename to tests/test_pipeline_media.py diff --git a/scrapy/tests/test_proxy_connect.py b/tests/test_proxy_connect.py similarity index 97% rename from scrapy/tests/test_proxy_connect.py rename to tests/test_proxy_connect.py index e0b47a3dc..8a494afea 100644 --- a/scrapy/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -9,8 +9,8 @@ from netlib import http_auth from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.utils.test import get_testlog, docrawl -from scrapy.tests.spiders import SimpleSpider -from scrapy.tests.mockserver import MockServer +from tests.spiders import SimpleSpider +from tests.mockserver import MockServer diff --git a/scrapy/tests/test_responsetypes.py b/tests/test_responsetypes.py similarity index 100% rename from scrapy/tests/test_responsetypes.py rename to tests/test_responsetypes.py diff --git a/scrapy/tests/test_selector.py b/tests/test_selector.py similarity index 100% rename from scrapy/tests/test_selector.py rename to tests/test_selector.py diff --git a/scrapy/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py similarity index 100% rename from scrapy/tests/test_selector_csstranslator.py rename to tests/test_selector_csstranslator.py diff --git a/scrapy/tests/test_selector_lxmldocument.py b/tests/test_selector_lxmldocument.py similarity index 100% rename from scrapy/tests/test_selector_lxmldocument.py rename to tests/test_selector_lxmldocument.py diff --git a/scrapy/tests/test_settings/__init__.py b/tests/test_settings/__init__.py similarity index 99% rename from scrapy/tests/test_settings/__init__.py rename to tests/test_settings/__init__.py index b7c33a5d3..39e47dec6 100644 --- a/scrapy/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -127,7 +127,7 @@ class SettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule( - 'scrapy.tests.test_settings.default_settings', 10) + 'tests.test_settings.default_settings', 10) self.assertItemsEqual(six.iterkeys(self.settings.attributes), six.iterkeys(ctrl_attributes)) diff --git a/scrapy/tests/test_settings/default_settings.py b/tests/test_settings/default_settings.py similarity index 100% rename from scrapy/tests/test_settings/default_settings.py rename to tests/test_settings/default_settings.py diff --git a/scrapy/tests/test_spider.py b/tests/test_spider.py similarity index 100% rename from scrapy/tests/test_spider.py rename to tests/test_spider.py diff --git a/scrapy/tests/test_spidermanager/__init__.py b/tests/test_spidermanager/__init__.py similarity index 92% rename from scrapy/tests/test_spidermanager/__init__.py rename to tests/test_spidermanager/__init__.py index eebd6b298..b0dd9a851 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/tests/test_spidermanager/__init__.py @@ -59,11 +59,11 @@ class SpiderManagerTest(unittest.TestCase): ['spider3']) def test_load_spider_module(self): - self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider1']) + self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider1']) assert len(self.spiderman._spiders) == 1 def test_load_base_spider(self): - self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0']) + self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider0']) assert len(self.spiderman._spiders) == 0 def test_load_from_crawler(self): diff --git a/scrapy/tests/test_spidermanager/test_spiders/__init__.py b/tests/test_spidermanager/test_spiders/__init__.py similarity index 100% rename from scrapy/tests/test_spidermanager/test_spiders/__init__.py rename to tests/test_spidermanager/test_spiders/__init__.py diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider0.py b/tests/test_spidermanager/test_spiders/spider0.py similarity index 100% rename from scrapy/tests/test_spidermanager/test_spiders/spider0.py rename to tests/test_spidermanager/test_spiders/spider0.py diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider1.py b/tests/test_spidermanager/test_spiders/spider1.py similarity index 100% rename from scrapy/tests/test_spidermanager/test_spiders/spider1.py rename to tests/test_spidermanager/test_spiders/spider1.py diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider2.py b/tests/test_spidermanager/test_spiders/spider2.py similarity index 100% rename from scrapy/tests/test_spidermanager/test_spiders/spider2.py rename to tests/test_spidermanager/test_spiders/spider2.py diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider3.py b/tests/test_spidermanager/test_spiders/spider3.py similarity index 100% rename from scrapy/tests/test_spidermanager/test_spiders/spider3.py rename to tests/test_spidermanager/test_spiders/spider3.py diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider4.py b/tests/test_spidermanager/test_spiders/spider4.py similarity index 100% rename from scrapy/tests/test_spidermanager/test_spiders/spider4.py rename to tests/test_spidermanager/test_spiders/spider4.py diff --git a/scrapy/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py similarity index 100% rename from scrapy/tests/test_spidermiddleware_depth.py rename to tests/test_spidermiddleware_depth.py diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py similarity index 99% rename from scrapy/tests/test_spidermiddleware_httperror.py rename to tests/test_spidermiddleware_httperror.py index b078ba4ba..788a0986b 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -4,7 +4,7 @@ from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer from scrapy.utils.test import docrawl, get_testlog -from scrapy.tests.mockserver import MockServer +from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py similarity index 100% rename from scrapy/tests/test_spidermiddleware_offsite.py rename to tests/test_spidermiddleware_offsite.py diff --git a/scrapy/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py similarity index 100% rename from scrapy/tests/test_spidermiddleware_referer.py rename to tests/test_spidermiddleware_referer.py diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py similarity index 100% rename from scrapy/tests/test_spidermiddleware_urllength.py rename to tests/test_spidermiddleware_urllength.py diff --git a/scrapy/tests/test_squeue.py b/tests/test_squeue.py similarity index 100% rename from scrapy/tests/test_squeue.py rename to tests/test_squeue.py diff --git a/scrapy/tests/test_stats.py b/tests/test_stats.py similarity index 100% rename from scrapy/tests/test_stats.py rename to tests/test_stats.py diff --git a/scrapy/tests/test_toplevel.py b/tests/test_toplevel.py similarity index 100% rename from scrapy/tests/test_toplevel.py rename to tests/test_toplevel.py diff --git a/scrapy/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py similarity index 100% rename from scrapy/tests/test_urlparse_monkeypatches.py rename to tests/test_urlparse_monkeypatches.py diff --git a/scrapy/tests/test_utils_conf.py b/tests/test_utils_conf.py similarity index 100% rename from scrapy/tests/test_utils_conf.py rename to tests/test_utils_conf.py diff --git a/scrapy/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py similarity index 100% rename from scrapy/tests/test_utils_datatypes.py rename to tests/test_utils_datatypes.py diff --git a/scrapy/tests/test_utils_defer.py b/tests/test_utils_defer.py similarity index 100% rename from scrapy/tests/test_utils_defer.py rename to tests/test_utils_defer.py diff --git a/scrapy/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py similarity index 94% rename from scrapy/tests/test_utils_deprecate.py rename to tests/test_utils_deprecate.py index 2a042ba85..d31b1d1f3 100644 --- a/scrapy/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -43,9 +43,9 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertEqual(len(w), 1) self.assertEqual( str(w[0].message), - "scrapy.tests.test_utils_deprecate.UserClass inherits from " - "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, " - "please inherit from scrapy.tests.test_utils_deprecate.NewName." + "tests.test_utils_deprecate.UserClass inherits from " + "deprecated class tests.test_utils_deprecate.Deprecated, " + "please inherit from tests.test_utils_deprecate.NewName." " (warning only on first subclass, there may be others)" ) self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1]) @@ -120,8 +120,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertEqual(len(w), 1) self.assertEqual( str(w[0].message), - "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate scrapy.tests.test_utils_deprecate.NewName instead." + "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." ) self.assertEqual(w[0].lineno, lineno) @@ -133,8 +133,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass msg = str(w[0].message) - self.assertIn("scrapy.tests.test_utils_deprecate.NewName", msg) - self.assertIn("scrapy.tests.test_utils_deprecate.Deprecated", msg) + self.assertIn("tests.test_utils_deprecate.NewName", msg) + self.assertIn("tests.test_utils_deprecate.Deprecated", msg) def test_issubclass(self): with warnings.catch_warnings(record=True): diff --git a/scrapy/tests/test_utils_gz.py b/tests/test_utils_gz.py similarity index 95% rename from scrapy/tests/test_utils_gz.py rename to tests/test_utils_gz.py index c7107f6a8..7250a870e 100644 --- a/scrapy/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,7 +1,7 @@ import unittest from os.path import join -from scrapy.tests import tests_datadir +from tests import tests_datadir from scrapy.utils.gz import gunzip SAMPLEDIR = join(tests_datadir, 'compressed') diff --git a/scrapy/tests/test_utils_http.py b/tests/test_utils_http.py similarity index 100% rename from scrapy/tests/test_utils_http.py rename to tests/test_utils_http.py diff --git a/scrapy/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py similarity index 100% rename from scrapy/tests/test_utils_httpobj.py rename to tests/test_utils_httpobj.py diff --git a/scrapy/tests/test_utils_iterators.py b/tests/test_utils_iterators.py similarity index 99% rename from scrapy/tests/test_utils_iterators.py rename to tests/test_utils_iterators.py index a0e152be7..fe53f831f 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -4,7 +4,7 @@ from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str from scrapy.contrib_exp.iterators import xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response -from scrapy.tests import get_testdata +from tests import get_testdata FOOBAR_NL = u"foo" + os.linesep + u"bar" diff --git a/scrapy/tests/test_utils_jsonrpc.py b/tests/test_utils_jsonrpc.py similarity index 98% rename from scrapy/tests/test_utils_jsonrpc.py rename to tests/test_utils_jsonrpc.py index 8902e9efa..e0aaef952 100644 --- a/scrapy/tests/test_utils_jsonrpc.py +++ b/tests/test_utils_jsonrpc.py @@ -4,7 +4,7 @@ from io import BytesIO from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \ JsonRpcError, jsonrpc_errors from scrapy.utils.serialize import ScrapyJSONDecoder -from scrapy.tests.test_utils_serialize import CrawlerMock +from tests.test_utils_serialize import CrawlerMock class urllib_mock(object): def __init__(self, result=None, error=None): diff --git a/scrapy/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py similarity index 76% rename from scrapy/tests/test_utils_misc/__init__.py rename to tests/test_utils_misc/__init__.py index 18fc6d532..01460a10b 100644 --- a/scrapy/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -16,25 +16,25 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') def test_walk_modules(self): - mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules') + mods = walk_modules('tests.test_utils_misc.test_walk_modules') expected = [ - 'scrapy.tests.test_utils_misc.test_walk_modules', - 'scrapy.tests.test_utils_misc.test_walk_modules.mod', - 'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0', - 'scrapy.tests.test_utils_misc.test_walk_modules.mod1', + 'tests.test_utils_misc.test_walk_modules', + 'tests.test_utils_misc.test_walk_modules.mod', + 'tests.test_utils_misc.test_walk_modules.mod.mod0', + 'tests.test_utils_misc.test_walk_modules.mod1', ] self.assertEquals(set([m.__name__ for m in mods]), set(expected)) - mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod') + mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod') expected = [ - 'scrapy.tests.test_utils_misc.test_walk_modules.mod', - 'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0', + 'tests.test_utils_misc.test_walk_modules.mod', + 'tests.test_utils_misc.test_walk_modules.mod.mod0', ] self.assertEquals(set([m.__name__ for m in mods]), set(expected)) - mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod1') + mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1') expected = [ - 'scrapy.tests.test_utils_misc.test_walk_modules.mod1', + 'tests.test_utils_misc.test_walk_modules.mod1', ] self.assertEquals(set([m.__name__ for m in mods]), set(expected)) diff --git a/scrapy/tests/test_utils_misc/test.egg b/tests/test_utils_misc/test.egg similarity index 100% rename from scrapy/tests/test_utils_misc/test.egg rename to tests/test_utils_misc/test.egg diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/__init__.py b/tests/test_utils_misc/test_walk_modules/__init__.py similarity index 100% rename from scrapy/tests/test_utils_misc/test_walk_modules/__init__.py rename to tests/test_utils_misc/test_walk_modules/__init__.py diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py b/tests/test_utils_misc/test_walk_modules/mod/__init__.py similarity index 100% rename from scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py rename to tests/test_utils_misc/test_walk_modules/mod/__init__.py diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py b/tests/test_utils_misc/test_walk_modules/mod/mod0.py similarity index 100% rename from scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py rename to tests/test_utils_misc/test_walk_modules/mod/mod0.py diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod1.py b/tests/test_utils_misc/test_walk_modules/mod1.py similarity index 100% rename from scrapy/tests/test_utils_misc/test_walk_modules/mod1.py rename to tests/test_utils_misc/test_walk_modules/mod1.py diff --git a/scrapy/tests/test_utils_python.py b/tests/test_utils_python.py similarity index 100% rename from scrapy/tests/test_utils_python.py rename to tests/test_utils_python.py diff --git a/scrapy/tests/test_utils_reqser.py b/tests/test_utils_reqser.py similarity index 100% rename from scrapy/tests/test_utils_reqser.py rename to tests/test_utils_reqser.py diff --git a/scrapy/tests/test_utils_request.py b/tests/test_utils_request.py similarity index 100% rename from scrapy/tests/test_utils_request.py rename to tests/test_utils_request.py diff --git a/scrapy/tests/test_utils_response.py b/tests/test_utils_response.py similarity index 100% rename from scrapy/tests/test_utils_response.py rename to tests/test_utils_response.py diff --git a/scrapy/tests/test_utils_serialize.py b/tests/test_utils_serialize.py similarity index 100% rename from scrapy/tests/test_utils_serialize.py rename to tests/test_utils_serialize.py diff --git a/scrapy/tests/test_utils_signal.py b/tests/test_utils_signal.py similarity index 100% rename from scrapy/tests/test_utils_signal.py rename to tests/test_utils_signal.py diff --git a/scrapy/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py similarity index 100% rename from scrapy/tests/test_utils_sitemap.py rename to tests/test_utils_sitemap.py diff --git a/scrapy/tests/test_utils_spider.py b/tests/test_utils_spider.py similarity index 89% rename from scrapy/tests/test_utils_spider.py rename to tests/test_utils_spider.py index a75a0e353..405b36715 100644 --- a/scrapy/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -28,8 +28,8 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import scrapy.tests.test_utils_spider - it = iter_spider_classes(scrapy.tests.test_utils_spider) + import tests.test_utils_spider + it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), set([MySpider1, MySpider2])) if __name__ == "__main__": diff --git a/scrapy/tests/test_utils_template.py b/tests/test_utils_template.py similarity index 100% rename from scrapy/tests/test_utils_template.py rename to tests/test_utils_template.py diff --git a/scrapy/tests/test_utils_url.py b/tests/test_utils_url.py similarity index 100% rename from scrapy/tests/test_utils_url.py rename to tests/test_utils_url.py diff --git a/scrapy/tests/test_webclient.py b/tests/test_webclient.py similarity index 100% rename from scrapy/tests/test_webclient.py rename to tests/test_webclient.py diff --git a/tox.ini b/tox.ini index 1474cce39..d86f750dd 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, pypy, precise, trunk, py33, docs +envlist = py27 [testenv] deps = @@ -14,34 +14,34 @@ deps = Pillow django leveldb - -rtests-requirements.txt + -rtests/requirements.txt commands = - py.test --twisted {posargs:scrapy} + py.test {posargs:scrapy tests} [testenv:precise] basepython = python2.7 deps = pyOpenSSL==0.13 lxml==2.3.2 - twisted==11.1.0 + Twisted==11.1.0 boto==2.2.2 Pillow<2.0 django==1.3.1 cssselect==0.9.1 - -rtests-requirements.txt + -rtests/requirements.txt [testenv:trunk] basepython = python2.7 commands = pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib - py.test --twisted {posargs:scrapy} + py.test {posargs:scrapy tests} [testenv:py33] basepython = python3.3 deps = ; svn+svn://svn.twistedmatrix.com/svn/Twisted/trunk#egg=Twisted - twisted >= 14.0.0 + Twisted >= 14.0.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9