diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..93cfd469e --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,24 @@ +platform: x86 +version: '{branch}-{build}' +environment: + matrix: + - PYTHON: "C:\\Python36" + TOX_ENV: py36 + +branches: + only: + - master + - /d+\.\d+\.\d+[\w\-]*$/ + +install: + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "pip install -U tox" + +build: false +skip_tags: true +test_script: + - "tox -e %TOX_ENV%" + +cache: + - '%LOCALAPPDATA%\pip\cache' diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index c17aaf442..67337c26e 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -107,8 +107,8 @@ class Command(ScrapyCommand): string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project %r, using template directory %r, created in:" % \ - (project_name, self.templates_dir)) + print("New Scrapy project '%s', using template directory '%s', " + "created in:" % (project_name, self.templates_dir)) print(" %s\n" % abspath(project_dir)) print("You can start your first spider with:") print(" cd %s" % project_dir) diff --git a/tests/mockserver.py b/tests/mockserver.py index f36ce3c44..bf62fe907 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -209,7 +209,7 @@ class MockServer(): time.sleep(0.2) def url(self, path, is_secure=False): - host = self.http_address + host = self.http_address.replace('0.0.0.0', '127.0.0.1') if is_secure: host = self.https_address return host + path diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 8d9ce5231..7c1aacd81 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -3,9 +3,10 @@ pytest-twisted pytest-cov==2.5.1 testfixtures jmespath -leveldb +leveldb; sys_platform != "win32" botocore # optional for shell wrapper tests bpython ipython brotlipy +pywin32; sys_platform == "win32" diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 10076bbca..68dfb1cca 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -52,7 +52,8 @@ class CmdlineTest(unittest.TestCase): stats.print_stats() out.seek(0) stats = out.read() - self.assertIn('scrapy/commands/version.py', stats) + self.assertIn(os.path.join('scrapy', 'commands', 'version.py'), + stats) self.assertIn('tottime', stats) finally: shutil.rmtree(path) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 66dd17110..02037b866 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,3 +1,4 @@ +import os from os.path import join, abspath from twisted.trial import unittest from twisted.internet import defer @@ -7,6 +8,11 @@ from scrapy.utils.python import to_native_str from tests.test_commands import CommandTest +def _textmode(bstr): + """Normalize input the same as writing to a file + and reading from it in text mode""" + return to_native_str(bstr).replace(os.linesep, '\n') + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' @@ -97,7 +103,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '-a', 'test_arg=1', '-c', 'parse', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_with_meta(self): @@ -106,13 +112,13 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--meta', raw_json_string, '-c', 'parse_request_with_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) _, _, stderr = yield self.execute(['--spider', self.spider_name, '-m', raw_json_string, '-c', 'parse_request_with_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks @@ -120,7 +126,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} _, _, stderr = yield self.execute(['--spider', self.spider_name, '-c', 'parse_request_without_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks @@ -129,29 +135,29 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--pipelines', '-c', 'parse', self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) + self.assertIn("INFO: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find callback""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): @@ -159,7 +165,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): @@ -167,7 +173,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) @defer.inlineCallbacks def test_spider_with_no_rules_attribute(self): @@ -175,15 +181,15 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -191,5 +197,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7d9071b64..b8445ae6c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,18 +3,17 @@ import os import sys import subprocess import tempfile -from time import sleep from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp from contextlib import contextmanager +from threading import Timer from twisted.trial import unittest from twisted.internet import defer import scrapy from scrapy.utils.python import to_native_str -from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest @@ -46,16 +45,18 @@ class ProjectTest(unittest.TestCase): stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_kwargs) - waited = 0 - interval = 0.2 - while p.poll() is None: - sleep(interval) - waited += interval - if waited > 15: - p.kill() - assert False, 'Command took too much time to complete' + def kill_proc(): + p.kill() + assert False, 'Command took too much time to complete' - return p + timer = Timer(15, kill_proc) + try: + timer.start() + stdout, stderr = p.communicate() + finally: + timer.cancel() + + return p, to_native_str(stdout), to_native_str(stderr) class StartprojectTest(ProjectTest): @@ -111,9 +112,9 @@ class StartprojectTemplatesTest(ProjectTest): assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] - p = self.proc('startproject', self.project_name, *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) - self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) + p, out, err = self.proc('startproject', self.project_name, *args) + self.assertIn("New Scrapy project '%s', using template directory" + % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -140,12 +141,10 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = ['--template=%s' % tplname] if tplname else [] spname = 'test_spider' - p = self.proc('genspider', spname, 'test.com', *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - p = self.proc('genspider', spname, 'test.com', *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) def test_template_basic(self): @@ -212,8 +211,8 @@ class MySpider(scrapy.Spider): return self.proc('runspider', fname, *args) def get_log(self, code, name='myspider.py', args=()): - p = self.runspider(code, name=name, args=args) - return to_native_str(p.stderr.read()) + p, stdout, stderr = self.runspider(code, name=name, args=args) + return stderr def test_runspider(self): log = self.get_log(self.debug_log_spider) @@ -223,12 +222,12 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Spider closed (finished)", log) def test_run_fail_spider(self): - proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) ret = proc.returncode self.assertNotEqual(ret, 0) def test_run_good_spider(self): - proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) ret = proc.returncode self.assertEqual(ret, 0) @@ -279,8 +278,7 @@ class MySpider(scrapy.Spider): self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): - p = self.proc('runspider', 'some_non_existent_file') - log = to_native_str(p.stderr.read()) + _, _, log = self.proc('runspider', 'some_non_existent_file') self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): @@ -304,8 +302,7 @@ class BadSpider(scrapy.Spider): class BenchCommandTest(CommandTest): def test_run(self): - p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', - '-s', 'CLOSESPIDER_TIMEOUT=0.01') - log = to_native_str(p.stderr.read()) + _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', + '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6a8e11363..268948a70 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,10 +1,9 @@ import logging import tempfile import warnings -import unittest from twisted.internet import defer -import twisted.trial.unittest +from twisted.trial import unittest import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess @@ -94,26 +93,29 @@ class CrawlerLoggingTestCase(unittest.TestCase): assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): - with tempfile.NamedTemporaryFile() as log_file: - class MySpider(scrapy.Spider): - name = 'spider' - custom_settings = { - 'LOG_LEVEL': 'INFO', - 'LOG_FILE': log_file.name, - # disable telnet if not available to avoid an extra warning - 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, - } + log_file = self.mktemp() + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_LEVEL': 'INFO', + 'LOG_FILE': log_file, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + crawler = Crawler(MySpider, {}) + self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value('log_count/INFO') + logging.debug('debug message') + logging.info('info message') + logging.warning('warning message') + logging.error('error message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf8') - configure_logging() - self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) - crawler = Crawler(MySpider, {}) - self.assertEqual(get_scrapy_root_handler().level, logging.INFO) - info_count = crawler.stats.get_value('log_count/INFO') - logging.debug('debug message') - logging.info('info message') - logging.warning('warning message') - logging.error('error message') - logged = log_file.read().decode('utf8') self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -141,9 +143,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): settings = Settings({ 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' }) - with warnings.catch_warnings(record=True) as w, \ - self.assertRaises(AttributeError): - CrawlerRunner(settings) + with warnings.catch_warnings(record=True) as w: + self.assertRaises(AttributeError, CrawlerRunner, settings) self.assertEqual(len(w), 1) self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message)) self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) @@ -203,7 +204,7 @@ class NoRequestsSpider(scrapy.Spider): return [] -class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): +class CrawlerRunnerHasSpider(unittest.TestCase): @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c91be2c0c..2f8973054 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,7 +1,8 @@ import os import six -import contextlib import shutil +import tempfile +import contextlib try: from unittest import mock except ImportError: @@ -913,7 +914,9 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - local_fname = b"/tmp/file.txt" + f, local_fname = tempfile.mkstemp() + local_fname = to_bytes(local_fname) + os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, @@ -922,7 +925,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']}) + self.assertEqual(r.headers, {b'Local Filename': [local_fname], + b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2b57449d9..e46c8c14e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2,11 +2,12 @@ from __future__ import absolute_import import os import csv import json +import warnings from io import BytesIO import tempfile import shutil -from six.moves.urllib.parse import urlparse -import warnings +from six.moves.urllib.parse import urljoin, urlparse +from six.moves.urllib.request import pathname2url from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -226,9 +227,10 @@ class FeedExportTest(unittest.TestCase): def run_and_export(self, spider_cls, settings=None): """ Run spider with specified settings; return exported data. """ tmpdir = tempfile.mkdtemp() - res_name = tmpdir + '/res' + res_path = os.path.join(tmpdir, 'res') + res_uri = urljoin('file:', pathname2url(res_path)) defaults = { - 'FEED_URI': 'file://' + res_name, + 'FEED_URI': res_uri, 'FEED_FORMAT': 'csv', } defaults.update(settings or {}) @@ -238,11 +240,13 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(res_name, 'rb') as f: - defer.returnValue(f.read()) + with open(res_path, 'rb') as f: + content = f.read() finally: - shutil.rmtree(tmpdir) + shutil.rmtree(tmpdir, ignore_errors=True) + + defer.returnValue(content) @defer.inlineCallbacks def exported_data(self, items, settings): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index b2e8610f8..f953076b8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,7 +7,7 @@ from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = u"foo" + os.linesep + u"bar" +FOOBAR_NL = u"foo\nbar" class XmliterTestCase(unittest.TestCase): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 7e2caace8..bd74b0c34 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -25,8 +25,12 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEqual('.scrapy/somepath', data_path('somepath')) - self.assertEqual('/absolute/path', data_path('/absolute/path')) + self.assertEqual( + os.path.join('.scrapy', 'somepath'), + data_path('somepath') + ) + abspath = os.path.join(os.path.sep, 'absolute', 'path') + self.assertEqual(abspath, data_path(abspath)) def test_data_path_inside_project(self): with inside_a_project() as proj_path: @@ -35,4 +39,5 @@ class ProjectUtilsTest(unittest.TestCase): os.path.realpath(expected), os.path.realpath(data_path('somepath')) ) - self.assertEqual('/absolute/path', data_path('/absolute/path')) + abspath = os.path.join(os.path.sep, 'absolute', 'path') + self.assertEqual(abspath, data_path(abspath))