From 1d25c98eb337763f613d73fd534d3dc5e64eb66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 16:41:53 -0300 Subject: [PATCH 01/21] Add appveyor.yml --- appveyor.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..81432be39 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,14 @@ +platform: x86 +version: '{branch}-{build}' +environment: + matrix: + - PYTHON: "C:\\Python36" + TOX_ENV: py36 + +install: + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "pip install -U tox twine wheel" +build: false +test_script: + - "tox -e %TOX_ENV%" From 4c53957f5bd0f64b00d9488f171f0f8a9620af02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 16:56:05 -0300 Subject: [PATCH 02/21] Skip leveldb tests on windows --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 8d9ce5231..1e4a4b641 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -3,7 +3,7 @@ pytest-twisted pytest-cov==2.5.1 testfixtures jmespath -leveldb +leveldb; sys_platform != "win32" botocore # optional for shell wrapper tests bpython From dd75297e3fb37800c0ef763003a54081e9fad4cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 16:58:02 -0300 Subject: [PATCH 03/21] Run Appveyor CI for master and release branches only, but also PRs --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 81432be39..4f3c69847 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,6 +5,11 @@ environment: - PYTHON: "C:\\Python36" TOX_ENV: py36 +branches: + only: + - master + - /d+\.\d+\.\d+[\w\-]*$/ + install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" From 19ad94105f70a26ce47683d488b66fb77b79067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 17:15:48 -0300 Subject: [PATCH 04/21] pywin32 is required to run tests under windows --- tests/requirements-py3.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 1e4a4b641..7c1aacd81 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -9,3 +9,4 @@ botocore bpython ipython brotlipy +pywin32; sys_platform == "win32" From 152fde70b12f3c6e0e73230370f79f4b4a7ea906 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 2 Feb 2016 18:23:23 +0000 Subject: [PATCH 05/21] Fix FTPTestCase by using Windows-friendly temporary file name --- tests/test_downloader_handlers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c91be2c0c..fe76989f4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,7 +1,8 @@ import os import six -import contextlib import shutil +import tempfile +import contextlib try: from unittest import mock except ImportError: @@ -913,7 +914,8 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - local_fname = b"/tmp/file.txt" + f, local_fname = tempfile.mkstemp() + os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, @@ -922,7 +924,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']}) + self.assertEqual(r.headers, {b'Local Filename': [local_fname], + b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") From 57a1d66c61423c85703a4008d351e9d566580810 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 16:56:43 +0000 Subject: [PATCH 06/21] Fix test issues caused by Windows pipe buffer filling up --- tests/test_commands.py | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7d9071b64..84c38c0e9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,18 +3,17 @@ import os import sys import subprocess import tempfile -from time import sleep from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp from contextlib import contextmanager +from threading import Timer from twisted.trial import unittest from twisted.internet import defer import scrapy from scrapy.utils.python import to_native_str -from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest @@ -46,16 +45,18 @@ class ProjectTest(unittest.TestCase): stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_kwargs) - waited = 0 - interval = 0.2 - while p.poll() is None: - sleep(interval) - waited += interval - if waited > 15: - p.kill() - assert False, 'Command took too much time to complete' + def kill_proc(): + p.kill() + assert False, 'Command took too much time to complete' - return p + timer = Timer(15, kill_proc) + try: + timer.start() + stdout, stderr = p.communicate() + finally: + timer.cancel() + + return to_native_str(stdout), to_native_str(stderr) class StartprojectTest(ProjectTest): @@ -111,8 +112,7 @@ class StartprojectTemplatesTest(ProjectTest): assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] - p = self.proc('startproject', self.project_name, *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + out, err = self.proc('startproject', self.project_name, *args) self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -140,12 +140,10 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = ['--template=%s' % tplname] if tplname else [] spname = 'test_spider' - p = self.proc('genspider', spname, 'test.com', *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - p = self.proc('genspider', spname, 'test.com', *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) def test_template_basic(self): @@ -212,8 +210,8 @@ class MySpider(scrapy.Spider): return self.proc('runspider', fname, *args) def get_log(self, code, name='myspider.py', args=()): - p = self.runspider(code, name=name, args=args) - return to_native_str(p.stderr.read()) + stdout, stderr = self.runspider(code, name=name, args=args) + return stderr def test_runspider(self): log = self.get_log(self.debug_log_spider) @@ -279,14 +277,17 @@ class MySpider(scrapy.Spider): self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): - p = self.proc('runspider', 'some_non_existent_file') - log = to_native_str(p.stderr.read()) + _, log = self.proc('runspider', 'some_non_existent_file') self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) + +class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + command = 'parse' + def test_start_requests_errors(self): log = self.get_log(""" import scrapy @@ -304,8 +305,7 @@ class BadSpider(scrapy.Spider): class BenchCommandTest(CommandTest): def test_run(self): - p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', - '-s', 'CLOSESPIDER_TIMEOUT=0.01') - log = to_native_str(p.stderr.read()) + _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', + '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) From ed8255bde04d7e07747792022858f161c2296096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 6 Jul 2018 17:53:56 -0300 Subject: [PATCH 07/21] Fix merge issues with stderr/out fixes for windows buffering --- tests/test_commands.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 84c38c0e9..4963ef99c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -56,7 +56,7 @@ class ProjectTest(unittest.TestCase): finally: timer.cancel() - return to_native_str(stdout), to_native_str(stderr) + return p, to_native_str(stdout), to_native_str(stderr) class StartprojectTest(ProjectTest): @@ -112,7 +112,7 @@ class StartprojectTemplatesTest(ProjectTest): assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] - out, err = self.proc('startproject', self.project_name, *args) + p, out, err = self.proc('startproject', self.project_name, *args) self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -140,10 +140,10 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = ['--template=%s' % tplname] if tplname else [] spname = 'test_spider' - out, err = self.proc('genspider', spname, 'test.com', *args) + p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - out, err = self.proc('genspider', spname, 'test.com', *args) + p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) def test_template_basic(self): @@ -210,7 +210,7 @@ class MySpider(scrapy.Spider): return self.proc('runspider', fname, *args) def get_log(self, code, name='myspider.py', args=()): - stdout, stderr = self.runspider(code, name=name, args=args) + p, stdout, stderr = self.runspider(code, name=name, args=args) return stderr def test_runspider(self): @@ -221,12 +221,12 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Spider closed (finished)", log) def test_run_fail_spider(self): - proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) ret = proc.returncode self.assertNotEqual(ret, 0) def test_run_good_spider(self): - proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) ret = proc.returncode self.assertEqual(ret, 0) @@ -277,7 +277,7 @@ class MySpider(scrapy.Spider): self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): - _, log = self.proc('runspider', 'some_non_existent_file') + _, _, log = self.proc('runspider', 'some_non_existent_file') self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): @@ -305,7 +305,7 @@ class BadSpider(scrapy.Spider): class BenchCommandTest(CommandTest): def test_run(self): - _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', + _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) From 034152961d187930eb659bef9e87ceb3e589c103 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 18:12:08 +0000 Subject: [PATCH 08/21] Fix Feedexport test in Windows by using proper file URI --- tests/test_feedexport.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2b57449d9..6eefa14bf 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2,11 +2,12 @@ from __future__ import absolute_import import os import csv import json +import warnings from io import BytesIO import tempfile import shutil -from six.moves.urllib.parse import urlparse -import warnings +from six.moves.urllib.parse import urljoin, urlparse +from six.moves.urllib.request import pathname2url from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -226,9 +227,10 @@ class FeedExportTest(unittest.TestCase): def run_and_export(self, spider_cls, settings=None): """ Run spider with specified settings; return exported data. """ tmpdir = tempfile.mkdtemp() - res_name = tmpdir + '/res' + res_path = os.path.join(tmpdir, 'res') + res_uri = urljoin('file:', pathname2url(res_path)) defaults = { - 'FEED_URI': 'file://' + res_name, + 'FEED_URI': res_uri, 'FEED_FORMAT': 'csv', } defaults.update(settings or {}) @@ -238,7 +240,7 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(res_name, 'rb') as f: + with open(res_path, 'rb') as f: defer.returnValue(f.read()) finally: From 22505a34a9d0095dfc6e133bbb79d3b7aa651082 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 18:42:51 +0000 Subject: [PATCH 09/21] Fix cmdline profiling test on Windows by using proper path composing --- tests/test_cmdline/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 10076bbca..68dfb1cca 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -52,7 +52,8 @@ class CmdlineTest(unittest.TestCase): stats.print_stats() out.seek(0) stats = out.read() - self.assertIn('scrapy/commands/version.py', stats) + self.assertIn(os.path.join('scrapy', 'commands', 'version.py'), + stats) self.assertIn('tottime', stats) finally: shutil.rmtree(path) From fb09148c91f118e3a71243e527a0d39294ba59cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 6 Jul 2018 18:10:56 -0300 Subject: [PATCH 10/21] Fix bad merge on ParseCommandTest --- tests/test_commands.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 4963ef99c..78aa2a776 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -284,10 +284,6 @@ class MySpider(scrapy.Spider): log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) - -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' - def test_start_requests_errors(self): log = self.get_log(""" import scrapy From a21abac743865b628e52f11882db7b7ab70e0342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 10:15:20 -0300 Subject: [PATCH 11/21] fix ftp tests on windows --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fe76989f4..2f8973054 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -915,6 +915,7 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_local_filename(self): f, local_fname = tempfile.mkstemp() + local_fname = to_bytes(local_fname) os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) From ed068e59b746de6395c33f6f0f5454e69e88d1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 10:28:07 -0300 Subject: [PATCH 12/21] Cache pip cache and do not rebuild tags on appveyor and travis --- appveyor.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 4f3c69847..93cfd469e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,7 +13,12 @@ branches: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" - - "pip install -U tox twine wheel" + - "pip install -U tox" + build: false +skip_tags: true test_script: - "tox -e %TOX_ENV%" + +cache: + - '%LOCALAPPDATA%\pip\cache' From 0e532e3dd8b4dd07b1a287d6321c1e6ae6786c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 10:46:15 -0300 Subject: [PATCH 13/21] Creating a connection to 0.0.0.0 fails on windows but not on linux nor mac --- tests/mockserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index f36ce3c44..bf62fe907 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -209,7 +209,7 @@ class MockServer(): time.sleep(0.2) def url(self, path, is_secure=False): - host = self.http_address + host = self.http_address.replace('0.0.0.0', '127.0.0.1') if is_secure: host = self.https_address return host + path From ca53a8699a8cfc2a0a3be4a65d91bada8a43f94c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 11:07:05 -0300 Subject: [PATCH 14/21] Fix presentation of template directory in startproject command --- scrapy/commands/startproject.py | 4 ++-- tests/test_commands.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index c17aaf442..67337c26e 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -107,8 +107,8 @@ class Command(ScrapyCommand): string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project %r, using template directory %r, created in:" % \ - (project_name, self.templates_dir)) + print("New Scrapy project '%s', using template directory '%s', " + "created in:" % (project_name, self.templates_dir)) print(" %s\n" % abspath(project_dir)) print("You can start your first spider with:") print(" cd %s" % project_dir) diff --git a/tests/test_commands.py b/tests/test_commands.py index 78aa2a776..b8445ae6c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -113,7 +113,8 @@ class StartprojectTemplatesTest(ProjectTest): args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) + self.assertIn("New Scrapy project '%s', using template directory" + % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) From cb281757500b2e114fb554aa3bf618e3a8f8eb71 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 18:35:35 +0000 Subject: [PATCH 15/21] Fix csviter tests by explicitly using newline only --- tests/test_utils_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index b2e8610f8..f953076b8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,7 +7,7 @@ from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = u"foo" + os.linesep + u"bar" +FOOBAR_NL = u"foo\nbar" class XmliterTestCase(unittest.TestCase): From d93d960319e22badccd68499df11f2a728dbbc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 15 Aug 2018 01:53:20 -0300 Subject: [PATCH 16/21] Fix test_utils_project under Windows --- tests/test_utils_project.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 7e2caace8..bd74b0c34 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -25,8 +25,12 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEqual('.scrapy/somepath', data_path('somepath')) - self.assertEqual('/absolute/path', data_path('/absolute/path')) + self.assertEqual( + os.path.join('.scrapy', 'somepath'), + data_path('somepath') + ) + abspath = os.path.join(os.path.sep, 'absolute', 'path') + self.assertEqual(abspath, data_path(abspath)) def test_data_path_inside_project(self): with inside_a_project() as proj_path: @@ -35,4 +39,5 @@ class ProjectUtilsTest(unittest.TestCase): os.path.realpath(expected), os.path.realpath(data_path('somepath')) ) - self.assertEqual('/absolute/path', data_path('/absolute/path')) + abspath = os.path.join(os.path.sep, 'absolute', 'path') + self.assertEqual(abspath, data_path(abspath)) From 96517cb7de93da990f4f7dcb1cb7b7129e8c2064 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 15 Aug 2018 01:08:40 -0700 Subject: [PATCH 17/21] Fix test_command_parse under windows --- tests/test_command_parse.py | 38 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 66dd17110..02037b866 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,3 +1,4 @@ +import os from os.path import join, abspath from twisted.trial import unittest from twisted.internet import defer @@ -7,6 +8,11 @@ from scrapy.utils.python import to_native_str from tests.test_commands import CommandTest +def _textmode(bstr): + """Normalize input the same as writing to a file + and reading from it in text mode""" + return to_native_str(bstr).replace(os.linesep, '\n') + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' @@ -97,7 +103,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '-a', 'test_arg=1', '-c', 'parse', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_with_meta(self): @@ -106,13 +112,13 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--meta', raw_json_string, '-c', 'parse_request_with_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) _, _, stderr = yield self.execute(['--spider', self.spider_name, '-m', raw_json_string, '-c', 'parse_request_with_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks @@ -120,7 +126,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} _, _, stderr = yield self.execute(['--spider', self.spider_name, '-c', 'parse_request_without_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks @@ -129,29 +135,29 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--pipelines', '-c', 'parse', self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) + self.assertIn("INFO: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find callback""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): @@ -159,7 +165,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): @@ -167,7 +173,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) @defer.inlineCallbacks def test_spider_with_no_rules_attribute(self): @@ -175,15 +181,15 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -191,5 +197,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) From e7fe243c3e6f71660620441ec6a8ca4605d4333b Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 15 Aug 2018 01:09:23 -0700 Subject: [PATCH 18/21] Fix test_crawler under windows --- tests/test_crawler.py | 46 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6a8e11363..0aeb12e58 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,10 +1,9 @@ import logging import tempfile import warnings -import unittest from twisted.internet import defer -import twisted.trial.unittest +from twisted.trial import unittest import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess @@ -94,26 +93,29 @@ class CrawlerLoggingTestCase(unittest.TestCase): assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): - with tempfile.NamedTemporaryFile() as log_file: - class MySpider(scrapy.Spider): - name = 'spider' - custom_settings = { - 'LOG_LEVEL': 'INFO', - 'LOG_FILE': log_file.name, - # disable telnet if not available to avoid an extra warning - 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, - } + log_file = self.mktemp() + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_LEVEL': 'INFO', + 'LOG_FILE': log_file, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + crawler = Crawler(MySpider, {}) + self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value('log_count/INFO') + logging.debug('debug message') + logging.info('info message') + logging.warning('warning message') + logging.error('error message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf8') - configure_logging() - self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) - crawler = Crawler(MySpider, {}) - self.assertEqual(get_scrapy_root_handler().level, logging.INFO) - info_count = crawler.stats.get_value('log_count/INFO') - logging.debug('debug message') - logging.info('info message') - logging.warning('warning message') - logging.error('error message') - logged = log_file.read().decode('utf8') self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -203,7 +205,7 @@ class NoRequestsSpider(scrapy.Spider): return [] -class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): +class CrawlerRunnerHasSpider(unittest.TestCase): @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): From a304d6b692664f06c67fb23eb08fd985eda72e21 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 15 Aug 2018 02:02:20 -0700 Subject: [PATCH 19/21] Workaround to pass tests/test_feedexporter.py under windows --- tests/test_feedexport.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6eefa14bf..76452d450 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -241,10 +241,18 @@ class FeedExportTest(unittest.TestCase): yield runner.crawl(spider_cls) with open(res_path, 'rb') as f: - defer.returnValue(f.read()) + content = f.read() finally: - shutil.rmtree(tmpdir) + # FIXME: Windows fails to remove the file because FeedExporter + # keeps a reference to the temporal file even after + # the spider finished. + try: + shutil.rmtree(tmpdir) + except OSError: + pass + + defer.returnValue(content) @defer.inlineCallbacks def exported_data(self, items, settings): From 4eaf8690b14527d8d74ddd8103810de7c4dcdee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 15 Aug 2018 08:54:18 -0300 Subject: [PATCH 20/21] Twisted's unittest.Testcase assertRaiess can't be used as context manager --- tests/test_crawler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0aeb12e58..268948a70 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -143,9 +143,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): settings = Settings({ 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' }) - with warnings.catch_warnings(record=True) as w, \ - self.assertRaises(AttributeError): - CrawlerRunner(settings) + with warnings.catch_warnings(record=True) as w: + self.assertRaises(AttributeError, CrawlerRunner, settings) self.assertEqual(len(w), 1) self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message)) self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) From 38608bc2495189bd7de2dc48c0036df217138377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 15 Aug 2018 11:59:09 -0300 Subject: [PATCH 21/21] Use ignore_errors option from rmtree --- tests/test_feedexport.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 76452d450..e46c8c14e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -244,13 +244,7 @@ class FeedExportTest(unittest.TestCase): content = f.read() finally: - # FIXME: Windows fails to remove the file because FeedExporter - # keeps a reference to the temporal file even after - # the spider finished. - try: - shutil.rmtree(tmpdir) - except OSError: - pass + shutil.rmtree(tmpdir, ignore_errors=True) defer.returnValue(content)