From a175b6efc319ecbfd03b19705c25bd8bd44ca339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 27 Mar 2020 02:10:10 -0300 Subject: [PATCH 01/16] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..9bc324c32 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,32 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + + +pool: + vmImage: 'windows-2019' +strategy: + matrix: + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From 02206e5ffe74fe8107c272ad35920928544323b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 27 Mar 2020 02:20:39 -0300 Subject: [PATCH 02/16] Run tox --- azure-pipelines.yml | 9 ++++++--- tests/requirements-py3.txt | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9bc324c32..489cfe53b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,10 +10,13 @@ strategy: matrix: Python35: python.version: '3.5' + TOXENV: py35 Python36: python.version: '3.6' + TOXENV: py36 Python37: python.version: '3.7' + TOXENV: py37 steps: - task: UsePythonVersion@0 @@ -27,6 +30,6 @@ steps: displayName: 'Install dependencies' - script: | - pip install pytest pytest-azurepipelines - pytest - displayName: 'pytest' + pip install -U tox twine wheel codecov + tox + displayName: 'Run test suite' diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..8896f4614 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,6 +6,7 @@ pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist +pytest-azurepipelines sybil testfixtures From 0699e6bb1600ff943e131a3b7a299aa038145198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 27 Mar 2020 02:22:05 -0300 Subject: [PATCH 03/16] no need to install requirements.txt --- azure-pipelines.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 489cfe53b..ffc4d549b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,11 +24,6 @@ steps: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' -- script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' - - script: | pip install -U tox twine wheel codecov tox From 3fb0027138ab44b16be6e11677626d69b8c90c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 28 Mar 2020 17:36:50 +0100 Subject: [PATCH 04/16] =?UTF-8?q?Require=20sybil=20=E2=89=A5=201.3.0=20for?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..e7c86e0e9 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,7 +6,7 @@ pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist -sybil +sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures # optional for shell wrapper tests From 2f510fd47d8d10217f7b18b531205dd8c252eaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 21:10:05 +0200 Subject: [PATCH 05/16] Fix ShellTest.test_local_file on Windows --- scrapy/utils/url.py | 13 +++++++------ tests/test_command_shell.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c9abb12d5..c29ed4461 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -85,11 +85,9 @@ def add_http_if_no_scheme(url): def guess_scheme(url): - """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - parts = urlparse(url) - if parts.scheme: - return url - # Note: this does not match Windows filepath + """Add an URL scheme if missing: file:// for filepath-like input or + http:// otherwise.""" + # POSIX path if re.match(r'''^ # start with... ( \. # ...a single dot, @@ -99,7 +97,10 @@ def guess_scheme(url): )? # optional match of ".", ".." or ".blabla" / # at least one "/" for a file path, . # and something after the "/" - ''', parts.path, flags=re.VERBOSE): + ''', url, flags=re.VERBOSE): + return any_to_uri(url) + # Windows drive-letter path + elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE): return any_to_uri(url) else: return add_http_if_no_scheme(url) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..acf8e9f71 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -94,7 +94,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_file(self): - filepath = join(tests_datadir, 'test_site/index.html') + filepath = join(tests_datadir, 'test_site', 'index.html') _, out, _ = yield self.execute([filepath, '-c', 'item']) assert b'{}' in out From 94ee68695a42ec8d3ccdff71bfc2fa33d5ea7049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 15:36:56 +0200 Subject: [PATCH 06/16] Mock server: use 127.0.0.1 also for HTTPS Windows throws an error about 0.0.0.0 being external: https://stackoverflow.com/a/23857995/939364 --- tests/mockserver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..e3dbdcc68 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -218,9 +218,8 @@ class MockServer(): self.proc.communicate() def url(self, path, is_secure=False): - host = self.http_address.replace('0.0.0.0', '127.0.0.1') - if is_secure: - host = self.https_address + host = self.https_address if is_secure else self.http_address + host = host.replace('0.0.0.0', '127.0.0.1') return host + path From 7cc9601029274124804e63ebebfa8783ac175205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 16:57:48 +0200 Subject: [PATCH 07/16] Improve reporting on test_ipv6_alternative_name_resolver --- tests/test_crawler.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 169e763f0..a6b079395 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -309,14 +309,8 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)") + self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,") def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From cf4180308982d9cd017167238a3df8de7900646d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 17:07:29 +0200 Subject: [PATCH 08/16] Skip test_reactor_poll on Windows --- tests/test_crawler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a6b079395..3d166e14c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,6 @@ import logging import os +import platform import subprocess import sys import warnings @@ -317,6 +318,7 @@ class CrawlerProcessSubprocess(unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows") def test_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") self.assertIn("Spider closed (finished)", log) From ea3e675801fe41c7517c58e21b46831940fbd064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 17:10:45 +0200 Subject: [PATCH 09/16] test_utils_iterators: use os.linesep --- tests/test_utils_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 33fc4d570..5b0073fd1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -8,7 +8,7 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = u"foo\nbar" +FOOBAR_NL = "foo{}bar".format(os.linesep) class XmliterTestCase(unittest.TestCase): From 1fecacbb1a3c2c4c61c202561f9c51f7f6b191ba Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 20 Apr 2020 12:05:15 -0300 Subject: [PATCH 10/16] IPv6 test: check for the absence of DNSLookupError --- tests/test_crawler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 3d166e14c..d6756c266 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -310,8 +310,7 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)") - self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,") + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From 3199048520ebe3798c14cfa7362612b51d156b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 2 Jul 2020 20:10:08 +0200 Subject: [PATCH 11/16] Complete Azure Pipelines CI setup --- azure-pipelines.yml | 18 ++++++------------ tests/CrawlerRunner/ip_address.py | 15 ++++++++++++++- tests/mockserver.py | 5 ++--- tests/test_commands.py | 5 +++++ tests/test_crawler.py | 13 +++++++++++++ tests/test_feedexport.py | 8 +++++++- tests/test_proxy_connect.py | 3 +++ tests/test_spiderloader/__init__.py | 23 +++++++++++++++-------- tests/test_utils_asyncio.py | 7 ++++++- tox.ini | 18 +++++++++++++++--- 10 files changed, 86 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffc4d549b..710e42090 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,29 +1,23 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - - +variables: + TOXENV: py pool: - vmImage: 'windows-2019' + vmImage: 'windows-latest' strategy: matrix: Python35: python.version: '3.5' - TOXENV: py35 + TOXENV: windows-pinned Python36: python.version: '3.6' - TOXENV: py36 Python37: python.version: '3.7' - TOXENV: py37 - + Python38: + python.version: '3.8' steps: - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' - - script: | pip install -U tox twine wheel codecov tox diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 826374cd4..ea75bc3c9 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,7 +1,9 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names.client import createResolver +from twisted.names import cache, hosts as hostsModule, resolve +from twisted.names.client import Resolver +from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner @@ -10,6 +12,17 @@ from scrapy.utils.log import configure_logging from tests.mockserver import MockServer, MockDNSServer +# https://stackoverflow.com/a/32784190 +def createResolver(servers=None, resolvconf=None, hosts=None): + if hosts is None: + hosts = (b'/etc/hosts' if platform.getType() == 'posix' + else r'c:\windows\hosts') + theResolver = Resolver(resolvconf, servers) + hostResolver = hostsModule.Resolver(hosts) + L = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(L) + + class LocalhostSpider(Spider): name = "localhost_spider" diff --git a/tests/mockserver.py b/tests/mockserver.py index df30feab6..1f40473ba 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -247,9 +247,8 @@ class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) - host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") - self.host = host - self.port = int(port) + self.host = '127.0.0.1' + self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1]) return self def __exit__(self, exc_type, exc_value, traceback): diff --git a/tests/test_commands.py b/tests/test_commands.py index 24a341759..ee0e4511a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import inspect import json import optparse import os +import platform import subprocess import sys import tempfile @@ -10,6 +11,7 @@ from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp from threading import Timer +from unittest import skipIf from twisted.trial import unittest @@ -319,6 +321,9 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 78704fb2c..1a4cfe813 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import platform import subprocess import sys import warnings +from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -252,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -293,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -327,6 +337,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e38644214..f7b997560 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -455,9 +455,15 @@ class FeedExportTest(unittest.TestCase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + def printf_escape(string): + return string.replace('%', '%%') + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', pathname2url(str(file_path))): feed + printf_escape(path_to_url(file_path)): feed for file_path, feed in FEEDS.items() } diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index eb4ecc91d..fc5658ae7 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,5 +1,6 @@ import json import os +import platform import re import sys from subprocess import Popen, PIPE @@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url): @skipIf(sys.version_info < (3, 5, 4), "requires mitmproxy < 3.0.0, which these tests do not support") +@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), + "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d922c6059..4929f1e3e 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner module_dir = os.path.dirname(os.path.abspath(__file__)) +def _copytree(source, target): + try: + shutil.copytree(source, target) + except shutil.Error: + pass + + class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.spider_loader = SpiderLoader.from_settings(settings) @@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.tmpdir = self.mktemp() os.mkdir(self.tmpdir) self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) @@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4d..a2114bd18 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,4 +1,6 @@ -from unittest import TestCase +import platform +import sys +from unittest import skipIf, TestCase from pytest import mark @@ -12,6 +14,9 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tox.ini b/tox.ini index 27d21ade2..f729327ca 100644 --- a/tox.ini +++ b/tox.ini @@ -63,14 +63,12 @@ basepython = pypy3 commands = py.test {posargs:--durations=10 docs scrapy tests} -[testenv:pinned] -basepython = python3 +[pinned] deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 - lxml==3.5.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 @@ -85,6 +83,20 @@ deps = botocore==1.3.23 Pillow==3.4.2 +[testenv:pinned] +basepython = python3 +deps = + {[pinned]deps} + lxml==3.5.0 + +[testenv:windows-pinned] +basepython = python3 +deps = + {[pinned]deps} + # First lxml version that includes a Windows wheel for Python 3.5, so we do + # not need to build lxml from sources in a CI Windows job: + lxml==3.8.0 + [testenv:extra-deps] deps = {[testenv]deps} From eb937742566105f3525a9f76e4ae68cc18e9fd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 3 Jul 2020 01:41:47 +0200 Subject: [PATCH 12/16] TrackrefTestCase.test_get_oldest: protect from lack of precision --- tests/test_utils_trackref.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 16e02f919..b8e8c3130 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,7 +1,10 @@ import unittest from io import StringIO +from time import sleep, time from unittest import mock +from twisted.trial.unittest import SkipTest + from scrapy.utils import trackref @@ -55,7 +58,18 @@ Foo 1 oldest: 0s ago\n\n''') def test_get_oldest(self): o1 = Foo() # NOQA + + o1_time = time() + o2 = Bar() # NOQA + + o3_time = time() + if o3_time <= o1_time: + sleep(0.01) + o3_time = time() + if o3_time <= o1_time: + raise SkipTest('time.time is not precise enough') + o3 = Foo() # NOQA self.assertIs(trackref.get_oldest('Foo'), o1) self.assertIs(trackref.get_oldest('Bar'), o2) From ec06cf79a6a7264ec3e32d7de8c4e305c2afa05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 6 Jul 2020 10:47:11 +0200 Subject: [PATCH 13/16] Update tests/CrawlerRunner/ip_address.py Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- tests/CrawlerRunner/ip_address.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index ea75bc3c9..b8254afdf 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -15,8 +15,7 @@ from tests.mockserver import MockServer, MockDNSServer # https://stackoverflow.com/a/32784190 def createResolver(servers=None, resolvconf=None, hosts=None): if hosts is None: - hosts = (b'/etc/hosts' if platform.getType() == 'posix' - else r'c:\windows\hosts') + hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) L = [hostResolver, cache.CacheResolver(), theResolver] From 17aec5944cab33b3cdcd497d2362cacbf7773e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 6 Jul 2020 10:47:25 +0200 Subject: [PATCH 14/16] Update tests/CrawlerRunner/ip_address.py Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- tests/CrawlerRunner/ip_address.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b8254afdf..3f9738798 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -18,8 +18,8 @@ def createResolver(servers=None, resolvconf=None, hosts=None): hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) - L = [hostResolver, cache.CacheResolver(), theResolver] - return resolve.ResolverChain(L) + chain = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(chain) class LocalhostSpider(Spider): From d54c4496ee57785f3d6f882e2d128bb64b6b262c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 13 Jul 2020 14:36:33 +0200 Subject: [PATCH 15/16] Refactor guess_scheme --- scrapy/utils/url.py | 60 ++++++++++++++++++++++++++++++----------- tests/test_utils_url.py | 33 +++++++++++++++++++++-- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 60e2b50eb..1e431d3bf 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -83,26 +83,54 @@ def add_http_if_no_scheme(url): return url +def _is_posix_path(string): + return bool( + re.match( + r''' + ^ # start with... + ( + \. # ...a single dot, + ( + \. | [^/\.]+ # optionally followed by + )? # either a second dot or some characters + | + ~ # $HOME + )? # optional match of ".", ".." or ".blabla" + / # at least one "/" for a file path, + . # and something after the "/" + ''', + string, + flags=re.VERBOSE, + ) + ) + + +def _is_windows_path(string): + return bool( + re.match( + r''' + ^ + ( + [a-z]:\\ + | \\\\ + ) + ''', + string, + flags=re.IGNORECASE | re.VERBOSE, + ) + ) + + +def _is_path(string): + return _is_posix_path(string) or _is_windows_path(string) + + def guess_scheme(url): """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - # POSIX path - if re.match(r'''^ # start with... - ( - \. # ...a single dot, - ( - \. | [^/\.]+ # optionally followed by - )? # either a second dot or some characters - )? # optional match of ".", ".." or ".blabla" - / # at least one "/" for a file path, - . # and something after the "/" - ''', url, flags=re.VERBOSE): + if _is_path(url): return any_to_uri(url) - # Windows drive-letter path - elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE): - return any_to_uri(url) - else: - return add_http_if_no_scheme(url) + return add_http_if_no_scheme(url) def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 09a6d6c70..6a5254d54 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,8 +1,14 @@ import unittest from scrapy.spiders import Spider -from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, strip_url) +from scrapy.utils.url import ( + add_http_if_no_scheme, + guess_scheme, + _is_path, + strip_url, + url_is_from_any_domain, + url_is_from_spider, +) __doctests__ = ['scrapy.utils.url'] @@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase): self.assertEqual(strip_url(i, origin_only=True), o) +class IsPathTestCase(unittest.TestCase): + + def test_path(self): + for input_value, output_value in ( + # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell + # Unix-like OS, Microsoft Windows / cmd.exe + ("/home/user/docs/Letter.txt", True), + ("./inthisdir", True), + ("../../greatgrandparent", True), + ("~/.rcinfo", True), + (r"C:\user\docs\Letter.txt", True), + ("/user/docs/Letter.txt", True), + (r"C:\Letter.txt", True), + (r"\\Server01\user\docs\Letter.txt", True), + (r"\\?\UNC\Server01\user\docs\Letter.txt", True), + (r"\\?\C:\user\docs\Letter.txt", True), + (r"C:\user\docs\somefile.ext:alternate_stream_name", True), + + (r"https://example.com", False), + ): + self.assertEqual(_is_path(input_value), output_value, input_value) + + if __name__ == "__main__": unittest.main() From 53c323b19d81784e6c376ce8b9602de24d8e3037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 13 Jul 2020 15:29:30 +0200 Subject: [PATCH 16/16] =?UTF-8?q?=5Fis=5Fpath=20=E2=86=92=20=5Fis=5Ffilesy?= =?UTF-8?q?stem=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/url.py | 4 ++-- tests/test_utils_url.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1e431d3bf..b23ddb459 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -121,14 +121,14 @@ def _is_windows_path(string): ) -def _is_path(string): +def _is_filesystem_path(string): return _is_posix_path(string) or _is_windows_path(string) def guess_scheme(url): """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - if _is_path(url): + if _is_filesystem_path(url): return any_to_uri(url) return add_http_if_no_scheme(url) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 6a5254d54..3a143ba2f 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,7 +4,7 @@ from scrapy.spiders import Spider from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, - _is_path, + _is_filesystem_path, strip_url, url_is_from_any_domain, url_is_from_spider, @@ -460,7 +460,7 @@ class IsPathTestCase(unittest.TestCase): (r"https://example.com", False), ): - self.assertEqual(_is_path(input_value), output_value, input_value) + self.assertEqual(_is_filesystem_path(input_value), output_value, input_value) if __name__ == "__main__":