Complete Azure Pipelines CI setup

azure-pipelines.yml

@@ -1,29 +1,23 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-
+variables:
+  TOXENV: py
 pool:
-  vmImage: 'windows-2019'
+  vmImage: 'windows-latest'
 strategy:
   matrix:
     Python35:
       python.version: '3.5'
-      TOXENV: py35
+      TOXENV: windows-pinned
     Python36:
       python.version: '3.6'
-      TOXENV: py36
     Python37:
       python.version: '3.7'
-      TOXENV: py37
-
+    Python38:
+      python.version: '3.8'
 steps:
 - task: UsePythonVersion@0
   inputs:
     versionSpec: '$(python.version)'
   displayName: 'Use Python $(python.version)'
-
 - script: |
     pip install -U tox twine wheel codecov
     tox

tests/CrawlerRunner/ip_address.py

@@ -1,7 +1,9 @@
 from urllib.parse import urlparse
 
 from twisted.internet import reactor
-from twisted.names.client import createResolver
+from twisted.names import cache, hosts as hostsModule, resolve
+from twisted.names.client import Resolver
+from twisted.python.runtime import platform
 
 from scrapy import Spider, Request
 from scrapy.crawler import CrawlerRunner
@@ -10,6 +12,17 @@ from scrapy.utils.log import configure_logging
 from tests.mockserver import MockServer, MockDNSServer
 
 
+# https://stackoverflow.com/a/32784190
+def createResolver(servers=None, resolvconf=None, hosts=None):
+    if hosts is None:
+        hosts = (b'/etc/hosts' if platform.getType() == 'posix'
+                 else r'c:\windows\hosts')
+    theResolver = Resolver(resolvconf, servers)
+    hostResolver = hostsModule.Resolver(hosts)
+    L = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(L)
+
+
 class LocalhostSpider(Spider):
     name = "localhost_spider"
 

tests/mockserver.py

@@ -247,9 +247,8 @@ class MockDNSServer:
     def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
                           stdout=PIPE, env=get_testenv())
-        host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
-        self.host = host
-        self.port = int(port)
+        self.host = '127.0.0.1'
+        self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1])
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):

tests/test_commands.py

@@ -2,6 +2,7 @@ import inspect
 import json
 import optparse
 import os
+import platform
 import subprocess
 import sys
 import tempfile
@@ -10,6 +11,7 @@ from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from threading import Timer
+from unittest import skipIf
 
 from twisted.trial import unittest
 
@@ -319,6 +321,9 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'

tests/test_crawler.py

@@ -4,6 +4,7 @@ import platform
 import subprocess
 import sys
 import warnings
+from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
@@ -252,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
 
     @defer.inlineCallbacks
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
@@ -293,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -327,6 +337,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)

tests/test_feedexport.py

@@ -455,9 +455,15 @@ class FeedExportTest(unittest.TestCase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def path_to_url(path):
+            return urljoin('file:', pathname2url(str(path)))
+
+        def printf_escape(string):
+            return string.replace('%', '%%')
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', pathname2url(str(file_path))): feed
+            printf_escape(path_to_url(file_path)): feed
             for file_path, feed in FEEDS.items()
         }
 

tests/test_proxy_connect.py

@@ -1,5 +1,6 @@
 import json
 import os
+import platform
 import re
 import sys
 from subprocess import Popen, PIPE
@@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url):
 
 @skipIf(sys.version_info < (3, 5, 4),
         "requires mitmproxy < 3.0.0, which these tests do not support")
+@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
+        "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):

tests/test_spiderloader/__init__.py

@@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
 
+def _copytree(source, target):
+    try:
+        shutil.copytree(source, target)
+    except shutil.Error:
+        pass
+
+
 class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
         self.tmpdir = tempfile.mkdtemp()
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
         self.spider_loader = SpiderLoader.from_settings(settings)
@@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
         self.tmpdir = self.mktemp()
         os.mkdir(self.tmpdir)
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
 
@@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
     def test_dupename_warning(self):
         # copy 1 spider module so as to have duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
@@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
         # This should issue 2 warning, 1 for each duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py'))
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)

tests/test_utils_asyncio.py

@@ -1,4 +1,6 @@
-from unittest import TestCase
+import platform
+import sys
+from unittest import skipIf, TestCase
 
 from pytest import mark
 
@@ -12,6 +14,9 @@ class AsyncioTest(TestCase):
         # the result should depend only on the pytest --reactor argument
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_install_asyncio_reactor(self):
         # this should do nothing
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

tox.ini

@@ -63,14 +63,12 @@ basepython = pypy3
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 
-[testenv:pinned]
-basepython = python3
+[pinned]
 deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==3.5.0
     parsel==1.5.0
     Protego==0.1.15
     PyDispatcher==2.0.5
@@ -85,6 +83,20 @@ deps =
     botocore==1.3.23
     Pillow==3.4.2
 
+[testenv:pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    lxml==3.5.0
+
+[testenv:windows-pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    # First lxml version that includes a Windows wheel for Python 3.5, so we do
+    # not need to build lxml from sources in a CI Windows job:
+    lxml==3.8.0
+
 [testenv:extra-deps]
 deps =
     {[testenv]deps}