From 80db569aeae3e05480ed228c95332c17f67e81e4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Fri, 1 Jan 2021 19:13:39 -0300
Subject: [PATCH] Migrate CI to GitHub actions (#4924)

---
 .github/workflows/checks.yml        | 38 ++++++++++++++
 .github/workflows/main.yml          | 31 ------------
 .github/workflows/publish.yml       | 31 ++++++++++++
 .github/workflows/tests-macos.yml   | 25 ++++++++++
 .github/workflows/tests-ubuntu.yml  | 69 ++++++++++++++++++++++++++
 .github/workflows/tests-windows.yml | 32 ++++++++++++
 .travis.yml                         | 77 -----------------------------
 README.rst                          | 14 ++++--
 tests/requirements-py3.txt          |  2 +-
 tox.ini                             | 24 ++-------
 10 files changed, 211 insertions(+), 132 deletions(-)
 create mode 100644 .github/workflows/checks.yml
 delete mode 100644 .github/workflows/main.yml
 create mode 100644 .github/workflows/publish.yml
 create mode 100644 .github/workflows/tests-macos.yml
 create mode 100644 .github/workflows/tests-ubuntu.yml
 create mode 100644 .github/workflows/tests-windows.yml
 delete mode 100644 .travis.yml

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
new file mode 100644
index 000000000..2748bf5fe
--- /dev/null
+++ b/.github/workflows/checks.yml
@@ -0,0 +1,38 @@
+name: Checks
+on: [push, pull_request]
+
+jobs:
+  checks:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+        - python-version: 3.8
+          env:
+            TOXENV: security
+        - python-version: 3.8
+          env:
+            TOXENV: flake8
+        - python-version: 3.8
+          env:
+            TOXENV: pylint
+        - python-version: 3.8
+          env:
+            TOXENV: typing
+        - python-version: 3.7  # Keep in sync with .readthedocs.yml
+          env:
+            TOXENV: docs
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run check
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index 28771216c..000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Run test suite
-on: [push, pull_request]
-
-jobs:
-  test-windows:
-    name: "Windows Tests"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest]
-        python-version: [3.7, 3.8]
-        env: [TOXENV: py]
-        include:
-        - os: windows-latest
-          python-version: 3.6
-          env:
-            TOXENV: windows-pinned
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Run test suite
-      env: ${{ matrix.env }}
-      run: |
-        pip install -U tox twine wheel codecov
-        tox
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 000000000..aec6b8696
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,31 @@
+name: Publish
+on: [push]
+
+jobs:
+  publish:
+    runs-on: ubuntu-18.04
+    if: startsWith(github.event.ref, 'refs/tags/')
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Check Tag
+      id: check-release-tag
+      run: |
+        if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
+          echo ::set-output name=release_tag::true
+        fi
+
+    - name: Publish to PyPI
+      if: steps.check-release-tag.outputs.release_tag == 'true'
+      run: |
+        pip install --upgrade setuptools wheel twine
+        python setup.py sdist bdist_wheel
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+        twine upload dist/*
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
new file mode 100644
index 000000000..51d27c405
--- /dev/null
+++ b/.github/workflows/tests-macos.yml
@@ -0,0 +1,25 @@
+name: macOS
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: macos-10.15
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run tests
+      run: |
+        pip install -U tox
+        tox -e py
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
new file mode 100644
index 000000000..89c0334e2
--- /dev/null
+++ b/.github/workflows/tests-ubuntu.yml
@@ -0,0 +1,69 @@
+name: Ubuntu
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+        - python-version: 3.7
+          env:
+            TOXENV: py
+        - python-version: 3.8
+          env:
+            TOXENV: py
+        - python-version: pypy3
+          env:
+            TOXENV: pypy3
+            PYPY_VERSION: 3.6-v7.3.1
+
+        # pinned deps
+        - python-version: 3.6.12
+          env:
+            TOXENV: pinned
+        - python-version: 3.6.12
+          env:
+            TOXENV: asyncio-pinned
+        - python-version: pypy3
+          env:
+            TOXENV: pypy3-pinned
+            PYPY_VERSION: 3.6-v7.2.0
+
+        # extras
+        - python-version: 3.8
+          env:
+            TOXENV: extra-deps
+        - python-version: 3.8
+          env:
+            TOXENV: asyncio
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system libraries
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+      run: |
+        sudo apt-get update
+        sudo apt-get install libxml2-dev libxslt-dev
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        if [[ ! -z "$PYPY_VERSION" ]]; then
+          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
+          tar -jxf ${PYPY_VERSION}.tar.bz2
+          $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
+          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        fi
+        pip install -U tox
+        tox
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
new file mode 100644
index 000000000..ed2e4075d
--- /dev/null
+++ b/.github/workflows/tests-windows.yml
@@ -0,0 +1,32 @@
+name: Windows
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: 3.6
+          env:
+            TOXENV: windows-pinned
+        - python-version: 3.7
+          env:
+            TOXENV: py
+        - python-version: 3.8
+          env:
+            TOXENV: py
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 880c8772b..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-language: python
-dist: xenial
-branches:
-  only:
-    - master
-    - /^\d\.\d+$/
-    - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
-matrix:
-  include:
-    - env: TOXENV=security
-      python: 3.8
-    - env: TOXENV=flake8
-      python: 3.8
-    - env: TOXENV=pylint
-      python: 3.8
-    - env: TOXENV=docs
-      python: 3.7  # Keep in sync with .readthedocs.yml
-    - env: TOXENV=typing
-      python: 3.8
-
-    - env: TOXENV=pinned
-      python: 3.6.1
-    - env: TOXENV=asyncio-pinned
-      python: 3.6.1
-    - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
-      dist: bionic
-
-    - env: TOXENV=py
-      python: 3.6
-    - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
-      dist: bionic
-
-    - env: TOXENV=py
-      python: 3.7
-
-    - env: TOXENV=py PYPI_RELEASE_JOB=true
-      python: 3.8
-      dist: bionic
-    - env: TOXENV=extra-deps
-      python: 3.8
-      dist: bionic
-    - env: TOXENV=asyncio
-      python: 3.8
-      dist: bionic
-install:
-  - |
-      if [[ ! -z "$PYPY_VERSION" ]]; then
-        export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
-        wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
-  - pip install -U tox twine wheel codecov
-
-script: tox
-after_success:
-  - codecov
-notifications:
-  irc:
-    use_notice: true
-    skip_join: true
-    channels:
-    - irc.freenode.org#scrapy
-cache:
-  directories:
-    - $HOME/.cache/pip
-deploy:
-  provider: pypi
-  distributions: "sdist bdist_wheel"
-  user: scrapy
-  password:
-    secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
-  on:
-    tags: true
-    repo: scrapy/scrapy
-    condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
diff --git a/README.rst b/README.rst
index a8f2ba52b..9418d270f 100644
--- a/README.rst
+++ b/README.rst
@@ -10,9 +10,17 @@ Scrapy
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: Supported Python Versions
 
-.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
-   :target: https://travis-ci.org/scrapy/scrapy
-   :alt: Build Status
+.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
+   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
+   :alt: Ubuntu
+
+.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
+   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
+   :alt: macOS
+
+.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
+   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
+   :alt: Windows
 
 .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
    :target: https://pypi.python.org/pypi/Scrapy
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 68b856a88..a86c4ae4f 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -14,6 +14,6 @@ uvloop; platform_system != "Windows"
 # optional for shell wrapper tests
 bpython
 brotlipy  # optional for HTTP compress downloader middleware tests
-zstandard  # optional for HTTP compress downloader middleware tests
+zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index 6c39299bb..e70aef2d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@ deps =
     -rtests/requirements-py3.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
-    mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
+    mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
@@ -25,9 +25,9 @@ passenv =
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 #allow tox virtualenv to upgrade pip/wheel/setuptools
-download = true 
+download = true
 commands =
-    py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
+    py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 
 [testenv:typing]
 basepython = python3
@@ -87,10 +87,6 @@ deps =
     botocore==1.4.87
     google-cloud-storage==1.29.0
     Pillow==4.0.0
-install_command =
-    # --use-feature=2020-resolver is required, otherwise the latest verion of
-    # Twisted gets installed.
-    pip install --use-feature=2020-resolver {opts} {packages}
 setenv =
     _SCRAPY_PINNED=true
 
@@ -99,11 +95,9 @@ deps =
     {[pinned]deps}
     lxml==3.5.0
     PyDispatcher==2.0.5
-install_command =
-    {[pinned]install_command}
 setenv =
     {[pinned]setenv}
-    
+
 [testenv:windows-pinned]
 basepython = python3
 deps =
@@ -112,8 +106,6 @@ deps =
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0
     PyDispatcher==2.0.5
-install_command =
-    {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 
@@ -122,10 +114,6 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
-install_command =
-    # Test --use-feature=2020-resolver for the latest version of all
-    # dependencies.
-    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:asyncio]
 commands =
@@ -133,8 +121,6 @@ commands =
 
 [testenv:asyncio-pinned]
 deps = {[testenv:pinned]deps}
-install_command =
-    {[pinned]install_command}
 commands = {[testenv:asyncio]commands}
 setenv =
     {[pinned]setenv}
@@ -150,8 +136,6 @@ deps =
     {[pinned]deps}
     lxml==4.0.0
     PyPyDispatcher==2.1.0
-install_command =
-    {[pinned]install_command}
 commands = {[testenv:pypy3]commands}
 setenv =
     {[pinned]setenv}