mirror of https://github.com/scrapy/scrapy.git
Travis CI → GitHub Actions
This commit is contained in:
Adrián Chaves
parent
be2e910dd0
commit
4183925b4b
|
|
@ -0,0 +1,27 @@
|
|||
name: Checks
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
checks:
|
||||
runs-on: ubuntu-18.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.8 # Keep in sync with .readthedocs.yml
|
||||
env:
|
||||
TOXENV: docs
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
name: Publish
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-18.04
|
||||
if: startsWith(github.event.ref, 'refs/tags/')
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.8
|
||||
|
||||
- name: Check Tag
|
||||
id: check-release-tag
|
||||
run: |
|
||||
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
|
||||
echo ::set-output name=release_tag::true
|
||||
fi
|
||||
|
||||
- name: Publish to PyPI
|
||||
if: steps.check-release-tag.outputs.release_tag == 'true'
|
||||
run: |
|
||||
pip install --upgrade setuptools wheel twine
|
||||
python setup.py sdist bdist_wheel
|
||||
export TWINE_USERNAME=__token__
|
||||
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
|
||||
twine upload dist/*
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
name: Ubuntu
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-18.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 2.7
|
||||
env:
|
||||
TOXENV: py27
|
||||
- python-version: 2.7
|
||||
env:
|
||||
TOXENV: py27-pinned
|
||||
- python-version: 2.7
|
||||
env:
|
||||
TOXENV: py27-extra-deps
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy
|
||||
PYPY_VERSION: '2.7-v7.3.5'
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: '3.7-v7.3.5'
|
||||
- python-version: 3.5
|
||||
env:
|
||||
TOXENV: py35
|
||||
- python-version: 3.5
|
||||
env:
|
||||
TOXENV: py35-pinned
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: py36
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py37
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py38
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py38-extra-deps
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install system libraries
|
||||
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
|
||||
sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
if [ "${PYPY_VERSION:4:1}" = "2" ]; then
|
||||
pip install -U virtualenv
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
else
|
||||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
fi
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
- name: Upload coverage report
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
version: 2
|
||||
formats: all
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
fail_on_warning: true
|
||||
|
||||
build:
|
||||
image: latest
|
||||
|
||||
python:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
|
||||
version: 3.8 # Keep in sync with .github/workflows/checks.yml
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- path: .
|
||||
73
.travis.yml
73
.travis.yml
|
|
@ -1,73 +0,0 @@
|
|||
language: python
|
||||
dist: xenial
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^\d\.\d+$/
|
||||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=py27
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-pinned
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-extra-deps
|
||||
python: 2.7
|
||||
- env: TOXENV=pypy
|
||||
python: 2.7
|
||||
- env: TOXENV=pypy3
|
||||
python: 3.5
|
||||
- env: TOXENV=py35
|
||||
python: 3.5
|
||||
- env: TOXENV=py35-pinned
|
||||
python: 3.5
|
||||
- env: TOXENV=py36
|
||||
python: 3.6
|
||||
- env: TOXENV=py37
|
||||
python: 3.7
|
||||
- env: TOXENV=py38
|
||||
python: 3.8
|
||||
- env: TOXENV=py38-extra-deps
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.6
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy" ]; then
|
||||
export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
- pip install -U tox twine wheel codecov
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- codecov
|
||||
notifications:
|
||||
irc:
|
||||
use_notice: true
|
||||
skip_join: true
|
||||
channels:
|
||||
- irc.freenode.org#scrapy
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
deploy:
|
||||
provider: pypi
|
||||
distributions: "sdist bdist_wheel"
|
||||
user: scrapy
|
||||
password:
|
||||
secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
|
||||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
|
|
@ -828,7 +828,9 @@ REDIRECT_MAX_TIMES
|
|||
|
||||
Default: ``20``
|
||||
|
||||
The maximum number of redirections that will be followed for a single request.
|
||||
Defines the maximum times a request can be redirected. After this maximum the
|
||||
request's response is returned as is. We used Firefox default value for the
|
||||
same task.
|
||||
|
||||
MetaRefreshMiddleware
|
||||
---------------------
|
||||
|
|
|
|||
|
|
@ -1104,17 +1104,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
|
|||
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
|
||||
this value if you're experiencing problems with insufficient blocking IO.
|
||||
|
||||
.. setting:: REDIRECT_MAX_TIMES
|
||||
|
||||
REDIRECT_MAX_TIMES
|
||||
------------------
|
||||
|
||||
Default: ``20``
|
||||
|
||||
Defines the maximum times a request can be redirected. After this maximum the
|
||||
request's response is returned as is. We used Firefox default value for the
|
||||
same task.
|
||||
|
||||
.. setting:: REDIRECT_PRIORITY_ADJUST
|
||||
|
||||
REDIRECT_PRIORITY_ADJUST
|
||||
|
|
@ -1396,17 +1385,6 @@ Default: ``True``
|
|||
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
|
||||
will be enabled (provided its extension is also enabled).
|
||||
|
||||
.. setting:: TELNETCONSOLE_PORT
|
||||
|
||||
TELNETCONSOLE_PORT
|
||||
------------------
|
||||
|
||||
Default: ``[6023, 6073]``
|
||||
|
||||
The port range to use for the telnet console. If set to ``None`` or ``0``, a
|
||||
dynamically assigned port is used. For more info see
|
||||
:ref:`topics-telnetconsole`.
|
||||
|
||||
.. setting:: TEMPLATES_DIR
|
||||
|
||||
TEMPLATES_DIR
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ PyDispatcher>=2.0.5
|
|||
w3lib>=1.17.0
|
||||
protego>=0.1.15
|
||||
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2,<1.6.0 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
# Greater versions implement peek support, which breaks tests.
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
|
||||
# Reference versions taken from
|
||||
# https://packages.ubuntu.com/xenial/python/
|
||||
|
|
@ -14,5 +15,5 @@ cssselect>=0.9.1
|
|||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
Twisted>=16.0.0
|
||||
zope.interface>=4.1.3
|
||||
Twisted>=16.0.0,<21.2.0 # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020
|
||||
zope.interface>=4.1.3,<5 # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
parsel>=1.5.0
|
||||
PyDispatcher>=2.0.5
|
||||
Twisted>=17.9.0
|
||||
Twisted>=17.9.0,<21.2.0 # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020
|
||||
w3lib>=1.17.0
|
||||
protego>=0.1.15
|
||||
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2,<1.6.0 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
# Greater versions implement peek support, which breaks tests.
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
|
||||
# Reference versions taken from
|
||||
# https://packages.ubuntu.com/xenial/python/
|
||||
|
|
@ -15,4 +16,4 @@ cssselect>=0.9.1
|
|||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
zope.interface>=4.1.3
|
||||
zope.interface>=4.1.3,<5 # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import unittest
|
|||
from itertools import count
|
||||
import platform
|
||||
import six
|
||||
from sys import version_info
|
||||
|
||||
from scrapy.utils.python import (
|
||||
memoizemethod_noargs, binary_is_text, equal_attributes,
|
||||
|
|
@ -234,10 +235,12 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
stripself = not six.PY2 # PyPy3 exposes them as methods
|
||||
self.assertEqual(
|
||||
get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
|
||||
self.assertEqual(get_func_args(" ".join, stripself), ['list'])
|
||||
self.assertEqual(
|
||||
get_func_args(operator.itemgetter(2), stripself), ['obj'])
|
||||
|
||||
if version_info < (3, 6):
|
||||
self.assertEqual(get_func_args(" ".join, stripself), ['list'])
|
||||
else:
|
||||
self.assertEqual(get_func_args(" ".join, stripself), ['iterable'])
|
||||
|
||||
def test_without_none_values(self):
|
||||
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
|
||||
|
|
|
|||
Loading…
Reference in New Issue