diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..6a1458e18 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,27 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + include: + - python-version: 3.8 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..aec6b8696 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..dd43ca9ee --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,81 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + include: + - python-version: 2.7 + env: + TOXENV: py27 + - python-version: 2.7 + env: + TOXENV: py27-pinned + - python-version: 2.7 + env: + TOXENV: py27-extra-deps + - python-version: pypy3 + env: + TOXENV: pypy + PYPY_VERSION: '2.7-v7.3.5' + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: '3.7-v7.3.5' + - python-version: 3.5 + env: + TOXENV: py35 + - python-version: 3.5 + env: + TOXENV: py35-pinned + - python-version: 3.6 + env: + TOXENV: py36 + - python-version: 3.7 + env: + TOXENV: py37 + - python-version: 3.8 + env: + TOXENV: py38 + - python-version: 3.8 + env: + TOXENV: py38-extra-deps + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + run: | + sudo apt-get update + # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version + sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + if [ "${PYPY_VERSION:4:1}" = "2" ]; then + pip install -U virtualenv + virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" + else + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + fi + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..80a1cd036 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,16 @@ +version: 2 +formats: all +sphinx: + configuration: docs/conf.py + fail_on_warning: true + +build: + image: latest + +python: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image + version: 3.8 # Keep in sync with .github/workflows/checks.yml + install: + - requirements: docs/requirements.txt + - path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4c2498053..000000000 --- a/.travis.yml +++ /dev/null @@ -1,73 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=py27 - python: 2.7 - - env: TOXENV=py27-pinned - python: 2.7 - - env: TOXENV=py27-extra-deps - python: 2.7 - - env: TOXENV=pypy - python: 2.7 - - env: TOXENV=pypy3 - python: 3.5 - - env: TOXENV=py35 - python: 3.5 - - env: TOXENV=py35-pinned - python: 3.5 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=py37 - python: 3.7 - - env: TOXENV=py38 - python: 3.8 - - env: TOXENV=py38-extra-deps - python: 3.8 - - env: TOXENV=docs - python: 3.6 -install: - - | - if [ "$TOXENV" = "pypy" ]; then - export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable" - wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - if [ "$TOXENV" = "pypy3" ]; then - export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" - wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8048e1c86..9e5ba1350 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -828,7 +828,9 @@ REDIRECT_MAX_TIMES Default: ``20`` -The maximum number of redirections that will be followed for a single request. +Defines the maximum times a request can be redirected. After this maximum the +request's response is returned as is. We used Firefox default value for the +same task. MetaRefreshMiddleware --------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 56375664f..4374d8042 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1104,17 +1104,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if you're experiencing problems with insufficient blocking IO. -.. setting:: REDIRECT_MAX_TIMES - -REDIRECT_MAX_TIMES ------------------- - -Default: ``20`` - -Defines the maximum times a request can be redirected. After this maximum the -request's response is returned as is. We used Firefox default value for the -same task. - .. setting:: REDIRECT_PRIORITY_ADJUST REDIRECT_PRIORITY_ADJUST @@ -1396,17 +1385,6 @@ Default: ``True`` A boolean which specifies if the :ref:`telnet console ` will be enabled (provided its extension is also enabled). -.. setting:: TELNETCONSOLE_PORT - -TELNETCONSOLE_PORT ------------------- - -Default: ``[6023, 6073]`` - -The port range to use for the telnet console. If set to ``None`` or ``0``, a -dynamically assigned port is used. For more info see -:ref:`topics-telnetconsole`. - .. setting:: TEMPLATES_DIR TEMPLATES_DIR diff --git a/requirements-py2.txt b/requirements-py2.txt index dde8d1c9c..1f641f7e8 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -3,9 +3,10 @@ PyDispatcher>=2.0.5 w3lib>=1.17.0 protego>=0.1.15 -pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" -queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" -cryptography>=2.0 # Earlier versions would fail to install +pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" +queuelib>=1.4.2,<1.6.0 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" + # Greater versions implement peek support, which breaks tests. +cryptography>=2.0 # Earlier versions would fail to install # Reference versions taken from # https://packages.ubuntu.com/xenial/python/ @@ -14,5 +15,5 @@ cssselect>=0.9.1 lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 -Twisted>=16.0.0 -zope.interface>=4.1.3 +Twisted>=16.0.0,<21.2.0 # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020 +zope.interface>=4.1.3,<5 # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489 diff --git a/requirements-py3.txt b/requirements-py3.txt index 2c98e6f6d..2f27119fe 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,12 +1,13 @@ parsel>=1.5.0 PyDispatcher>=2.0.5 -Twisted>=17.9.0 +Twisted>=17.9.0,<21.2.0 # >=21.2.0 causes https://github.com/scrapy/scrapy/issues/5020 w3lib>=1.17.0 protego>=0.1.15 -pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" -queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" -cryptography>=2.0 # Earlier versions would fail to install +pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" +queuelib>=1.4.2,<1.6.0 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" + # Greater versions implement peek support, which breaks tests. +cryptography>=2.0 # Earlier versions would fail to install # Reference versions taken from # https://packages.ubuntu.com/xenial/python/ @@ -15,4 +16,4 @@ cssselect>=0.9.1 lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 -zope.interface>=4.1.3 +zope.interface>=4.1.3,<5 # >=5 requires https://github.com/scrapy/scrapy/commit/532cd1d93ed58f038346ca6b753462563c85b489 diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3e1148354..b42330e0c 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,6 +5,7 @@ import unittest from itertools import count import platform import six +from sys import version_info from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, @@ -234,10 +235,12 @@ class UtilsPythonTestCase(unittest.TestCase): stripself = not six.PY2 # PyPy3 exposes them as methods self.assertEqual( get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join, stripself), ['list']) self.assertEqual( get_func_args(operator.itemgetter(2), stripself), ['obj']) - + if version_info < (3, 6): + self.assertEqual(get_func_args(" ".join, stripself), ['list']) + else: + self.assertEqual(get_func_args(" ".join, stripself), ['iterable']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])