From 9186e5a686e04703baf78974824917cf7a5121ec Mon Sep 17 00:00:00 2001 From: dswij Date: Tue, 29 Sep 2020 00:27:39 +0700 Subject: [PATCH 01/65] add pip 20.2 test for tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 12e40295c..f73eaef36 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,8 @@ passenv = GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 From 774ebe8796d204b38b332ee71a6ffa678c9b32b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 30 Sep 2020 14:17:30 +0200 Subject: [PATCH 02/65] Mention contributors in the README --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index a8f2ba52b..19faa9a87 100644 --- a/README.rst +++ b/README.rst @@ -34,9 +34,16 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored +by `Scrapinghub Ltd`_. + +.. _many users: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapinghub Ltd: https://www.scrapinghub.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ From f4629fe2cc8469e540b68a48a00fdcbb5ecd976a Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 14:58:14 +0700 Subject: [PATCH 03/65] Update travis-pip and tox deps conflict for pip20.2 --- .travis.yml | 1 + tox.ini | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b883c5b78..de5bbe9ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,7 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + - python -m pip install --upgrade pip #force travis to use newest version of pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index f73eaef36..9a41fc240 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - cryptography==2.0 + #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>= + cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From 66201737a0d245a2568f9b17541d38d6134fa03c Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 15:47:19 +0700 Subject: [PATCH 04/65] fix travis and deps --- .travis.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index de5bbe9ab..d53695e01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + python -m pip install -U pip fi - - python -m pip install --upgrade pip #force travis to use newest version of pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index 9a41fc240..abbc0752e 100644 --- a/tox.ini +++ b/tox.ini @@ -64,14 +64,15 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>= + #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>= cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 - pyOpenSSL==16.2.0 + #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= + pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==17.9.0 From 0ea6ff11360963b7a26f11087cb8f62113070dd5 Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 16:14:36 +0700 Subject: [PATCH 05/65] travis pip version and deps --- .travis.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d53695e01..1ad5c4e9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - python -m pip install -U pip fi + - pip install --upgrade pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index abbc0752e..e923aeb85 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>= - cryptography==2.1.4 + #using cryptography-3.1 to solve dependencies conflict on pip20.2>= + cryptography==3.1 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From f7201b1427749cc364c6f76100f01938d581340c Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 19:59:23 +0700 Subject: [PATCH 06/65] travis and deps --- .travis.yml | 3 ++- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ad5c4e9b..4c4bc0ef7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,8 @@ matrix: - env: TOXENV=asyncio python: 3.8 dist: bionic +before_install: + - pip install -U pip install: - | if [[ ! -z "$PYPY_VERSION" ]]; then @@ -49,7 +51,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - - pip install --upgrade pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e923aeb85..e1e46921e 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-3.1 to solve dependencies conflict on pip20.2>= - cryptography==3.1 + #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>= + cryptography==2.3.1 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From 392b489a65b62244fa6cc1e1406c8e2e2d50d966 Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 20:32:49 +0700 Subject: [PATCH 07/65] travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4c4bc0ef7..da37c85c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ matrix: dist: bionic before_install: - pip install -U pip + - pip --version install: - | if [[ ! -z "$PYPY_VERSION" ]]; then From c83a16898f35c99b0383736e24293edbeaa5d46f Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 20:59:05 +0700 Subject: [PATCH 08/65] try removing cache in travis to install pip --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index da37c85c6..65fa3fa45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,9 +63,6 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" From cc81f9ed06399bd9c4e730a76f54032d9a7e9106 Mon Sep 17 00:00:00 2001 From: dswij Date: Fri, 2 Oct 2020 00:56:59 +0700 Subject: [PATCH 09/65] add download setting for tox --- .travis.yml | 4 +--- tox.ini | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65fa3fa45..5351abf87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,9 +40,6 @@ matrix: - env: TOXENV=asyncio python: 3.8 dist: bionic -before_install: - - pip install -U pip - - pip --version install: - | if [[ ! -z "$PYPY_VERSION" ]]; then @@ -63,6 +60,7 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy + deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/tox.ini b/tox.ini index e1e46921e..f51be432d 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID +download = true #allow tox virtualenv to upgarde pip/wheel/setuptools commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = From 95b2e94496f86d1a225d573adc2af2facce35401 Mon Sep 17 00:00:00 2001 From: dswij Date: Fri, 2 Oct 2020 01:05:45 +0700 Subject: [PATCH 10/65] fix comment error on tox and re-add cache for travis --- .travis.yml | 4 +++- tox.ini | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5351abf87..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,9 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy - +cache: + directories: + - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/tox.ini b/tox.ini index f51be432d..8cc522378 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,8 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID -download = true #allow tox virtualenv to upgarde pip/wheel/setuptools +#allow tox virtualenv to upgrade pip/wheel/setuptools +download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = From 371bb808689bdb6e2bfd1027069e85716c153b51 Mon Sep 17 00:00:00 2001 From: dswij Date: Tue, 6 Oct 2020 19:44:48 +0700 Subject: [PATCH 11/65] Explicitly declare PyDispatcher as dependencies --- .travis.yml | 1 + tox.ini | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b883c5b78..fb4802070 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip fi - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index 8cc522378..10b144619 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ deps = pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted==20.3.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt @@ -115,6 +115,9 @@ deps = {[testenv:pinned]deps} [testenv:pypy3] basepython = pypy3 +deps = + {[testenv]deps} + PyDispatcher==2.0.5 commands = py.test {posargs:--durations=10 docs scrapy tests} @@ -124,7 +127,6 @@ commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 - PyPyDispatcher==2.1.0 [docs] changedir = docs From ce6884d517abb75b884ae23ab59e1405daa83187 Mon Sep 17 00:00:00 2001 From: dswij <44697459+dswij@users.noreply.github.com> Date: Tue, 6 Oct 2020 19:51:42 +0700 Subject: [PATCH 12/65] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 10b144619..4e6ca3ec9 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ deps = service_identity==16.0.0 Twisted==20.3.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==5.1.2 -rtests/requirements-py3.txt # Extras botocore==1.4.87 From 9f02df20c51c80739d2362c388930ad93e3de34a Mon Sep 17 00:00:00 2001 From: dswij Date: Wed, 7 Oct 2020 01:10:01 +0700 Subject: [PATCH 13/65] Remove PyDispatcher from general requirements --- setup.py | 2 +- tox.ini | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 0c2281400..c046684c0 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ install_requires = [ 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', @@ -38,6 +37,7 @@ extras_require = {} if has_environment_marker_platform_impl_support(): extras_require[':platform_python_implementation == "CPython"'] = [ 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by diff --git a/tox.ini b/tox.ini index 4e6ca3ec9..d29cea0fc 100644 --- a/tox.ini +++ b/tox.ini @@ -72,7 +72,6 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - PyDispatcher==2.0.5 #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= pyOpenSSL==18.0.0 queuelib==1.4.2 @@ -89,7 +88,12 @@ deps = [testenv:pinned] deps = {[pinned]deps} - lxml==3.5.0 + PyDispatcher==2.0.5 + +[testenv: pypy-pinned] +deps = + {[pinned]deps} + PyPyDispatcher==2.1.0 [testenv:windows-pinned] basepython = python3 @@ -111,13 +115,16 @@ commands = [testenv:asyncio-pinned] commands = {[testenv:asyncio]commands} -deps = {[testenv:pinned]deps} +deps = + {[testenv:pinned]deps} + lxml==3.5.0 [testenv:pypy3] basepython = pypy3 deps = {[testenv]deps} - PyDispatcher==2.0.5 + PyPyDispatcher>=2.1.0 + lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} From 9461414b14a7444dd52eda517dc5e373f79c2b7e Mon Sep 17 00:00:00 2001 From: dswij Date: Wed, 7 Oct 2020 11:26:53 +0700 Subject: [PATCH 14/65] minor changes to remove unnecessary lines --- .travis.yml | 1 - tox.ini | 19 +++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index fb4802070..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,6 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip fi - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index d29cea0fc..5dc99621e 100644 --- a/tox.ini +++ b/tox.ini @@ -72,12 +72,14 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 + !pypy3: PyDispatcher==2.0.5 #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==20.3.0 w3lib==1.17.0 + #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0 zope.interface==5.1.2 -rtests/requirements-py3.txt # Extras @@ -88,13 +90,8 @@ deps = [testenv:pinned] deps = {[pinned]deps} - PyDispatcher==2.0.5 - -[testenv: pypy-pinned] -deps = - {[pinned]deps} - PyPyDispatcher==2.1.0 - + lxml==3.5.0 + [testenv:windows-pinned] basepython = python3 deps = @@ -114,26 +111,24 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} -deps = - {[testenv:pinned]deps} - lxml==3.5.0 [testenv:pypy3] basepython = pypy3 deps = {[testenv]deps} - PyPyDispatcher>=2.1.0 lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 + PyPyDispatcher==2.1.0 +commands = {[testenv:pypy3]commands} [docs] changedir = docs From 7187247c01d3629000b8b460eea26064043c5595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:23:59 +0100 Subject: [PATCH 15/65] Add PyDispatcher>=2.0.5 back to dependencies for old pip --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index c046684c0..aa75c7a34 100644 --- a/setup.py +++ b/setup.py @@ -33,12 +33,12 @@ install_requires = [ 'itemadapter>=0.1.0', ] extras_require = {} - +cpython_dependencies = [ + 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', +] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = [ - 'lxml>=3.5.0', - 'PyDispatcher>=2.0.5', - ] + extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by # https://foss.heptapod.net/pypy/pypy/-/issues/2498, @@ -49,7 +49,7 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] else: - install_requires.append('lxml>=3.5.0') + install_requires.extend(cpython_dependencies) setup( From 13bcdc9f881727f5a140d4b8671c81c7de3bc572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:28:42 +0100 Subject: [PATCH 16/65] Restore pinned dependencies in tox.ini --- tox.ini | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 5dc99621e..b90ed867f 100644 --- a/tox.ini +++ b/tox.ini @@ -66,21 +66,17 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>= - cryptography==2.3.1 + cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - !pypy3: PyDispatcher==2.0.5 - #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= - pyOpenSSL==18.0.0 + pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==20.3.0 + Twisted==17.9.0 w3lib==1.17.0 - #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0 - zope.interface==5.1.2 + zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras botocore==1.4.87 @@ -91,6 +87,7 @@ deps = deps = {[pinned]deps} lxml==3.5.0 + PyDispatcher==2.0.5 [testenv:windows-pinned] basepython = python3 @@ -99,6 +96,7 @@ deps = # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 + PyDispatcher==2.0.5 [testenv:extra-deps] deps = From 3e5bc7773732aa7e9839934cc45cac8252a36aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:31:33 +0100 Subject: [PATCH 17/65] Remove pinning from the PyPy job --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index b90ed867f..878395e6f 100644 --- a/tox.ini +++ b/tox.ini @@ -114,9 +114,6 @@ commands = {[testenv:asyncio]commands} [testenv:pypy3] basepython = pypy3 -deps = - {[testenv]deps} - lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} From 7327145bf3d45b83b110a46ff79d2e337e36b520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 21:34:15 +0100 Subject: [PATCH 18/65] Remove mitmproxy from pinned environments --- tests/requirements-py3.txt | 2 -- tests/test_proxy_connect.py | 11 +++++------ tox.ini | 10 +++++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..d44632d8b 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,6 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.7' -mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 9eabe6b49..0db148910 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -5,8 +5,6 @@ import re import sys from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit -from unittest import skipIf - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -57,13 +55,14 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf("pypy" in sys.executable, - "mitmproxy does not support PyPy") -@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), - "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): + try: + import mitmproxy + except ImportError: + self.skipTest('mitmproxy is not installed') + self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() diff --git a/tox.ini b/tox.ini index 878395e6f..6149a9c58 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ minversion = 1.7.0 deps = -ctests/constraints.txt -rtests/requirements-py3.txt + # mitmproxy does not support PyPy + # mitmproxy does not support Windows when running Python < 3.7 + mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras boto3>=1.13.0 botocore>=1.4.87 @@ -26,7 +30,7 @@ download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = - pip install --use-feature=2020-resolver {opts} {packages} + pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 @@ -78,6 +82,10 @@ deps = w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt + + # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # above, hence we do not install it in pinned environments at the moment + # Extras botocore==1.4.87 google-cloud-storage==1.29.0 From 8e7b756727cbd8c207acce3083ae29c85a6c6d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:26:55 +0100 Subject: [PATCH 19/65] Solve Flake8-reported issues --- tests/test_proxy_connect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 0db148910..afdfb2578 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,6 +1,5 @@ import json import os -import platform import re import sys from subprocess import Popen, PIPE @@ -59,7 +58,7 @@ class ProxyConnectTestCase(TestCase): def setUp(self): try: - import mitmproxy + import mitmproxy # noqa: F401 except ImportError: self.skipTest('mitmproxy is not installed') From 906626cf0befe332cb61b9cdda7108d61afec776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:50:12 +0100 Subject: [PATCH 20/65] Skip MiddlewareUsingCoro::test_asyncdef on asyncio and old Twisted --- tests/test_downloadermiddleware.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 79f24c8a1..55af4c240 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,11 +1,12 @@ import asyncio -from unittest import mock +from unittest import mock, SkipTest from pytest import mark from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.python.versions import Version from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -211,10 +212,24 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) +@mark.usefixtures('reactor_pytest') class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): + import twisted + if ( + self.reactor_pytest == 'asyncio' + and twisted.version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using AsyncIO and Twisted versions lower than ' + '18.4.0' + ) + + from twisted.python.versions import Version + resp = Response('http://example.com/index.html') class CoroMiddleware: From 6eaf0c5cc99204daed601369c9cea6f7c5e2f878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:54:00 +0100 Subject: [PATCH 21/65] Use Ubuntu Bionic for PyPy tests to try to get a newer OpenSSL version recognized --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b883c5b78..f0eafd137 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ matrix: - env: TOXENV=py python: 3.6 - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 + dist: bionic - env: TOXENV=py python: 3.7 From ea851b910e580e840653274182548314e15b4312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 12:34:29 +0100 Subject: [PATCH 22/65] Clean up Twisted version check --- tests/test_downloadermiddleware.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 55af4c240..6168e0176 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -2,6 +2,7 @@ import asyncio from unittest import mock, SkipTest from pytest import mark +from twisted import version as twisted_version from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -217,10 +218,9 @@ class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): - import twisted if ( self.reactor_pytest == 'asyncio' - and twisted.version < Version('twisted', 18, 4, 0) + and twisted_version < Version('twisted', 18, 4, 0) ): raise SkipTest( 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' @@ -228,8 +228,6 @@ class MiddlewareUsingCoro(ManagerTestCase): '18.4.0' ) - from twisted.python.versions import Version - resp = Response('http://example.com/index.html') class CoroMiddleware: From fea5a118993855808183b32809c2c304db28c364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 12:59:46 +0100 Subject: [PATCH 23/65] Also skip test_asyncdef_asyncio on old Twisted versions --- tests/test_downloadermiddleware.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 6168e0176..b538a0ed3 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -248,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using Twisted versions lower than 18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: From 1941f607ca54694d4822933eba973665baa1b45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:25:56 +0100 Subject: [PATCH 24/65] Skip 2 additional tests with older Twisted versions --- tests/test_utils_signal.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b66588efb..18a8fb49c 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,11 +1,14 @@ import asyncio +from unittest import SkipTest +from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure +from twisted import version as twisted_version from twisted.internet import defer, reactor -from pydispatch import dispatcher +from twisted.python.failure import Failure +from twisted.python.versions import Version +from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue @@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d +@mark.usefixtures('reactor_pytest') class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): @@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" + def test_send_catch_log(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using AsyncIO and Twisted ' + 'versions lower than 18.4.0' + ) + + return super().test_send_catch_log() + @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") + def test_send_catch_log(self): + if (twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using Twisted versions lower ' + 'than 18.4.0' + ) + + return super().test_send_catch_log() + class SendCatchLogTest2(unittest.TestCase): From 4b28da433384f67d0e6c7d4d557f5bcf268ae846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:46:22 +0100 Subject: [PATCH 25/65] Fix syntax error --- tests/test_utils_signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 18a8fb49c..ad7394232 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -104,7 +104,7 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") def test_send_catch_log(self): - if (twisted_version < Version('twisted', 18, 4, 0): + if twisted_version < Version('twisted', 18, 4, 0): raise SkipTest( 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' 'fails due to a timeout when using Twisted versions lower ' From 99cc853d6953d336ca65e0eecc0cb3286306bacf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:53:37 -0300 Subject: [PATCH 26/65] Response.protocol attribute --- scrapy/core/downloader/handlers/http11.py | 1 + scrapy/http/response/__init__.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1b041c8a8..0f30b01f9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -442,6 +442,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], + protocol=getattr(result["txresponse"], "version", None), ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c635fde69..f4ef79c72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, - request=None, certificate=None, ip_address=None): + def __init__( + self, + url, + status=200, + headers=None, + body=b"", + flags=None, + request=None, + certificate=None, + ip_address=None, + protocol=None, + ): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -27,6 +37,7 @@ class Response(object_ref): self.flags = [] if flags is None else list(flags) self.certificate = certificate self.ip_address = ip_address + self.protocol = protocol @property def cb_kwargs(self): @@ -90,7 +101,7 @@ class Response(object_ref): given new values. """ for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address']: + 'request', 'flags', 'certificate', 'ip_address', 'protocol']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 5b6b56240c24d02ef69e6cc591ffb2529bc3f36a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:08:37 -0300 Subject: [PATCH 27/65] Test Response.protocol attribute --- tests/test_downloader_handlers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3e8d7e6b9..eb6d40df7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -489,6 +489,13 @@ class Http11TestCase(HttpTestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + return d + class Https11TestCase(Http11TestCase): scheme = 'https' From 587b4dd71fca12fa5fcc766b891540af77cb27c8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:20:50 -0300 Subject: [PATCH 28/65] Docs for the Response.protocol attribute --- docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f3aaa2c8f..d7d5cd44e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,9 +693,22 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + :param protocol: A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + :type protocol: :class:`tuple` + + .. versionadded:: 2.X.X + The ``protocol`` parameter. + .. versionadded:: 2.1.0 The ``ip_address`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -780,6 +793,8 @@ Response objects .. attribute:: Response.certificate + .. versionadded:: 2.0.0 + A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. @@ -795,6 +810,20 @@ Response objects handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. + .. attribute:: Response.protocol + + .. versionadded:: 2.X.X + + A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`protocol` is always ``None``. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. From 0fb7bcb2cf1606f63f1863bea254a34386c6a0f1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:26:03 -0300 Subject: [PATCH 29/65] Style adjustment --- scrapy/http/response/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f4ef79c72..185a9bb67 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -100,8 +100,9 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address', 'protocol']: + for x in [ + "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", + ]: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 61d089485c7ba66649b936e34833b2013fa12458 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:31:15 -0300 Subject: [PATCH 30/65] Docs: sort versionadded directives --- docs/topics/request-response.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d7d5cd44e..f1be41dde 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -700,14 +700,14 @@ Response objects to represent "HTTP/1.1". :type protocol: :class:`tuple` - .. versionadded:: 2.X.X - The ``protocol`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.0.0 - The ``certificate`` parameter. + .. versionadded:: 2.X.X + The ``protocol`` parameter. .. attribute:: Response.url From 22424125560496c9d131c9b7226aaf0f794e5ad8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 10:50:54 -0300 Subject: [PATCH 31/65] Docs: placeholder for versionadded directive --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f1be41dde..1cb724227 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -706,7 +706,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.X.X + .. versionadded:: VERSION The ``protocol`` parameter. .. attribute:: Response.url @@ -812,7 +812,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: 2.X.X + .. versionadded:: VERSION A tuple containing information about the protocol that was used to download the response. Taken from the ``version`` attribute of the From 5e9a99e6a1e940864d9157252592f04658eaf851 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:15:29 -0300 Subject: [PATCH 32/65] Reponse.protocol as string --- docs/topics/request-response.rst | 20 +++++++------------- scrapy/core/downloader/handlers/http11.py | 7 ++++++- scrapy/core/downloader/webclient.py | 4 ++-- tests/test_downloader_handlers.py | 9 ++++++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1cb724227..98906992d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,12 +693,9 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` - :param protocol: A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". - :type protocol: :class:`tuple` + :param protocol: The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + :type protocol: :class:`str` .. versionadded:: 2.0.0 The ``certificate`` parameter. @@ -814,14 +811,11 @@ Response objects .. versionadded:: VERSION - A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". + The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" - This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. For other handlers, + This attribute is currently only populated by the HTTP download + handlers, i.e. for ``http(s)`` responses. For other handlers, :attr:`protocol` is always ``None``. .. method:: Response.copy() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0f30b01f9..c7553eb87 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -434,6 +434,11 @@ class ScrapyAgent: def _cb_bodydone(self, result, request, url): headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + try: + version = result["txresponse"].version + protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" + except (AttributeError, TypeError): + protocol = None response = respcls( url=url, status=int(result["txresponse"].code), @@ -442,7 +447,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], - protocol=getattr(result["txresponse"], "version", None), + protocol=protocol, ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c13683393..9524cce2b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode from scrapy.responsetypes import responsetypes @@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index eb6d40df7..a8763a7a5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -360,6 +360,13 @@ class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" download_handler_cls = HTTP10DownloadHandler + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + class Https10TestCase(Http10TestCase): scheme = 'https' @@ -493,7 +500,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + d.addCallback(self.assertEqual, "HTTP/1.1") return d From b0368228d7f6391c0df41fca1609c6548613ad6b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:18:03 -0300 Subject: [PATCH 33/65] Add exception to catch --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c7553eb87..a0fd837b1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -437,7 +437,7 @@ class ScrapyAgent: try: version = result["txresponse"].version protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" - except (AttributeError, TypeError): + except (AttributeError, TypeError, IndexError): protocol = None response = respcls( url=url, From 034d61e6cb8391f045eb9b20824964e951549e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 11 Nov 2020 16:46:03 +0100 Subject: [PATCH 34/65] =?UTF-8?q?Restrict=20pip=E2=80=99s=20--use-feature?= =?UTF-8?q?=3D2020-resolver=20to=20the=20extra-deps=20environment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 6149a9c58..f35466d05 100644 --- a/tox.ini +++ b/tox.ini @@ -29,8 +29,6 @@ passenv = download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} -install_command = - pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 @@ -111,6 +109,8 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser +install_command = + pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = From 2405df49f14cbc052d73e58a819a87417b2502e8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 16 Nov 2020 12:50:33 -0300 Subject: [PATCH 35/65] Add tests for Response.protocol=None --- tests/test_downloader_handlers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8763a7a5..f51a6cd3c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -976,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + self.assertIsNone(r.protocol) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -1134,3 +1136,10 @@ class DataURITestCase(unittest.TestCase): request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) + + def test_protocol(self): + def _test(response): + self.assertIsNone(response.protocol) + + request = Request("data:,") + return self.download_request(request, self.spider).addCallback(_test) From 6ef3dc2029fea1f692aa3054ea47d799772151d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 22:28:20 +0100 Subject: [PATCH 36/65] Use the new pip resolver for Tox environments with pinned dependencies --- tox.ini | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tox.ini b/tox.ini index f35466d05..56e736fbf 100644 --- a/tox.ini +++ b/tox.ini @@ -88,12 +88,18 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 +install_command = + # --use-feature=2020-resolver is required, otherwise the latest verion of + # Twisted gets installed. + pip install --use-feature=2020-resolver {opts} {packages} [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} [testenv:windows-pinned] basepython = python3 @@ -103,6 +109,8 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} [testenv:extra-deps] deps = @@ -110,6 +118,8 @@ deps = reppy robotexclusionrulesparser install_command = + # Test --use-feature=2020-resolver for the latest version of all + # dependencies. pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] @@ -118,6 +128,8 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} +install_command = + {[pinned]install_command} commands = {[testenv:asyncio]commands} [testenv:pypy3] @@ -131,6 +143,8 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 +install_command = + {[pinned]install_command} commands = {[testenv:pypy3]commands} [docs] From 075ab156b0ac7bf56828fd40a843f3e9b63a3de3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 11:59:59 -0300 Subject: [PATCH 37/65] Deprecate scrapy.utils.py36 module --- scrapy/utils/py36.py | 17 +++++++++-------- scrapy/utils/python.py | 7 +++++++ scrapy/utils/spider.py | 7 ++----- setup.cfg | 3 --- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index c8c24076e..070b145a4 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,10 +1,11 @@ -""" -Helpers using Python 3.6+ syntax (ignore SyntaxError on import). -""" +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.python import collect_asyncgen # noqa: F401 -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results +warnings.warn( + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, +) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..7bd286523 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -355,3 +355,10 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() + + +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f3a9a67a3..5319604ba 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,17 +4,14 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -try: - from scrapy.utils.py36 import collect_asyncgen -except SyntaxError: - collect_asyncgen = None +from scrapy.utils.python import collect_asyncgen logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + if inspect.isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d diff --git a/setup.cfg b/setup.cfg index 8101443e3..0c9f6b963 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,9 +55,6 @@ ignore_errors = True [mypy-scrapy.utils.response] ignore_errors = True -[mypy-scrapy.utils.spider] -ignore_errors = True - [mypy-scrapy.utils.trackref] ignore_errors = True From 4075e1eadd0a9e83356d18b9ca73090e4a0bc770 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:07:56 -0300 Subject: [PATCH 38/65] Remove deprecated modules (utils.http/markup/multipart) --- pytest.ini | 3 --- scrapy/utils/http.py | 36 ------------------------------------ scrapy/utils/markup.py | 14 -------------- scrapy/utils/multipart.py | 15 --------------- 4 files changed, 68 deletions(-) delete mode 100644 scrapy/utils/http.py delete mode 100644 scrapy/utils/markup.py delete mode 100644 scrapy/utils/multipart.py diff --git a/pytest.ini b/pytest.ini index 1c95f715a..d4deeb57c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -36,8 +36,5 @@ flake8-ignore = scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/utils/http.py F403 - scrapy/utils/markup.py F403 - scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py deleted file mode 100644 index ceb3f0509..000000000 --- a/scrapy/utils/http.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.http instead of this module -""" - -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.decorators import deprecated -from w3lib.http import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.http` is deprecated, " - "Please import from `w3lib.http` instead.", - ScrapyDeprecationWarning, stacklevel=2) - - -@deprecated -def decode_chunked_transfer(chunked_body): - """Parsed body received with chunked transfer encoding, and return the - decoded body. - - For more info see: - https://en.wikipedia.org/wiki/Chunked_transfer_encoding - - """ - body, h, t = '', '', chunked_body - while t: - h, t = t.split('\r\n', 1) - if h == '0': - break - size = int(h, 16) - body += t[:size] - t = t[size + 2:] - return body diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py deleted file mode 100644 index 9728c542a..000000000 --- a/scrapy/utils/markup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.html instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.markup` is deprecated. " - "Please import from `w3lib.html` instead.", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py deleted file mode 100644 index 5dcf791b8..000000000 --- a/scrapy/utils/multipart.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.form instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.multipart` is deprecated. " - "If you're using `encode_multipart` function, please use " - "`urllib3.filepost.encode_multipart_formdata` instead", - ScrapyDeprecationWarning, stacklevel=2) From 51ca4d0138e5c9cf637074f59c839ff9b5839db6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:47:08 -0300 Subject: [PATCH 39/65] Remove deprecated scrapy.utils.gz.is_gzipped function --- scrapy/utils/gz.py | 15 +----------- tests/test_utils_gz.py | 55 ++++++++++++------------------------------ 2 files changed, 16 insertions(+), 54 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 11d433cf5..76156a4b8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ +import struct from gzip import GzipFile from io import BytesIO -import re -import struct from scrapy.utils.decorators import deprecated @@ -42,17 +41,5 @@ def gunzip(data): return b''.join(output_list) -_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search -_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search - - -@deprecated -def is_gzipped(response): - """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'') - cenc = response.headers.get('Content-Encoding', b'').lower() - return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') - - def gzip_magic_number(response): return response.body[:3] == b'\x1f\x8b\x08' diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7148185f4..4943731cb 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -3,10 +3,11 @@ from os.path import join from w3lib.encoding import html_to_unicode -from scrapy.utils.gz import gunzip, is_gzipped -from scrapy.http import Response, Headers +from scrapy.utils.gz import gunzip, gzip_magic_number +from scrapy.http import Response from tests import tests_datadir + SAMPLEDIR = join(tests_datadir, 'compressed') @@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: - text = gunzip(f.read()) - self.assertEqual(len(text), 9950) + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + self.assertFalse(gzip_magic_number(r2)) + self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: @@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase): def test_gunzip_truncated_short(self): with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: - text = gunzip(f.read()) - assert text.endswith(b'') + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) - def test_is_x_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/x-gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_not_quite(self): - hdrs = Headers({"Content-Type": "application/gzippppp"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_case_insensitive(self): - hdrs = Headers({"Content-Type": "Application/X-Gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_wrong(self): - hdrs = Headers({"Content-Type": "application/javascript"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_with_charset(self): - hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: From 462014bc5738a5ed18ec4c15a91384eb17e57096 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:51:59 -0300 Subject: [PATCH 40/65] Scheduler: remove support for deprecated queuelib.PriorityQueue --- scrapy/core/scheduler.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a18c26b17..9ce823dbc 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,14 +1,10 @@ import os import json import logging -import warnings from os.path import join, exists -from queuelib import PriorityQueue - from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir -from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) @@ -56,14 +52,6 @@ class Scheduler: dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - if pqclass is PriorityQueue: - warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" - " is no longer supported because of API changes; " - "please use 'scrapy.pqueues.ScrapyPriorityQueue'", - ScrapyDeprecationWarning) - from scrapy.pqueues import ScrapyPriorityQueue - pqclass = ScrapyPriorityQueue - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('SCHEDULER_DEBUG') From 0a93df9efd20a4bcd34d2ee5e6bdf6d365d70409 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:16:18 -0300 Subject: [PATCH 41/65] Move collect_asyncgen to utils.asyncgen --- scrapy/utils/asyncgen.py | 5 +++++ scrapy/utils/py36.py | 4 ++-- scrapy/utils/python.py | 7 ------- 3 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 scrapy/utils/asyncgen.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py new file mode 100644 index 000000000..7f697af5f --- /dev/null +++ b/scrapy/utils/asyncgen.py @@ -0,0 +1,5 @@ +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index 070b145a4..653e2bbbb 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,11 +1,11 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.python import collect_asyncgen # noqa: F401 +from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401 warnings.warn( - "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.", + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7bd286523..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -355,10 +355,3 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() - - -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results From 18b05af87783d71f9c8f9f1ebe027083037e6f86 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:18:58 -0300 Subject: [PATCH 42/65] Remove tests/test_utils_http.py --- tests/test_utils_http.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 tests/test_utils_http.py diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py deleted file mode 100644 index 363b015a8..000000000 --- a/tests/test_utils_http.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from scrapy.utils.http import decode_chunked_transfer - - -class ChunkedTest(unittest.TestCase): - - def test_decode_chunked_transfer(self): - """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding""" - chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n" - chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n" - chunked_body += "3\r\n" + "con\r\n" - chunked_body += "8\r\n" + "sequence\r\n" - chunked_body += "0\r\n\r\n" - body = decode_chunked_transfer(chunked_body) - self.assertEqual( - body, - "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" - ) From fe8bb6bd905ad2a733ba5e876f4197888605902e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:51:04 -0300 Subject: [PATCH 43/65] Fix import --- scrapy/utils/spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 5319604ba..59fc9202f 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,7 +4,7 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import collect_asyncgen +from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) From 0dc3e6350c230028addd8e73833c93341bab4b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 22:10:45 +0100 Subject: [PATCH 44/65] Add a test to check the Twisted version in pinned environments --- tests/test_dependencies.py | 32 ++++++++++++++++++++++++++++++++ tox.ini | 10 ++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5d0a1d0c9..4e4f190ab 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,8 +1,14 @@ +import os +import re +from configparser import ConfigParser from importlib import import_module + +from twisted import version as twisted_version from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): + def test_required_openssl_version(self): try: module = import_module('OpenSSL') @@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + def test_pinned_twisted_version(self): + """When running tests within a Tox environment with pinned + dependencies, make sure that the version of Twisted is the pinned + version. + + See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 + """ + if not os.environ.get('SCRAPY_PINNED', None): + self.skipTest('Not in a pinned environment') + + tox_config_file_path = os.path.join( + os.path.dirname(__file__), + '..', + 'tox.ini', + ) + config_parser = ConfigParser() + config_parser.read(tox_config_file_path) + pattern = r'Twisted==([\d.]+)' + match = re.search(pattern, config_parser['pinned']['deps']) + pinned_twisted_version_string = match[1] + + self.assertEqual( + twisted_version.short(), + pinned_twisted_version_string + ) + if __name__ == "__main__": unittest.main() diff --git a/tox.ini b/tox.ini index 56e736fbf..ea71a2476 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,8 @@ install_command = # --use-feature=2020-resolver is required, otherwise the latest verion of # Twisted gets installed. pip install --use-feature=2020-resolver {opts} {packages} +setenv = + SCRAPY_PINNED=true [testenv:pinned] deps = @@ -100,6 +102,8 @@ deps = PyDispatcher==2.0.5 install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:windows-pinned] basepython = python3 @@ -111,6 +115,8 @@ deps = PyDispatcher==2.0.5 install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:extra-deps] deps = @@ -131,6 +137,8 @@ deps = {[testenv:pinned]deps} install_command = {[pinned]install_command} commands = {[testenv:asyncio]commands} +setenv = + {[pinned]setenv} [testenv:pypy3] basepython = pypy3 @@ -146,6 +154,8 @@ deps = install_command = {[pinned]install_command} commands = {[testenv:pypy3]commands} +setenv = + {[pinned]setenv} [docs] changedir = docs From f6879c681ebf94b22ee777f3249dfe59a99559cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 23:53:03 +0100 Subject: [PATCH 45/65] =?UTF-8?q?SCRAPY=5FPINNED=20=E2=86=92=20=5FSCRAPY?= =?UTF-8?q?=5FPINNED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_dependencies.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 4e4f190ab..93e7311d2 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -26,7 +26,7 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ - if not os.environ.get('SCRAPY_PINNED', None): + if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') tox_config_file_path = os.path.join( diff --git a/tox.ini b/tox.ini index ea71a2476..66866301c 100644 --- a/tox.ini +++ b/tox.ini @@ -93,7 +93,7 @@ install_command = # Twisted gets installed. pip install --use-feature=2020-resolver {opts} {packages} setenv = - SCRAPY_PINNED=true + _SCRAPY_PINNED=true [testenv:pinned] deps = From 95d39d5cb464ca22516a30f96c0a323613421090 Mon Sep 17 00:00:00 2001 From: etimoz Date: Sun, 29 Nov 2020 13:24:04 +0100 Subject: [PATCH 46/65] removed wrong super argument in overriding serialize_fields code example --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ef50c9f5c..0a0a1765a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -123,7 +123,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': return f'$ {str(value)}' - return super(Product, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) .. _topics-exporters-reference: From 7fec9f991f0bd415900df2bf288c6cda909ecd77 Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Mon, 30 Nov 2020 21:47:28 +0100 Subject: [PATCH 47/65] [Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals (#4911) [Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c2281400..35736b75f 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', From a80bafe5cdcb4b5e2542524fe641570f9107a121 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 30 Nov 2020 19:03:13 -0300 Subject: [PATCH 48/65] Remove deprecated SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE --- scrapy/utils/project.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fd13d85e3..35e59a258 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -68,18 +68,10 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - if pickled_settings: - warnings.warn("Use of environment variable " - "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' " - "is deprecated.", ScrapyDeprecationWarning) - settings.setdict(pickle.loads(pickled_settings), priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', - 'PICKLED_SETTINGS_TO_OVERRIDE', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', From 6091f3cc03835b24def08bf358d7b19207be685d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Dec 2020 10:26:21 -0300 Subject: [PATCH 49/65] Remove unused pickle import --- scrapy/utils/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 35e59a258..c66af497e 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,4 @@ import os -import pickle import warnings from importlib import import_module From db10aaf9eb858c8deb35e7ef96538549a8e36be0 Mon Sep 17 00:00:00 2001 From: gunadhya <6939749+gunadhya@users.noreply.github.com> Date: Thu, 3 Dec 2020 15:26:36 +0530 Subject: [PATCH 50/65] Update links in installation guide (#4899) --- docs/conf.py | 1 + docs/intro/install.rst | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 27d2b5dff..543507a46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 3bfd3bc3b..73d7ede42 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -69,10 +69,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -265,7 +264,6 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ .. _Scrapinghub: https://scrapinghub.com From 0dff5781bc9089884e27b0a59c1e92fc111b2817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 11:13:14 +0100 Subject: [PATCH 51/65] Blind attempt to fix the build of the cryptography-provided OpenSSL --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 6c39299bb..d3ff930c3 100644 --- a/tox.ini +++ b/tox.ini @@ -146,6 +146,8 @@ commands = [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} +before_install: + - sudo apt-get -y remove libssl-dev deps = {[pinned]deps} lxml==4.0.0 From 798a818cafb21eeb62790d905d43cec678d30693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 13:35:40 +0100 Subject: [PATCH 52/65] Move apt-get command from Tox to Travis CI --- .travis.yml | 1 + tox.ini | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0eafd137..055304abe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ matrix: python: 3.8 dist: bionic install: + - apt-get -y remove libssl-dev - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" diff --git a/tox.ini b/tox.ini index d3ff930c3..6c39299bb 100644 --- a/tox.ini +++ b/tox.ini @@ -146,8 +146,6 @@ commands = [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -before_install: - - sudo apt-get -y remove libssl-dev deps = {[pinned]deps} lxml==4.0.0 From 1c1255a75d315cd535dbcc5689ddda9c80c79684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 14:41:02 +0100 Subject: [PATCH 53/65] Use sudo for apt-get --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 055304abe..1a296c5e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ matrix: python: 3.8 dist: bionic install: - - apt-get -y remove libssl-dev + - sudo apt-get -y remove libssl-dev - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" From 45eb099ed1937a530208eb91e28e013d56b542ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 15:37:02 +0100 Subject: [PATCH 54/65] =?UTF-8?q?Maybe=20it=E2=80=99s=20about=20having=20a?= =?UTF-8?q?=20newer=20libssl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1a296c5e6..880c8772b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ matrix: - env: TOXENV=asyncio-pinned python: 3.6.1 - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 + dist: bionic - env: TOXENV=py python: 3.6 @@ -42,7 +43,6 @@ matrix: python: 3.8 dist: bionic install: - - sudo apt-get -y remove libssl-dev - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" From b83a1a6fbfe12c205c7c57830b1d4a6fc1097be0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 16 Dec 2020 18:02:47 -0300 Subject: [PATCH 55/65] Disable test under pypy --- tests/test_webclient.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a60181a3a..f935a8689 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,7 +4,10 @@ Tests borrowed from the twisted.web.client tests. """ import os import shutil +import sys +from pkg_resources import parse_version +import cryptography import OpenSSL.SSL from twisted.trial import unittest from twisted.web import server, static, util, resource @@ -414,6 +417,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): + if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): + self.skipTest("This does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From 6dccf82eaa097f9e40e922d04a8b5ec9a0a9dbd7 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 22 Dec 2020 07:49:13 +1100 Subject: [PATCH 56/65] docs: fix simple typo, wihout -> without There is a small typo in scrapy/http/request/form.py. Should read `without` rather than `wihout`. --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2815303a2..7f267c800 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -160,7 +160,7 @@ def _select_value(ele, n, v): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected - # And for select tags wihout options + # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) elif v is not None and multiple: From 44a7ab5bf06be3bfdf4b4a304b24f544d2f833cd Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Wed, 30 Dec 2020 15:22:27 +0100 Subject: [PATCH 57/65] Fix warning: Expected type 'bool', got 'int' instead (#4940) * Fix warning: Expected type 'bool', got 'int' instead * Update defer.py --- scrapy/pipelines/media.py | 2 +- scrapy/utils/defer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0a12f3e2c..0c2ee6856 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -86,7 +86,7 @@ class MediaPipeline: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=1) + dfd = DeferredList(dlist, consumeErrors=True) return dfd.addCallback(self.item_completed, item, info) def _process_request(self, request, info, item): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21ba02a0b..6db9cc117 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -105,7 +105,7 @@ def process_parallel(callbacks, input, *a, **kw): callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1) + d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d From 80db569aeae3e05480ed228c95332c17f67e81e4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 1 Jan 2021 19:13:39 -0300 Subject: [PATCH 58/65] Migrate CI to GitHub actions (#4924) --- .github/workflows/checks.yml | 38 ++++++++++++++ .github/workflows/main.yml | 31 ------------ .github/workflows/publish.yml | 31 ++++++++++++ .github/workflows/tests-macos.yml | 25 ++++++++++ .github/workflows/tests-ubuntu.yml | 69 ++++++++++++++++++++++++++ .github/workflows/tests-windows.yml | 32 ++++++++++++ .travis.yml | 77 ----------------------------- README.rst | 14 ++++-- tests/requirements-py3.txt | 2 +- tox.ini | 24 ++------- 10 files changed, 211 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/tests-macos.yml create mode 100644 .github/workflows/tests-ubuntu.yml create mode 100644 .github/workflows/tests-windows.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..2748bf5fe --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,38 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.8 + env: + TOXENV: security + - python-version: 3.8 + env: + TOXENV: flake8 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.8 + env: + TOXENV: typing + - python-version: 3.7 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 28771216c..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Run test suite -on: [push, pull_request] - -jobs: - test-windows: - name: "Windows Tests" - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest] - python-version: [3.7, 3.8] - env: [TOXENV: py] - include: - - os: windows-latest - python-version: 3.6 - env: - TOXENV: windows-pinned - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Run test suite - env: ${{ matrix.env }} - run: | - pip install -U tox twine wheel codecov - tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..aec6b8696 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 000000000..51d27c405 --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,25 @@ +name: macOS +on: [push, pull_request] + +jobs: + tests: + runs-on: macos-10.15 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 000000000..89c0334e2 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,69 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 + + # pinned deps + - python-version: 3.6.12 + env: + TOXENV: pinned + - python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 + + # extras + - python-version: 3.8 + env: + TOXENV: extra-deps + - python-version: 3.8 + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 000000000..ed2e4075d --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,32 @@ +name: Windows +on: [push, pull_request] + +jobs: + tests: + runs-on: windows-latest + strategy: + matrix: + include: + - python-version: 3.6 + env: + TOXENV: windows-pinned + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 880c8772b..000000000 --- a/.travis.yml +++ /dev/null @@ -1,77 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pylint - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml - - env: TOXENV=typing - python: 3.8 - - - env: TOXENV=pinned - python: 3.6.1 - - env: TOXENV=asyncio-pinned - python: 3.6.1 - - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - dist: bionic - - - env: TOXENV=py - python: 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - dist: bionic - - - env: TOXENV=py - python: 3.7 - - - env: TOXENV=py PYPI_RELEASE_JOB=true - python: 3.8 - dist: bionic - - env: TOXENV=extra-deps - python: 3.8 - dist: bionic - - env: TOXENV=asyncio - python: 3.8 - dist: bionic -install: - - | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/README.rst b/README.rst index a8f2ba52b..9418d270f 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu + +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 68b856a88..a86c4ae4f 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -14,6 +14,6 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython brotlipy # optional for HTTP compress downloader middleware tests -zstandard # optional for HTTP compress downloader middleware tests +zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 6c39299bb..e70aef2d2 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 @@ -25,9 +25,9 @@ passenv = GCS_TEST_FILE_URI GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools -download = true +download = true commands = - py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} + py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} [testenv:typing] basepython = python3 @@ -87,10 +87,6 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 -install_command = - # --use-feature=2020-resolver is required, otherwise the latest verion of - # Twisted gets installed. - pip install --use-feature=2020-resolver {opts} {packages} setenv = _SCRAPY_PINNED=true @@ -99,11 +95,9 @@ deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} - + [testenv:windows-pinned] basepython = python3 deps = @@ -112,8 +106,6 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} @@ -122,10 +114,6 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser -install_command = - # Test --use-feature=2020-resolver for the latest version of all - # dependencies. - pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = @@ -133,8 +121,6 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -install_command = - {[pinned]install_command} commands = {[testenv:asyncio]commands} setenv = {[pinned]setenv} @@ -150,8 +136,6 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 -install_command = - {[pinned]install_command} commands = {[testenv:pypy3]commands} setenv = {[pinned]setenv} From 0a1e2fefab04984a0f3c2b470346fd9f2ffde65b Mon Sep 17 00:00:00 2001 From: M Ikram Ullah Khan <44160462+IkramKhanNiazi@users.noreply.github.com> Date: Mon, 4 Jan 2021 18:30:23 +0500 Subject: [PATCH 59/65] Docs: fix typo in news.rst (#4942) --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 0391506c4..d9fe897ad 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2428,7 +2428,7 @@ Bug fixes - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, - ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). Documentation ~~~~~~~~~~~~~ From 6e7ae789f9d0b62ef5e07d16fab062ef7bdfa660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Jan 2021 11:59:38 +0100 Subject: [PATCH 60/65] Reuse the text from https://scrapy.org/ --- README.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 19faa9a87..551a06f6b 100644 --- a/README.rst +++ b/README.rst @@ -34,11 +34,10 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored -by `Scrapinghub Ltd`_. +Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_. -.. _many users: https://github.com/scrapy/scrapy/graphs/contributors -.. _Scrapinghub Ltd: https://www.scrapinghub.com/ +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapinghub: https://www.scrapinghub.com/ Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. From f30f53b3cc958770406628411db3d93c925db59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Feb 2021 15:03:20 +0100 Subject: [PATCH 61/65] =?UTF-8?q?Scrapinghub=20=E2=86=92=20Zyte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AUTHORS | 4 +-- CODE_OF_CONDUCT.md | 2 +- README.rst | 7 ++--- docs/intro/install.rst | 1 - docs/topics/deploy.rst | 32 +++++++++++------------ docs/topics/logging.rst | 4 +-- docs/topics/practices.rst | 6 ++--- docs/topics/selectors.rst | 4 +-- scrapy/core/downloader/handlers/http11.py | 13 +++++---- 9 files changed, 38 insertions(+), 35 deletions(-) diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..652460383 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/README.rst b/README.rst index bbe346522..5750e2c0f 100644 --- a/README.rst +++ b/README.rst @@ -42,10 +42,11 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors -.. _Scrapinghub: https://www.scrapinghub.com/ +.. _Zyte: https://www.zyte.com/ Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. @@ -95,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 73d7ede42..bf919ce25 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -266,7 +266,6 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.python.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html .. _conda-forge: https://conda-forge.org/ diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 55065a1a3..c3445d40e 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -101,7 +101,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -117,7 +117,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..502fd5fcd 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -63,7 +63,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('followall', domain='scrapinghub.com') + process.crawl('followall', domain='scrapy.org') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling @@ -244,7 +244,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is - `Crawlera`_ + `Zyte Smart Proxy Manager`_ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -254,5 +254,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b576fde91..c7ec2e0cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet, you may want to take a look first at this `XPath tutorial`_. .. note:: - Some of the tips are based on `this post from ScrapingHub's blog`_. + Some of the tips are based on `this post from Zyte's blog`_. .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ +.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ .. _topics-selectors-relative-xpaths: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a0fd837b1..513df2de9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -303,11 +303,14 @@ class ScrapyAgent: proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams if omitConnectTunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + warnings.warn( + "Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Zyte Smart Proxy Manager (formerly Crawlera), " + "it doesn't require this mode anymore, so you should " + "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "from the Zyte Smart Proxy Manager URL.", + ScrapyDeprecationWarning, + ) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) From 1e9b52c3e0e4eb51aaad37e796b3ee810fd919a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 Feb 2021 22:02:03 +0500 Subject: [PATCH 62/65] Refactor SpiderMiddlewareManager.scrape_response. --- scrapy/core/spidermw.py | 148 +++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 71 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 763e0cdf6..289292da7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -41,86 +41,92 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def scrape_response(self, scrape_func, response, request, spider): - - def process_spider_input(response): - for method in self.methods['process_spider_input']: - try: - result = method(response=response, spider=spider) - if result is not None: - msg = (f"Middleware {_fname(method)} must return None " - f"or raise an exception, got {type(result)}") - raise _InvalidOutput(msg) - except _InvalidOutput: - raise - except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) - - def _evaluate_iterable(iterable, exception_processor_index, recover_to): + def _process_spider_input(self, scrape_func, response, request, spider): + for method in self.methods['process_spider_input']: try: - for r in iterable: - yield r + result = method(response=response, spider=spider) + if result is not None: + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) + except _InvalidOutput: + raise + except Exception: + return scrape_func(Failure(), request, spider) + return scrape_func(response, request, spider) + + def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + + def _process_spider_exception(self, response, spider, _failure, start_index=0): + exception = _failure.value + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): + return _failure + method_list = islice(self.methods['process_spider_exception'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + result = method(response=response, exception=exception, spider=spider) + if _isiterable(result): + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned + return self._process_spider_output(response, spider, result, method_index + 1) + elif result is None: + continue + else: + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) + return _failure + + def _process_spider_output(self, response, spider, result, start_index=0): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + method_list = islice(self.methods['process_spider_output'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + try: + # might fail directly if the output value is not a generator + result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), exception_processor_index) + exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1) if isinstance(exception_result, Failure): raise - recover_to.extend(exception_result) + return exception_result + if _isiterable(result): + result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) + else: + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) - def process_spider_exception(_failure, start_index=0): - exception = _failure.value - # don't handle _InvalidOutput exception - if isinstance(exception, _InvalidOutput): - return _failure - method_list = islice(self.methods['process_spider_exception'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - result = method(response=response, exception=exception, spider=spider) - if _isiterable(result): - # stop exception handling by handing control over to the - # process_spider_output chain if an iterable has been returned - return process_spider_output(result, method_index + 1) - elif result is None: - continue - else: - msg = (f"Middleware {_fname(method)} must return None " - f"or an iterable, got {type(result)}") - raise _InvalidOutput(msg) - return _failure + return MutableChain(result, recovered) - def process_spider_output(result, start_index=0): - # items in this iterable do not need to go through the process_spider_output - # chain, they went through it already from the process_spider_exception method - recovered = MutableChain() - - method_list = islice(self.methods['process_spider_output'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - try: - # might fail directly if the output value is not a generator - result = method(response=response, result=result, spider=spider) - except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index + 1) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = _evaluate_iterable(result, method_index + 1, recovered) - else: - msg = (f"Middleware {_fname(method)} must return an " - f"iterable, got {type(result)}") - raise _InvalidOutput(msg) - - return MutableChain(result, recovered) + def _process_callback_output(self, response, spider, result): + recovered = MutableChain() + result = self._evaluate_iterable(response, spider, result, 0, recovered) + return MutableChain(self._process_spider_output(response, spider, result), recovered) + def scrape_response(self, scrape_func, response, request, spider): def process_callback_output(result): - recovered = MutableChain() - result = _evaluate_iterable(result, 0, recovered) - return MutableChain(process_spider_output(result), recovered) + return self._process_callback_output(response, spider, result) - dfd = mustbe_deferred(process_spider_input, response) + def process_spider_exception(_failure): + return self._process_spider_exception(response, spider, _failure) + + dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd From 54fd371481ef0ff80cc23bfab7e7f53126a31bc3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 11 Feb 2021 14:24:11 +0500 Subject: [PATCH 63/65] Skip uvloop 0.15.0+ on py36. --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index a86c4ae4f..21a554624 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -9,7 +9,8 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop; platform_system != "Windows" +uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6' +uvloop; platform_system != "Windows" and python_version > '3.6' # optional for shell wrapper tests bpython From abbbfbbb38a87b9f7259fe47493d5810b2439963 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Feb 2021 22:41:16 +0500 Subject: [PATCH 64/65] Add tests for deferred_f_from_coro_f. --- tests/test_utils_defer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e60242a3b..7a5f458c7 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,8 +1,10 @@ +from pytest import mark from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure from scrapy.utils.defer import ( + deferred_f_from_coro_f, iter_errback, mustbe_deferred, process_chain, @@ -117,3 +119,18 @@ class IterErrbackTest(unittest.TestCase): self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError) + + +class AsyncDefTestsuiteTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f(self): + pass + + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_generator(self): + yield + + @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_xfail(self): + raise Exception("This is expected to be raised") From 49af7c4c8b71f8643fe13c20261f97323117f980 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 18 Feb 2021 17:10:30 +0500 Subject: [PATCH 65/65] Drop pytest-twisted, use Scrapy code to install the reactor. --- conftest.py | 15 +++++++++++++++ pytest.ini | 1 - tests/requirements-py3.txt | 4 +--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index 68b855c08..e4dd80de0 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,8 @@ from pathlib import Path import pytest +from scrapy.utils.reactor import install_reactor + from tests.keys import generate_keys @@ -40,6 +42,14 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope='class') def reactor_pytest(request): if not request.cls: @@ -55,5 +65,10 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +def pytest_configure(config): + if config.getoption("--reactor") == "asyncio": + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/pytest.ini b/pytest.ini index d4deeb57c..0aae09ff5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -18,7 +18,6 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils -twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed flake8-max-line-length = 119 diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 21a554624..bd72c8c46 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,10 +2,8 @@ attrs dataclasses; python_version == '3.6' pyftpdlib -# https://github.com/pytest-dev/pytest-twisted/issues/93 -pytest != 5.4, != 5.4.1 +pytest pytest-cov -pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures