mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into http2
This commit is contained in:
Adrián Chaves
commit
c8d8b180bf
|
|
@ -0,0 +1,38 @@
|
|||
name: Checks
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
checks:
|
||||
runs-on: ubuntu-18.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: security
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: flake8
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: typing
|
||||
- python-version: 3.7 # Keep in sync with .readthedocs.yml
|
||||
env:
|
||||
TOXENV: docs
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
name: Run test suite
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test-windows:
|
||||
name: "Windows Tests"
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [windows-latest]
|
||||
python-version: [3.7, 3.8]
|
||||
env: [TOXENV: py]
|
||||
include:
|
||||
- os: windows-latest
|
||||
python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run test suite
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox twine wheel codecov
|
||||
tox
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
name: Publish
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-18.04
|
||||
if: startsWith(github.event.ref, 'refs/tags/')
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.8
|
||||
|
||||
- name: Check Tag
|
||||
id: check-release-tag
|
||||
run: |
|
||||
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
|
||||
echo ::set-output name=release_tag::true
|
||||
fi
|
||||
|
||||
- name: Publish to PyPI
|
||||
if: steps.check-release-tag.outputs.release_tag == 'true'
|
||||
run: |
|
||||
pip install --upgrade setuptools wheel twine
|
||||
python setup.py sdist bdist_wheel
|
||||
export TWINE_USERNAME=__token__
|
||||
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
|
||||
twine upload dist/*
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
name: macOS
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: macos-10.15
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.6, 3.7, 3.8]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox -e py
|
||||
|
||||
- name: Upload coverage report
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
name: Ubuntu
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-18.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.6-v7.3.1
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.6.12
|
||||
env:
|
||||
TOXENV: pinned
|
||||
- python-version: 3.6.12
|
||||
env:
|
||||
TOXENV: asyncio-pinned
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3-pinned
|
||||
PYPY_VERSION: 3.6-v7.2.0
|
||||
|
||||
# extras
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: extra-deps
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install system libraries
|
||||
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libxml2-dev libxslt-dev
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
- name: Upload coverage report
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
name: Windows
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
77
.travis.yml
77
.travis.yml
|
|
@ -1,77 +0,0 @@
|
|||
language: python
|
||||
dist: xenial
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^\d\.\d+$/
|
||||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=security
|
||||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pylint
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
- env: TOXENV=typing
|
||||
python: 3.8
|
||||
|
||||
- env: TOXENV=pinned
|
||||
python: 3.6.1
|
||||
- env: TOXENV=asyncio-pinned
|
||||
python: 3.6.1
|
||||
- env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
|
||||
dist: bionic
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
|
||||
dist: bionic
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
- pip install -U tox twine wheel codecov
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- codecov
|
||||
notifications:
|
||||
irc:
|
||||
use_notice: true
|
||||
skip_join: true
|
||||
channels:
|
||||
- irc.freenode.org#scrapy
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
deploy:
|
||||
provider: pypi
|
||||
distributions: "sdist bdist_wheel"
|
||||
user: scrapy
|
||||
password:
|
||||
secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
|
||||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
4
AUTHORS
4
AUTHORS
|
|
@ -1,8 +1,8 @@
|
|||
Scrapy was brought to life by Shane Evans while hacking a scraping framework
|
||||
prototype for Mydeco (mydeco.com). It soon became maintained, extended and
|
||||
improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to
|
||||
bootstrap the project. In mid-2011, Scrapinghub became the new official
|
||||
maintainer.
|
||||
bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new
|
||||
official maintainer.
|
||||
|
||||
Here is the list of the primary authors & contributors:
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
|
|||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported by contacting the project team at opensource@scrapinghub.com. All
|
||||
reported by contacting the project team at opensource@zyte.com. All
|
||||
complaints will be reviewed and investigated and will result in a response that
|
||||
is deemed necessary and appropriate to the circumstances. The project team is
|
||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
||||
|
|
|
|||
23
README.rst
23
README.rst
|
|
@ -10,9 +10,17 @@ Scrapy
|
|||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: Supported Python Versions
|
||||
|
||||
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
|
||||
:target: https://travis-ci.org/scrapy/scrapy
|
||||
:alt: Build Status
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
|
||||
:alt: Ubuntu
|
||||
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
|
||||
:alt: macOS
|
||||
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
|
||||
:alt: Windows
|
||||
|
||||
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
|
||||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
|
|
@ -34,9 +42,16 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
|
|||
crawl websites and extract structured data from their pages. It can be used for
|
||||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other
|
||||
contributors`_.
|
||||
|
||||
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
|
||||
.. _Zyte: https://www.zyte.com/
|
||||
|
||||
Check the Scrapy homepage at https://scrapy.org for more information,
|
||||
including a list of features.
|
||||
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
|
|
@ -81,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct
|
|||
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
|
||||
|
||||
By participating in this project you agree to abide by its terms.
|
||||
Please report unacceptable behavior to opensource@scrapinghub.com.
|
||||
Please report unacceptable behavior to opensource@zyte.com.
|
||||
|
||||
Companies using Scrapy
|
||||
======================
|
||||
|
|
|
|||
|
|
@ -266,7 +266,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
|
|||
.. _setuptools: https://pypi.python.org/pypi/setuptools
|
||||
.. _homebrew: https://brew.sh/
|
||||
.. _zsh: https://www.zsh.org/
|
||||
.. _Scrapinghub: https://scrapinghub.com
|
||||
.. _Anaconda: https://docs.anaconda.com/anaconda/
|
||||
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
|
|
|
|||
|
|
@ -2428,7 +2428,7 @@ Bug fixes
|
|||
- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
|
||||
- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
|
||||
- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
|
||||
``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
|
||||
``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ spiders come in.
|
|||
Popular choices for deploying Scrapy spiders are:
|
||||
|
||||
* :ref:`Scrapyd <deploy-scrapyd>` (open source)
|
||||
* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
|
||||
* :ref:`Zyte Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
|
||||
|
||||
.. _deploy-scrapyd:
|
||||
|
||||
|
|
@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers.
|
|||
|
||||
.. _deploy-scrapy-cloud:
|
||||
|
||||
Deploying to Scrapy Cloud
|
||||
=========================
|
||||
Deploying to Zyte Scrapy Cloud
|
||||
==============================
|
||||
|
||||
`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
|
||||
the company behind Scrapy.
|
||||
`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company
|
||||
behind Scrapy.
|
||||
|
||||
Scrapy Cloud removes the need to setup and monitor servers
|
||||
and provides a nice UI to manage spiders and review scraped items,
|
||||
logs and stats.
|
||||
Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a
|
||||
nice UI to manage spiders and review scraped items, logs and stats.
|
||||
|
||||
To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
|
||||
Please refer to the `Scrapy Cloud documentation`_ for more information.
|
||||
To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line
|
||||
tool.
|
||||
Please refer to the `Zyte Scrapy Cloud documentation`_ for more information.
|
||||
|
||||
Scrapy Cloud is compatible with Scrapyd and one can switch between
|
||||
Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between
|
||||
them as needed - the configuration is read from the ``scrapy.cfg`` file
|
||||
just like ``scrapyd-deploy``.
|
||||
|
||||
.. _Scrapyd: https://github.com/scrapy/scrapyd
|
||||
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
|
||||
.. _Scrapyd: https://github.com/scrapy/scrapyd
|
||||
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
|
||||
.. _shub: https://doc.scrapinghub.com/shub.html
|
||||
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
|
||||
.. _Scrapinghub: https://scrapinghub.com/
|
||||
.. _shub: https://shub.readthedocs.io/en/latest/
|
||||
.. _Zyte: https://zyte.com/
|
||||
.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/
|
||||
.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ instance, which can be accessed and used like this::
|
|||
class MySpider(scrapy.Spider):
|
||||
|
||||
name = 'myspider'
|
||||
start_urls = ['https://scrapinghub.com']
|
||||
start_urls = ['https://scrapy.org']
|
||||
|
||||
def parse(self, response):
|
||||
self.logger.info('Parse function called on %s', response.url)
|
||||
|
|
@ -117,7 +117,7 @@ Python logger you want. For example::
|
|||
class MySpider(scrapy.Spider):
|
||||
|
||||
name = 'myspider'
|
||||
start_urls = ['https://scrapinghub.com']
|
||||
start_urls = ['https://scrapy.org']
|
||||
|
||||
def parse(self, response):
|
||||
logger.info('Parse function called on %s', response.url)
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ project as example.
|
|||
process = CrawlerProcess(get_project_settings())
|
||||
|
||||
# 'followall' is the name of one of the spiders of the project.
|
||||
process.crawl('followall', domain='scrapinghub.com')
|
||||
process.crawl('followall', domain='scrapy.org')
|
||||
process.start() # the script will block here until the crawling is finished
|
||||
|
||||
There's another Scrapy utility that provides more control over the crawling
|
||||
|
|
@ -244,7 +244,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
super proxy that you can attach your own proxies to.
|
||||
* use a highly distributed downloader that circumvents bans internally, so you
|
||||
can just focus on parsing clean pages. One example of such downloaders is
|
||||
`Crawlera`_
|
||||
`Zyte Smart Proxy Manager`_
|
||||
|
||||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
|
@ -254,5 +254,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _Crawlera: https://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
|
|
|
|||
|
|
@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet,
|
|||
you may want to take a look first at this `XPath tutorial`_.
|
||||
|
||||
.. note::
|
||||
Some of the tips are based on `this post from ScrapingHub's blog`_.
|
||||
Some of the tips are based on `this post from Zyte's blog`_.
|
||||
|
||||
.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
|
||||
.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
|
||||
.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/
|
||||
|
||||
|
||||
.. _topics-selectors-relative-xpaths:
|
||||
|
|
|
|||
|
|
@ -280,11 +280,14 @@ class ScrapyAgent:
|
|||
proxyHost = to_unicode(proxyHost)
|
||||
omitConnectTunnel = b'noconnect' in proxyParams
|
||||
if omitConnectTunnel:
|
||||
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
|
||||
"If you use Crawlera, it doesn't require this mode anymore, "
|
||||
"so you should update scrapy-crawlera to 1.3.0+ "
|
||||
"and remove '?noconnect' from the Crawlera URL.",
|
||||
ScrapyDeprecationWarning)
|
||||
warnings.warn(
|
||||
"Using HTTPS proxies in the noconnect mode is deprecated. "
|
||||
"If you use Zyte Smart Proxy Manager (formerly Crawlera), "
|
||||
"it doesn't require this mode anymore, so you should "
|
||||
"update scrapy-crawlera to 1.3.0+ and remove '?noconnect' "
|
||||
"from the Zyte Smart Proxy Manager URL.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
if scheme == b'https' and not omitConnectTunnel:
|
||||
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
|
||||
proxyConf = (proxyHost, proxyPort, proxyAuth)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,6 @@ uvloop; platform_system != "Windows"
|
|||
# optional for shell wrapper tests
|
||||
bpython
|
||||
brotlipy # optional for HTTP compress downloader middleware tests
|
||||
zstandard # optional for HTTP compress downloader middleware tests
|
||||
zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
|
||||
ipython
|
||||
pywin32; sys_platform == "win32"
|
||||
|
|
|
|||
24
tox.ini
24
tox.ini
|
|
@ -13,7 +13,7 @@ deps =
|
|||
-rtests/requirements-py3.txt
|
||||
# mitmproxy does not support PyPy
|
||||
# mitmproxy does not support Windows when running Python < 3.7
|
||||
mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
|
||||
# Extras
|
||||
botocore>=1.4.87
|
||||
|
|
@ -26,9 +26,9 @@ passenv =
|
|||
GCS_TEST_FILE_URI
|
||||
GCS_PROJECT_ID
|
||||
#allow tox virtualenv to upgrade pip/wheel/setuptools
|
||||
download = true
|
||||
download = true
|
||||
commands =
|
||||
py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
|
||||
py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
|
||||
|
||||
[testenv:typing]
|
||||
basepython = python3
|
||||
|
|
@ -89,10 +89,6 @@ deps =
|
|||
botocore==1.4.87
|
||||
google-cloud-storage==1.29.0
|
||||
Pillow==4.0.0
|
||||
install_command =
|
||||
# --use-feature=2020-resolver is required, otherwise the latest verion of
|
||||
# Twisted gets installed.
|
||||
pip install --use-feature=2020-resolver {opts} {packages}
|
||||
setenv =
|
||||
_SCRAPY_PINNED=true
|
||||
|
||||
|
|
@ -101,11 +97,9 @@ deps =
|
|||
{[pinned]deps}
|
||||
lxml==3.5.0
|
||||
PyDispatcher==2.0.5
|
||||
install_command =
|
||||
{[pinned]install_command}
|
||||
setenv =
|
||||
{[pinned]setenv}
|
||||
|
||||
|
||||
[testenv:windows-pinned]
|
||||
basepython = python3
|
||||
deps =
|
||||
|
|
@ -114,8 +108,6 @@ deps =
|
|||
# not need to build lxml from sources in a CI Windows job:
|
||||
lxml==3.8.0
|
||||
PyDispatcher==2.0.5
|
||||
install_command =
|
||||
{[pinned]install_command}
|
||||
setenv =
|
||||
{[pinned]setenv}
|
||||
|
||||
|
|
@ -124,10 +116,6 @@ deps =
|
|||
{[testenv]deps}
|
||||
reppy
|
||||
robotexclusionrulesparser
|
||||
install_command =
|
||||
# Test --use-feature=2020-resolver for the latest version of all
|
||||
# dependencies.
|
||||
pip install --use-feature=2020-resolver {opts} {packages}
|
||||
|
||||
[testenv:asyncio]
|
||||
commands =
|
||||
|
|
@ -135,8 +123,6 @@ commands =
|
|||
|
||||
[testenv:asyncio-pinned]
|
||||
deps = {[testenv:pinned]deps}
|
||||
install_command =
|
||||
{[pinned]install_command}
|
||||
commands = {[testenv:asyncio]commands}
|
||||
setenv =
|
||||
{[pinned]setenv}
|
||||
|
|
@ -152,8 +138,6 @@ deps =
|
|||
{[pinned]deps}
|
||||
lxml==4.0.0
|
||||
PyPyDispatcher==2.1.0
|
||||
install_command =
|
||||
{[pinned]install_command}
|
||||
commands = {[testenv:pypy3]commands}
|
||||
setenv =
|
||||
{[pinned]setenv}
|
||||
|
|
|
|||
Loading…
Reference in New Issue