mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into more-lenient-proxying
This commit is contained in:
Adrián Chaves
commit
46dd152b3e
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.6.2
|
||||
current_version = 2.7.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
19
.flake8
19
.flake8
|
|
@ -4,16 +4,19 @@ max-line-length = 119
|
|||
ignore = W503
|
||||
|
||||
exclude =
|
||||
docs/conf.py
|
||||
|
||||
per-file-ignores =
|
||||
# Exclude files that are meant to provide top-level imports
|
||||
# E402: Module level import not at top of file
|
||||
# F401: Module imported but unused
|
||||
scrapy/__init__.py E402
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/linkextractors/__init__.py E402 F401
|
||||
scrapy/selector/__init__.py F401
|
||||
scrapy/spiders/__init__.py E402 F401
|
||||
scrapy/__init__.py:E402
|
||||
scrapy/core/downloader/handlers/http.py:F401
|
||||
scrapy/http/__init__.py:F401
|
||||
scrapy/linkextractors/__init__.py:E402,F401
|
||||
scrapy/selector/__init__.py:F401
|
||||
scrapy/spiders/__init__.py:E402,F401
|
||||
|
||||
# Issues pending a review:
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
scrapy/utils/url.py:F403,F405
|
||||
tests/test_loader.py:E741
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ on: [push, pull_request]
|
|||
|
||||
jobs:
|
||||
checks:
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -25,12 +25,15 @@ jobs:
|
|||
- python-version: "3.10" # Keep in sync with .readthedocs.yml
|
||||
env:
|
||||
TOXENV: docs
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: twinecheck
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@ on: [push]
|
|||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.event.ref, 'refs/tags/')
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ jobs:
|
|||
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ on: [push, pull_request]
|
|||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -20,10 +20,15 @@ jobs:
|
|||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
- python-version: pypy3
|
||||
- python-version: "3.11.0-rc.2"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.11.0-rc.2"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
- python-version: pypy3.9
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.9-v7.3.9
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.7.13
|
||||
|
|
@ -32,10 +37,9 @@ jobs:
|
|||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: asyncio-pinned
|
||||
- python-version: pypy3
|
||||
- python-version: pypy3.7
|
||||
env:
|
||||
TOXENV: pypy3-pinned
|
||||
PYPY_VERSION: 3.7-v7.3.5
|
||||
|
||||
# extras
|
||||
# extra-deps includes reppy, which does not support Python 3.9
|
||||
|
|
@ -45,30 +49,22 @@ jobs:
|
|||
TOXENV: extra-deps
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install system libraries
|
||||
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
|
||||
if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
|
||||
sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
|
||||
sudo apt-get install libxml2-dev libxslt-dev
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
|
|
|
|||
|
|
@ -25,10 +25,10 @@ jobs:
|
|||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,3 +23,6 @@ test-output.*
|
|||
|
||||
# Windows
|
||||
Thumbs.db
|
||||
|
||||
# OSX miscellaneous
|
||||
.DS_Store
|
||||
4
INSTALL
4
INSTALL
|
|
@ -1,4 +0,0 @@
|
|||
For information about installing Scrapy see:
|
||||
|
||||
* docs/intro/install.rst (local file)
|
||||
* https://docs.scrapy.org/en/latest/intro/install.html (online version)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
For information about installing Scrapy see:
|
||||
|
||||
* [Local docs](docs/intro/install.rst)
|
||||
* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)
|
||||
|
|
@ -64,7 +64,9 @@ Requirements
|
|||
Install
|
||||
=======
|
||||
|
||||
The quick way::
|
||||
The quick way:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
pip install scrapy
|
||||
|
||||
|
|
@ -95,8 +97,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
|
|||
Code of Conduct
|
||||
---------------
|
||||
|
||||
Please note that this project is released with a Contributor Code of Conduct
|
||||
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
|
||||
Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.
|
||||
|
||||
By participating in this project you agree to abide by its terms.
|
||||
Please report unacceptable behavior to opensource@zyte.com.
|
||||
|
|
|
|||
10
conftest.py
10
conftest.py
|
|
@ -42,16 +42,6 @@ def chdir(tmpdir):
|
|||
tmpdir.chdir()
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(session, config, items):
|
||||
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
|
||||
try:
|
||||
from pytest_flake8 import Flake8Item
|
||||
if config.getoption('--flake8'):
|
||||
items[:] = [item for item in items if isinstance(item, Flake8Item)]
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--reactor",
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class SettingsListDirective(Directive):
|
|||
|
||||
|
||||
def is_setting_index(node):
|
||||
if node.tagname == 'index':
|
||||
if node.tagname == 'index' and node['entries']:
|
||||
# index entries for setting directives look like:
|
||||
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
|
||||
entry_type, info, refid = node['entries'][0][:3]
|
||||
|
|
@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
|
|||
|
||||
def setup(app):
|
||||
app.add_crossref_type(
|
||||
directivename = "setting",
|
||||
rolename = "setting",
|
||||
indextemplate = "pair: %s; setting",
|
||||
directivename="setting",
|
||||
rolename="setting",
|
||||
indextemplate="pair: %s; setting",
|
||||
)
|
||||
app.add_crossref_type(
|
||||
directivename = "signal",
|
||||
rolename = "signal",
|
||||
indextemplate = "pair: %s; signal",
|
||||
directivename="signal",
|
||||
rolename="signal",
|
||||
indextemplate="pair: %s; signal",
|
||||
)
|
||||
app.add_crossref_type(
|
||||
directivename = "command",
|
||||
rolename = "command",
|
||||
indextemplate = "pair: %s; command",
|
||||
directivename="command",
|
||||
rolename="command",
|
||||
indextemplate="pair: %s; command",
|
||||
)
|
||||
app.add_crossref_type(
|
||||
directivename = "reqmeta",
|
||||
rolename = "reqmeta",
|
||||
indextemplate = "pair: %s; reqmeta",
|
||||
directivename="reqmeta",
|
||||
rolename="reqmeta",
|
||||
indextemplate="pair: %s; reqmeta",
|
||||
)
|
||||
app.add_role('source', source_role)
|
||||
app.add_role('commit', commit_role)
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ Tests
|
|||
=====
|
||||
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:core/development/policy/test-standard>`. Running tests requires
|
||||
<twisted:development/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
|
||||
.. _running-tests:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ Supported Python versions
|
|||
=========================
|
||||
|
||||
Scrapy requires Python 3.7+, either the CPython implementation (default) or
|
||||
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
|
||||
the PyPy implementation (see :ref:`python:implementations`).
|
||||
|
||||
.. _intro-install-scrapy:
|
||||
|
||||
|
|
@ -187,7 +187,7 @@ solutions:
|
|||
* Install `homebrew`_ following the instructions in https://brew.sh/
|
||||
|
||||
* Update your ``PATH`` variable to state that homebrew packages should be
|
||||
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
|
||||
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
|
||||
if you're using `zsh`_ as default shell)::
|
||||
|
||||
echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
|
||||
|
|
@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy::
|
|||
PyPy
|
||||
----
|
||||
|
||||
We recommend using the latest PyPy version. The version tested is 5.9.0.
|
||||
We recommend using the latest PyPy version.
|
||||
For PyPy3, only Linux installation was tested.
|
||||
|
||||
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ like this:
|
|||
Let's open up scrapy shell and play a bit to find out how to extract the data
|
||||
we want::
|
||||
|
||||
$ scrapy shell 'https://quotes.toscrape.com'
|
||||
scrapy shell 'https://quotes.toscrape.com'
|
||||
|
||||
We get a list of selectors for the quote HTML elements with:
|
||||
|
||||
|
|
|
|||
219
docs/news.rst
219
docs/news.rst
|
|
@ -3,6 +3,221 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.7.0:
|
||||
|
||||
Scrapy 2.7.0 (2022-10-17)
|
||||
-----------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
- Added Python 3.11 support, dropped Python 3.6 support
|
||||
- Improved support for :ref:`asynchronous callbacks <topics-coroutines>`
|
||||
- :ref:`Asyncio support <using-asyncio>` is enabled by default on new
|
||||
projects
|
||||
- Output names of item fields can now be arbitrary strings
|
||||
- Centralized :ref:`request fingerprinting <request-fingerprints>`
|
||||
configuration is now possible
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Python 3.7 or greater is now required; support for Python 3.6 has been dropped.
|
||||
Support for the upcoming Python 3.11 has been added.
|
||||
|
||||
The minimum required version of some dependencies has changed as well:
|
||||
|
||||
- lxml_: 3.5.0 → 4.3.0
|
||||
|
||||
- Pillow_ (:ref:`images pipeline <images-pipeline>`): 4.0.0 → 7.1.0
|
||||
|
||||
- zope.interface_: 5.0.0 → 5.1.0
|
||||
|
||||
(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`,
|
||||
:issue:`5670`, :issue:`5678`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- :meth:`ImagesPipeline.thumb_path
|
||||
<scrapy.pipelines.images.ImagesPipeline.thumb_path>` must now accept an
|
||||
``item`` parameter (:issue:`5504`, :issue:`5508`).
|
||||
|
||||
- The ``scrapy.downloadermiddlewares.decompression`` module is now
|
||||
deprecated (:issue:`5546`, :issue:`5547`).
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- The
|
||||
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
|
||||
method of :ref:`spider middlewares <topics-spider-middleware>` can now be
|
||||
defined as an :term:`asynchronous generator` (:issue:`4978`).
|
||||
|
||||
- The output of :class:`~scrapy.Request` callbacks defined as
|
||||
:ref:`coroutines <topics-coroutines>` is now processed asynchronously
|
||||
(:issue:`4978`).
|
||||
|
||||
- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous
|
||||
callbacks <topics-coroutines>` (:issue:`5657`).
|
||||
|
||||
- New projects created with the :command:`startproject` command have
|
||||
:ref:`asyncio support <using-asyncio>` enabled by default (:issue:`5590`,
|
||||
:issue:`5679`).
|
||||
|
||||
- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a
|
||||
dictionary to customize the output name of item fields, lifting the
|
||||
restriction that required output names to be valid Python identifiers, e.g.
|
||||
preventing them to have whitespace (:issue:`1008`, :issue:`3266`,
|
||||
:issue:`3696`).
|
||||
|
||||
- You can now customize :ref:`request fingerprinting <request-fingerprints>`
|
||||
through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of
|
||||
having to change it on every Scrapy component that relies on request
|
||||
fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`,
|
||||
:issue:`4524`).
|
||||
|
||||
- ``jsonl`` is now supported and encouraged as a file extension for `JSON
|
||||
Lines`_ files (:issue:`4848`).
|
||||
|
||||
.. _JSON Lines: https://jsonlines.org/
|
||||
|
||||
- :meth:`ImagesPipeline.thumb_path
|
||||
<scrapy.pipelines.images.ImagesPipeline.thumb_path>` now receives the
|
||||
source :ref:`item <topics-items>` (:issue:`5504`, :issue:`5508`).
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- When using Google Cloud Storage with a :ref:`media pipeline
|
||||
<topics-media-pipeline>`, :setting:`FILES_EXPIRES` now also works when
|
||||
:setting:`FILES_STORE` does not point at the root of your Google Cloud
|
||||
Storage bucket (:issue:`5317`, :issue:`5318`).
|
||||
|
||||
- The :command:`parse` command now supports :ref:`asynchronous callbacks
|
||||
<topics-coroutines>` (:issue:`5424`, :issue:`5577`).
|
||||
|
||||
- When using the :command:`parse` command with a URL for which there is no
|
||||
available spider, an exception is no longer raised (:issue:`3264`,
|
||||
:issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`).
|
||||
|
||||
- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte
|
||||
order mark`_ when determining the text encoding of the response body,
|
||||
following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`).
|
||||
|
||||
.. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
|
||||
.. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
|
||||
|
||||
- MIME sniffing takes the response body into account in FTP and HTTP/1.0
|
||||
requests, as well as in cached requests (:issue:`4873`).
|
||||
|
||||
- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag
|
||||
is missing (:issue:`4873`).
|
||||
|
||||
- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value
|
||||
that does not match the asyncio event loop actually installed
|
||||
(:issue:`5529`).
|
||||
|
||||
- Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
|
||||
returning only the last header (:issue:`5515`, :issue:`5526`).
|
||||
|
||||
- Fixed :class:`LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` not ignoring the
|
||||
``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`,
|
||||
:issue:`4066`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Clarified the return type of :meth:`Spider.parse <scrapy.Spider.parse>`
|
||||
(:issue:`5602`, :issue:`5608`).
|
||||
|
||||
- To enable
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
|
||||
to do `brotli compression`_, installing brotli_ is now recommended instead
|
||||
of installing brotlipy_, as the former provides a more recent version of
|
||||
brotli.
|
||||
|
||||
.. _brotli: https://github.com/google/brotli
|
||||
.. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt
|
||||
|
||||
- :ref:`Signal documentation <topics-signals>` now mentions :ref:`coroutine
|
||||
support <topics-coroutines>` and uses it in code examples (:issue:`4852`,
|
||||
:issue:`5358`).
|
||||
|
||||
- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_
|
||||
(:issue:`3582`, :issue:`5432`).
|
||||
|
||||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
|
||||
- The new :ref:`topics-components` topic covers enforcing requirements on
|
||||
Scrapy components, like :ref:`downloader middlewares
|
||||
<topics-downloader-middleware>`, :ref:`extensions <topics-extensions>`,
|
||||
:ref:`item pipelines <topics-item-pipeline>`, :ref:`spider middlewares
|
||||
<topics-spider-middleware>`, and more; :ref:`enforce-asyncio-requirement`
|
||||
has also been added (:issue:`4978`).
|
||||
|
||||
- :ref:`topics-settings` now indicates that setting values must be
|
||||
:ref:`picklable <pickle-picklable>` (:issue:`5607`, :issue:`5629`).
|
||||
|
||||
- Removed outdated documentation (:issue:`5446`, :issue:`5373`,
|
||||
:issue:`5369`, :issue:`5370`, :issue:`5554`).
|
||||
|
||||
- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`,
|
||||
:issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`).
|
||||
|
||||
- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`,
|
||||
:issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`).
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added a continuous integration job to run `twine check`_ (:issue:`5655`,
|
||||
:issue:`5656`).
|
||||
|
||||
.. _twine check: https://twine.readthedocs.io/en/stable/#twine-check
|
||||
|
||||
- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`,
|
||||
:issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`,
|
||||
:issue:`5671`, :issue:`5675`).
|
||||
|
||||
- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`,
|
||||
:issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`).
|
||||
|
||||
- Applied minor code improvements (:issue:`5661`).
|
||||
|
||||
|
||||
.. _release-2.6.3:
|
||||
|
||||
Scrapy 2.6.3 (2022-09-27)
|
||||
-------------------------
|
||||
|
||||
- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3
|
||||
(:issue:`5634`, :issue:`5635`, :issue:`5636`).
|
||||
|
||||
- Upgraded the minimum versions of the following dependencies:
|
||||
|
||||
- cryptography_: 2.0 → 3.3
|
||||
|
||||
- pyOpenSSL_: 16.2.0 → 21.0.0
|
||||
|
||||
- service_identity_: 16.0.0 → 18.1.0
|
||||
|
||||
- Twisted_: 17.9.0 → 18.9.0
|
||||
|
||||
- zope.interface_: 4.1.3 → 5.0.0
|
||||
|
||||
(:issue:`5621`, :issue:`5632`)
|
||||
|
||||
- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`,
|
||||
:issue:`5631`).
|
||||
|
||||
|
||||
.. _release-2.6.2:
|
||||
|
||||
Scrapy 2.6.2 (2022-07-25)
|
||||
|
|
@ -3113,7 +3328,7 @@ New Features
|
|||
~~~~~~~~~~~~
|
||||
|
||||
- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
|
||||
- Support `brotli`_-compressed content; requires optional `brotlipy`_
|
||||
- Support `brotli-compressed`_ content; requires optional `brotlipy`_
|
||||
(:issue:`2535`)
|
||||
- New :ref:`response.follow <response-follow-example>` shortcut
|
||||
for creating requests (:issue:`1940`)
|
||||
|
|
@ -3150,7 +3365,7 @@ New Features
|
|||
- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
|
||||
(:issue:`2740`)
|
||||
|
||||
.. _brotli: https://github.com/google/brotli
|
||||
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
||||
.. _brotlipy: https://github.com/python-hyper/brotlipy/
|
||||
|
||||
Bug fixes
|
||||
|
|
|
|||
|
|
@ -75,9 +75,9 @@ If your requirement is a minimum Scrapy version, you may use
|
|||
class MyComponent:
|
||||
|
||||
def __init__(self):
|
||||
if parse_version(scrapy.__version__) < parse_version('VERSION'):
|
||||
if parse_version(scrapy.__version__) < parse_version('2.7'):
|
||||
raise RuntimeError(
|
||||
f"{MyComponent.__qualname__} requires Scrapy VERSION or "
|
||||
f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
|
||||
f"later, which allow defining the process_spider_output "
|
||||
f"method of spider middlewares as an asynchronous "
|
||||
f"generator."
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ override three methods:
|
|||
.. method:: Contract.post_process(output)
|
||||
|
||||
This allows processing the output of the callback. Iterators are
|
||||
converted listified before being passed to this hook.
|
||||
converted to lists before being passed to this hook.
|
||||
|
||||
Raise :class:`~scrapy.exceptions.ContractFail` from
|
||||
:class:`~scrapy.contracts.Contract.pre_process` or
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
|||
If you are using any custom or third-party :ref:`spider middleware
|
||||
<topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
|
||||
|
||||
.. versionchanged:: VERSION
|
||||
.. versionchanged:: 2.7
|
||||
Output of async callbacks is now processed asynchronously instead of
|
||||
collecting all of it first.
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
|||
See also :ref:`sync-async-spider-middleware` and
|
||||
:ref:`universal-spider-middleware`.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.7
|
||||
|
||||
General usage
|
||||
=============
|
||||
|
|
@ -129,7 +129,7 @@ Common use cases for asynchronous code include:
|
|||
Mixing synchronous and asynchronous spider middlewares
|
||||
======================================================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.7
|
||||
|
||||
The output of a :class:`~scrapy.Request` callback is passed as the ``result``
|
||||
parameter to the
|
||||
|
|
@ -182,10 +182,10 @@ process_spider_output_async method <universal-spider-middleware>`.
|
|||
Universal spider middlewares
|
||||
============================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.7
|
||||
|
||||
To allow writing a spider middleware that supports asynchronous execution of
|
||||
its ``process_spider_output`` method in Scrapy VERSION and later (avoiding
|
||||
its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
|
||||
:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
|
||||
while maintaining support for older Scrapy versions, you may define
|
||||
``process_spider_output`` as a synchronous method and define an
|
||||
|
|
@ -206,7 +206,7 @@ For example::
|
|||
yield r
|
||||
|
||||
.. note:: This is an interim measure to allow, for a time, to write code that
|
||||
works in Scrapy VERSION and later without requiring
|
||||
works in Scrapy 2.7 and later without requiring
|
||||
asynchronous-to-synchronous conversions, and works in earlier Scrapy
|
||||
versions as well.
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ settings, just like any other Scrapy code.
|
|||
|
||||
It is customary for extensions to prefix their settings with their own name, to
|
||||
avoid collision with existing (and future) extensions. For example, a
|
||||
hypothetic extension to handle `Google Sitemaps`_ would use settings like
|
||||
hypothetical extension to handle `Google Sitemaps`_ would use settings like
|
||||
``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
|
||||
|
||||
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ Too many spiders?
|
|||
If your project has too many spiders executed in parallel,
|
||||
the output of :func:`prefs()` can be difficult to read.
|
||||
For this reason, that function has a ``ignore`` argument which can be used to
|
||||
ignore a particular class (and all its subclases). For
|
||||
ignore a particular class (and all its subclasses). For
|
||||
example, this won't show any live references to spiders:
|
||||
|
||||
>>> from scrapy.spiders import Spider
|
||||
|
|
|
|||
|
|
@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the
|
|||
REQUEST_FINGERPRINTER_CLASS
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.7
|
||||
|
||||
Default: :class:`scrapy.utils.request.RequestFingerprinter`
|
||||
|
||||
|
|
@ -409,54 +409,54 @@ import path.
|
|||
REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.7
|
||||
|
||||
Default: ``'PREVIOUS_VERSION'``
|
||||
Default: ``'2.6'``
|
||||
|
||||
Determines which request fingerprinting algorithm is used by the default
|
||||
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
|
||||
|
||||
Possible values are:
|
||||
|
||||
- ``'PREVIOUS_VERSION'`` (default)
|
||||
- ``'2.6'`` (default)
|
||||
|
||||
This implementation uses the same request fingerprinting algorithm as
|
||||
Scrapy PREVIOUS_VERSION and earlier versions.
|
||||
Scrapy 2.6 and earlier versions.
|
||||
|
||||
Even though this is the default value for backward compatibility reasons,
|
||||
it is a deprecated value.
|
||||
|
||||
- ``'VERSION'``
|
||||
- ``'2.7'``
|
||||
|
||||
This implementation was introduced in Scrapy VERSION to fix an issue of the
|
||||
This implementation was introduced in Scrapy 2.7 to fix an issue of the
|
||||
previous implementation.
|
||||
|
||||
New projects should use this value. The :command:`startproject` command
|
||||
sets this value in the generated ``settings.py`` file.
|
||||
|
||||
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
|
||||
If you are using the default value (``'2.6'``) for this setting, and you are
|
||||
using Scrapy components where changing the request fingerprinting algorithm
|
||||
would cause undesired results, you need to carefully decide when to change the
|
||||
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
|
||||
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
|
||||
setting to a custom request fingerprinter class that implements the 2.6 request
|
||||
fingerprinting algorithm and does not log this warning (
|
||||
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
|
||||
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
|
||||
class).
|
||||
|
||||
Scenarios where changing the request fingerprinting algorithm may cause
|
||||
undesired results include, for example, using the HTTP cache middleware (see
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
Changing the request fingerprinting algorithm would invalidade the current
|
||||
Changing the request fingerprinting algorithm would invalidate the current
|
||||
cache, requiring you to redownload all requests again.
|
||||
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
|
||||
your settings to switch already to the request fingerprinting implementation
|
||||
that will be the only request fingerprinting implementation available in a
|
||||
future version of Scrapy, and remove the deprecation warning triggered by using
|
||||
the default value (``'PREVIOUS_VERSION'``).
|
||||
the default value (``'2.6'``).
|
||||
|
||||
|
||||
.. _PREVIOUS_VERSION-request-fingerprinter:
|
||||
.. _2.6-request-fingerprinter:
|
||||
.. _custom-request-fingerprinter:
|
||||
|
||||
Writing your own request fingerprinter
|
||||
|
|
@ -464,6 +464,8 @@ Writing your own request fingerprinter
|
|||
|
||||
A request fingerprinter is a class that must implement the following method:
|
||||
|
||||
.. currentmodule:: None
|
||||
|
||||
.. method:: fingerprint(self, request)
|
||||
|
||||
Return a :class:`bytes` object that uniquely identifies *request*.
|
||||
|
|
@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method:
|
|||
Additionally, it may also implement the following methods:
|
||||
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
:noindex:
|
||||
|
||||
If present, this class method is called to create a request fingerprinter
|
||||
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
|
||||
|
|
@ -495,11 +498,13 @@ Additionally, it may also implement the following methods:
|
|||
:class:`~scrapy.settings.Settings` object. It must return a new instance of
|
||||
the request fingerprinter.
|
||||
|
||||
The ``fingerprint`` method of the default request fingerprinter,
|
||||
.. currentmodule:: scrapy.http
|
||||
|
||||
The :meth:`fingerprint` method of the default request fingerprinter,
|
||||
:class:`scrapy.utils.request.RequestFingerprinter`, uses
|
||||
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
|
||||
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
|
||||
in your ``fingerprint`` method implementation:
|
||||
common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
|
||||
in your :meth:`fingerprint` method implementation:
|
||||
|
||||
.. autofunction:: scrapy.utils.request.fingerprint
|
||||
|
||||
|
|
@ -519,7 +524,7 @@ account::
|
|||
|
||||
You can also write your own fingerprinting logic from scratch.
|
||||
|
||||
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
|
||||
However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
|
||||
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
|
||||
|
||||
- Caching saves CPU by ensuring that fingerprints are calculated only once
|
||||
|
|
@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary
|
|||
requests from your spider callbacks, you may implement a request fingerprinter
|
||||
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
|
||||
when available, and then falls back to
|
||||
:func:`~scrapy.utils.request.fingerprint`. For example::
|
||||
:func:`scrapy.utils.request.fingerprint`. For example::
|
||||
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
|
|
@ -564,8 +569,8 @@ when available, and then falls back to
|
|||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
|
||||
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
|
||||
If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
|
||||
without using the deprecated ``'2.6'`` value of the
|
||||
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
|
||||
request fingerprinter::
|
||||
|
||||
|
|
|
|||
|
|
@ -98,6 +98,10 @@ class.
|
|||
The global defaults are located in the ``scrapy.settings.default_settings``
|
||||
module and documented in the :ref:`topics-settings-ref` section.
|
||||
|
||||
Compatibility with pickle
|
||||
=========================
|
||||
|
||||
Setting values must be :ref:`picklable <pickle-picklable>`.
|
||||
|
||||
Import paths and classes
|
||||
========================
|
||||
|
|
@ -560,7 +564,6 @@ This setting must be one of these string values:
|
|||
set this if you want the behavior of Scrapy<1.1
|
||||
- ``'TLSv1.1'``: forces TLS version 1.1
|
||||
- ``'TLSv1.2'``: forces TLS version 1.2
|
||||
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
|
||||
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
|
@ -1639,6 +1642,11 @@ install the default reactor defined by Twisted for the current platform. This
|
|||
is to maintain backward compatibility and avoid possible problems caused by
|
||||
using a non-default reactor.
|
||||
|
||||
.. versionchanged:: 2.7
|
||||
The :command:`startproject` command now sets this setting to
|
||||
``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
|
||||
``settings.py`` file.
|
||||
|
||||
For additional information, see :doc:`core/howto/choosing-reactor`.
|
||||
|
||||
|
||||
|
|
@ -1653,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength``
|
|||
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
This setting can act as a stopping condition in case of URLs of ever-increasing
|
||||
length, which may be caused for example by a programming error either in the
|
||||
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
|
||||
This setting can act as a stopping condition in case of URLs of ever-increasing
|
||||
length, which may be caused for example by a programming error either in the
|
||||
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
|
||||
:setting:`DEPTH_LIMIT`.
|
||||
|
||||
Use ``0`` to allow URLs of any length.
|
||||
|
||||
The default value is copied from the `Microsoft Internet Explorer maximum URL
|
||||
The default value is copied from the `Microsoft Internet Explorer maximum URL
|
||||
length`_, even though this setting exists for different reasons.
|
||||
|
||||
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
|
||||
|
|
|
|||
|
|
@ -105,17 +105,17 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:class:`~scrapy.Request` objects and :ref:`item objects
|
||||
<topics-items>`.
|
||||
|
||||
.. versionchanged:: VERSION
|
||||
.. versionchanged:: 2.7
|
||||
This method may be defined as an :term:`asynchronous generator`, in
|
||||
which case ``result`` is an :term:`asynchronous iterable`.
|
||||
|
||||
Consider defining this method as an :term:`asynchronous generator`,
|
||||
which will be a requirement in a future version of Scrapy. However, if
|
||||
you plan on sharing your spider middleware with other people, consider
|
||||
either :ref:`enforcing Scrapy VERSION <enforce-component-requirements>`
|
||||
either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
|
||||
as a minimum requirement of your spider middleware, or :ref:`making
|
||||
your spider middleware universal <universal-spider-middleware>` so that
|
||||
it works with Scrapy versions earlier than Scrapy VERSION.
|
||||
it works with Scrapy versions earlier than Scrapy 2.7.
|
||||
|
||||
:param response: the response which generated this output from the
|
||||
spider
|
||||
|
|
@ -130,7 +130,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
|
||||
.. method:: process_spider_output_async(response, result, spider)
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.7
|
||||
|
||||
If defined, this method must be an :term:`asynchronous generator`,
|
||||
which will be called instead of :meth:`process_spider_output` if
|
||||
|
|
|
|||
|
|
@ -181,9 +181,10 @@ scrapy.Spider
|
|||
scraped data and/or more URLs to follow. Other Requests callbacks have
|
||||
the same requirements as the :class:`Spider` class.
|
||||
|
||||
This method, as well as any other Request callback, must return an
|
||||
iterable of :class:`~scrapy.Request` and/or :ref:`item objects
|
||||
<topics-items>`.
|
||||
This method, as well as any other Request callback, must return a
|
||||
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
|
||||
iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
|
||||
<topics-items>`, or ``None``.
|
||||
|
||||
:param response: the response to parse
|
||||
:type response: :class:`~scrapy.http.Response`
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.6.2
|
||||
2.7.0
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from typing import Dict
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
from w3lib.url import is_url
|
||||
|
||||
from twisted.internet.defer import maybeDeferred
|
||||
|
||||
from scrapy.commands import BaseRunSpiderCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils import display
|
||||
|
|
@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand):
|
|||
if not opts.nolinks:
|
||||
self.print_requests(colour=colour)
|
||||
|
||||
def run_callback(self, response, callback, cb_kwargs=None):
|
||||
cb_kwargs = cb_kwargs or {}
|
||||
def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
|
||||
items, requests = [], []
|
||||
|
||||
for x in iterate_spider_output(callback(response, **cb_kwargs)):
|
||||
for x in spider_output:
|
||||
if is_item(x):
|
||||
items.append(x)
|
||||
elif isinstance(x, Request):
|
||||
requests.append(x)
|
||||
return items, requests
|
||||
return items, requests, opts, depth, spider, callback
|
||||
|
||||
def run_callback(self, response, callback, cb_kwargs=None):
|
||||
cb_kwargs = cb_kwargs or {}
|
||||
d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
|
||||
return d
|
||||
|
||||
def get_callback_from_rules(self, spider, response):
|
||||
if getattr(spider, 'rules', None):
|
||||
|
|
@ -158,6 +163,25 @@ class Command(BaseRunSpiderCommand):
|
|||
logger.error('No response downloaded for: %(url)s',
|
||||
{'url': url})
|
||||
|
||||
def scraped_data(self, args):
|
||||
items, requests, opts, depth, spider, callback = args
|
||||
if opts.pipelines:
|
||||
itemproc = self.pcrawler.engine.scraper.itemproc
|
||||
for item in items:
|
||||
itemproc.process_item(item, spider)
|
||||
self.add_items(depth, items)
|
||||
self.add_requests(depth, requests)
|
||||
|
||||
scraped_data = items if opts.output else []
|
||||
if depth < opts.depth:
|
||||
for req in requests:
|
||||
req.meta['_depth'] = depth + 1
|
||||
req.meta['_callback'] = req.callback
|
||||
req.callback = callback
|
||||
scraped_data += requests
|
||||
|
||||
return scraped_data
|
||||
|
||||
def prepare_request(self, spider, request, opts):
|
||||
def callback(response, **cb_kwargs):
|
||||
# memorize first request
|
||||
|
|
@ -191,23 +215,10 @@ class Command(BaseRunSpiderCommand):
|
|||
# parse items and requests
|
||||
depth = response.meta['_depth']
|
||||
|
||||
items, requests = self.run_callback(response, cb, cb_kwargs)
|
||||
if opts.pipelines:
|
||||
itemproc = self.pcrawler.engine.scraper.itemproc
|
||||
for item in items:
|
||||
itemproc.process_item(item, spider)
|
||||
self.add_items(depth, items)
|
||||
self.add_requests(depth, requests)
|
||||
|
||||
scraped_data = items if opts.output else []
|
||||
if depth < opts.depth:
|
||||
for req in requests:
|
||||
req.meta['_depth'] = depth + 1
|
||||
req.meta['_callback'] = req.callback
|
||||
req.callback = callback
|
||||
scraped_data += requests
|
||||
|
||||
return scraped_data
|
||||
d = self.run_callback(response, cb, cb_kwargs)
|
||||
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
|
||||
d.addCallback(self.scraped_data)
|
||||
return d
|
||||
|
||||
# update request meta if any extra meta was passed through the --meta/-m opts.
|
||||
if opts.meta:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
which allows TLS protocol negotiation
|
||||
|
||||
'A TLS/SSL connection established with [this method] may
|
||||
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
|
||||
understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
|
||||
"""
|
||||
|
||||
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -384,8 +384,7 @@ class ScrapyAgent:
|
|||
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
|
||||
{"request": request, "handler": handler.__qualname__})
|
||||
txresponse._transport.stopProducing()
|
||||
with suppress(AttributeError):
|
||||
txresponse._transport._producer.loseConnection()
|
||||
txresponse._transport.loseConnection()
|
||||
return {
|
||||
"txresponse": txresponse,
|
||||
"body": b"",
|
||||
|
|
@ -417,7 +416,7 @@ class ScrapyAgent:
|
|||
|
||||
logger.warning(warning_msg, warning_args)
|
||||
|
||||
txresponse._transport._producer.loseConnection()
|
||||
txresponse._transport.loseConnection()
|
||||
raise defer.CancelledError(warning_msg % warning_args)
|
||||
|
||||
if warnsize and expected_size > warnsize:
|
||||
|
|
@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol):
|
|||
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
|
||||
{"request": self._request, "handler": handler.__qualname__})
|
||||
self.transport.stopProducing()
|
||||
self.transport._producer.loseConnection()
|
||||
self.transport.loseConnection()
|
||||
failure = result if result.value.fail else None
|
||||
self._finish_response(flags=["download_stopped"], failure=failure)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
METHOD_SSLv3 = 'SSLv3'
|
||||
METHOD_TLS = 'TLS'
|
||||
METHOD_TLSv10 = 'TLSv1.0'
|
||||
METHOD_TLSv11 = 'TLSv1.1'
|
||||
|
|
@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2'
|
|||
|
||||
openssl_methods = {
|
||||
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
|
||||
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
|
||||
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
|
||||
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
|
||||
|
|
|
|||
|
|
@ -31,7 +31,12 @@ from scrapy.utils.log import (
|
|||
)
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
|
||||
from scrapy.utils.reactor import install_reactor, verify_installed_reactor
|
||||
from scrapy.utils.reactor import (
|
||||
install_reactor,
|
||||
is_asyncio_reactor_installed,
|
||||
verify_installed_asyncio_event_loop,
|
||||
verify_installed_reactor,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -78,17 +83,20 @@ class Crawler:
|
|||
crawler=self,
|
||||
)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
reactor_class = self.settings["TWISTED_REACTOR"]
|
||||
event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
# but before something imports twisted.internet.reactor
|
||||
if reactor_class:
|
||||
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
install_reactor(reactor_class, event_loop)
|
||||
else:
|
||||
from twisted.internet import reactor # noqa: F401
|
||||
log_reactor_info()
|
||||
if reactor_class:
|
||||
verify_installed_reactor(reactor_class)
|
||||
if is_asyncio_reactor_installed() and event_loop:
|
||||
verify_installed_asyncio_event_loop(event_loop)
|
||||
|
||||
self.extensions = ExtensionManager.from_crawler(self)
|
||||
|
||||
|
|
|
|||
|
|
@ -160,7 +160,14 @@ class ImagesPipeline(FilesPipeline):
|
|||
|
||||
if size:
|
||||
image = image.copy()
|
||||
image.thumbnail(size, self._Image.ANTIALIAS)
|
||||
try:
|
||||
# Image.Resampling.LANCZOS was added in Pillow 9.1.0
|
||||
# remove this try except block,
|
||||
# when updating the minimum requirements for Pillow.
|
||||
resampling_filter = self._Image.Resampling.LANCZOS
|
||||
except AttributeError:
|
||||
resampling_filter = self._Image.ANTIALIAS
|
||||
image.thumbnail(size, resampling_filter)
|
||||
|
||||
buf = BytesIO()
|
||||
image.save(buf, 'JPEG')
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ REFERER_ENABLED = True
|
|||
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6'
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
|
|
|
|||
|
|
@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst
|
|||
"""
|
||||
|
||||
import copy
|
||||
from typing import Sequence
|
||||
from typing import AsyncIterable, Awaitable, Sequence
|
||||
|
||||
from scrapy.http import Request, HtmlResponse
|
||||
from scrapy.http import Request, Response, HtmlResponse
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.asyncgen import collect_asyncgen
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
|
|
@ -78,7 +79,7 @@ class CrawlSpider(Spider):
|
|||
def parse_start_url(self, response, **kwargs):
|
||||
return []
|
||||
|
||||
def process_results(self, response, results):
|
||||
def process_results(self, response: Response, results: list):
|
||||
return results
|
||||
|
||||
def _build_request(self, rule_index, link):
|
||||
|
|
@ -109,9 +110,13 @@ class CrawlSpider(Spider):
|
|||
rule = self._rules[failure.request.meta['rule']]
|
||||
return self._handle_failure(failure, rule.errback)
|
||||
|
||||
def _parse_response(self, response, callback, cb_kwargs, follow=True):
|
||||
async def _parse_response(self, response, callback, cb_kwargs, follow=True):
|
||||
if callback:
|
||||
cb_res = callback(response, **cb_kwargs) or ()
|
||||
if isinstance(cb_res, AsyncIterable):
|
||||
cb_res = await collect_asyncgen(cb_res)
|
||||
elif isinstance(cb_res, Awaitable):
|
||||
cb_res = await cb_res
|
||||
cb_res = self.process_results(response, cb_res)
|
||||
for request_or_item in iterate_spider_output(cb_res):
|
||||
yield request_or_item
|
||||
|
|
|
|||
|
|
@ -88,4 +88,5 @@ ROBOTSTXT_OBEY = True
|
|||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
|
||||
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from typing import AsyncGenerator, AsyncIterable, Iterable, Union
|
||||
|
||||
|
||||
async def collect_asyncgen(result: AsyncIterable):
|
||||
async def collect_asyncgen(result: AsyncIterable) -> list:
|
||||
results = []
|
||||
async for x in result:
|
||||
results.append(x)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support
|
|||
import ctypes
|
||||
import platform
|
||||
import sys
|
||||
from distutils.version import LooseVersion as parse_version
|
||||
from packaging.version import Version as parse_version
|
||||
from pprint import pformat as pformat_
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -226,7 +226,14 @@ def is_generator_with_return_value(callable):
|
|||
return value is None or isinstance(value, ast.NameConstant) and value.value is None
|
||||
|
||||
if inspect.isgeneratorfunction(callable):
|
||||
code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
|
||||
src = inspect.getsource(callable)
|
||||
pattern = re.compile(r"(^[\t ]+)")
|
||||
code = pattern.sub("", src)
|
||||
|
||||
match = pattern.match(src) # finds indentation
|
||||
if match:
|
||||
code = re.sub(f"\n{match.group(0)}", "\n", code) # remove indentation
|
||||
|
||||
tree = ast.parse(code)
|
||||
for node in walk_callable(tree):
|
||||
if isinstance(node, ast.Return) and not returns_none(node):
|
||||
|
|
|
|||
|
|
@ -180,23 +180,6 @@ def binary_is_text(data):
|
|||
return all(c not in _BINARYCHARS for c in data)
|
||||
|
||||
|
||||
def _getargspec_py23(func):
|
||||
"""_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
|
||||
defaults)
|
||||
|
||||
Was identical to inspect.getargspec() in python2, but uses
|
||||
inspect.getfullargspec() for python3 behind the scenes to avoid
|
||||
DeprecationWarning.
|
||||
|
||||
>>> def f(a, b=2, *ar, **kw):
|
||||
... pass
|
||||
|
||||
>>> _getargspec_py23(f)
|
||||
ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
|
||||
"""
|
||||
return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
|
||||
|
||||
|
||||
def get_func_args(func, stripself=False):
|
||||
"""Return the argument name list of a callable"""
|
||||
if inspect.isfunction(func):
|
||||
|
|
@ -248,9 +231,9 @@ def get_spec(func):
|
|||
"""
|
||||
|
||||
if inspect.isfunction(func) or inspect.ismethod(func):
|
||||
spec = _getargspec_py23(func)
|
||||
spec = inspect.getfullargspec(func)
|
||||
elif hasattr(func, '__call__'):
|
||||
spec = _getargspec_py23(func.__call__)
|
||||
spec = inspect.getfullargspec(func.__call__)
|
||||
else:
|
||||
raise TypeError(f'{type(func)} is not callable')
|
||||
|
||||
|
|
|
|||
|
|
@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path):
|
|||
raise Exception(msg)
|
||||
|
||||
|
||||
def verify_installed_asyncio_event_loop(loop_path):
|
||||
from twisted.internet import reactor
|
||||
loop_class = load_object(loop_path)
|
||||
if isinstance(reactor._asyncioEventloop, loop_class):
|
||||
return
|
||||
installed = (
|
||||
f"{reactor._asyncioEventloop.__class__.__module__}"
|
||||
f".{reactor._asyncioEventloop.__class__.__qualname__}"
|
||||
)
|
||||
specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
|
||||
raise Exception(
|
||||
"Scrapy found an asyncio Twisted reactor already "
|
||||
f"installed, and its event loop class ({installed}) does "
|
||||
"not match the one specified in the ASYNCIO_EVENT_LOOP "
|
||||
f"setting ({specified})"
|
||||
)
|
||||
|
||||
|
||||
def is_asyncio_reactor_installed():
|
||||
from twisted.internet import reactor
|
||||
return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
|
||||
|
|
|
|||
|
|
@ -236,10 +236,10 @@ class RequestFingerprinter:
|
|||
'REQUEST_FINGERPRINTER_IMPLEMENTATION'
|
||||
)
|
||||
else:
|
||||
implementation = 'PREVIOUS_VERSION'
|
||||
if implementation == 'PREVIOUS_VERSION':
|
||||
implementation = '2.6'
|
||||
if implementation == '2.6':
|
||||
message = (
|
||||
'\'PREVIOUS_VERSION\' is a deprecated value for the '
|
||||
'\'2.6\' is a deprecated value for the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
|
||||
'\n'
|
||||
'It is also the default value. In other words, it is normal '
|
||||
|
|
@ -254,14 +254,14 @@ class RequestFingerprinter:
|
|||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._fingerprint = _request_fingerprint_as_bytes
|
||||
elif implementation == 'VERSION':
|
||||
elif implementation == '2.7':
|
||||
self._fingerprint = fingerprint
|
||||
else:
|
||||
raise ValueError(
|
||||
f'Got an invalid value on setting '
|
||||
f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
|
||||
f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
|
||||
f'and \'VERSION\'.'
|
||||
f'{implementation!r}. Valid values are \'2.6\' (deprecated) '
|
||||
f'and \'2.7\'.'
|
||||
)
|
||||
|
||||
def fingerprint(self, request):
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
|
|||
# Set by default settings that prevent deprecation warnings.
|
||||
settings = {}
|
||||
if prevent_warnings:
|
||||
settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
|
||||
settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7'
|
||||
settings.update(settings_dict or {})
|
||||
runner = CrawlerRunner(settings)
|
||||
return runner.create_crawler(spidercls or Spider)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect):
|
|||
def render(self, request):
|
||||
content = util.Redirect.render(self, request)
|
||||
return content.replace(b'http-equiv=\"refresh\"',
|
||||
b'http-no-equiv=\"do-not-refresh-me\"')
|
||||
b'http-no-equiv=\"do-not-refresh-me\"')
|
||||
|
||||
|
||||
def test_site():
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ def url_has_any_extension(url, extensions):
|
|||
lowercase_path = parse_url(url).path.lower()
|
||||
return any(lowercase_path.endswith(ext) for ext in extensions)
|
||||
|
||||
|
||||
def parse_url(url, encoding=None):
|
||||
"""Return urlparsed url from the given argument (which could be an already
|
||||
parsed url)
|
||||
|
|
|
|||
8
setup.py
8
setup.py
|
|
@ -20,18 +20,19 @@ def has_environment_marker_platform_impl_support():
|
|||
|
||||
install_requires = [
|
||||
'Twisted>=18.9.0',
|
||||
'cryptography>=2.8',
|
||||
'cryptography>=3.3',
|
||||
'cssselect>=0.9.1',
|
||||
'itemloaders>=1.0.1',
|
||||
'parsel>=1.5.0',
|
||||
'pyOpenSSL>=19.1.0',
|
||||
'pyOpenSSL>=21.0.0',
|
||||
'queuelib>=1.4.2',
|
||||
'service_identity>=16.0.0',
|
||||
'service_identity>=18.1.0',
|
||||
'w3lib>=1.17.0',
|
||||
'zope.interface>=5.1.0',
|
||||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
'setuptools',
|
||||
'packaging',
|
||||
'tldextract',
|
||||
'lxml>=4.3.0',
|
||||
]
|
||||
|
|
@ -82,6 +83,7 @@ setup(
|
|||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Programming Language :: Python :: 3.11',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Programming Language :: Python :: Implementation :: PyPy',
|
||||
'Topic :: Internet :: WWW/HTTP',
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from typing import Optional
|
|||
from scrapy import Spider
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.defer import deferred_from_coro
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
||||
class UppercasePipeline:
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32":
|
|||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncioreactor.install(asyncio.get_event_loop())
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
import scrapy # noqa: E402
|
||||
from scrapy.crawler import CrawlerProcess # noqa: E402
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
import asyncio
|
||||
import sys
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
if sys.version_info >= (3, 8) and sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncioreactor.install(asyncio.get_event_loop())
|
||||
|
||||
import scrapy # noqa: E402
|
||||
from scrapy.crawler import CrawlerProcess # noqa: E402
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = 'no_request'
|
||||
|
||||
def start_requests(self):
|
||||
return []
|
||||
|
||||
|
||||
process = CrawlerProcess(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"ASYNCIO_EVENT_LOOP": "uvloop.Loop",
|
||||
})
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import asyncio
|
||||
import sys
|
||||
|
||||
from uvloop import Loop
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
if sys.version_info >= (3, 8) and sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncio.set_event_loop(Loop())
|
||||
asyncioreactor.install(asyncio.get_event_loop())
|
||||
|
||||
import scrapy # noqa: E402
|
||||
from scrapy.crawler import CrawlerProcess # noqa: E402
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = 'no_request'
|
||||
|
||||
def start_requests(self):
|
||||
return []
|
||||
|
||||
|
||||
process = CrawlerProcess(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"ASYNCIO_EVENT_LOOP": "uvloop.Loop",
|
||||
})
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from twisted.internet import reactor
|
||||
from twisted.internet import reactor # noqa: F401
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
|
|
@ -14,4 +14,3 @@ process = CrawlerProcess(settings={})
|
|||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from twisted.internet import reactor
|
||||
from twisted.internet import reactor # noqa: F401
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
|
|
@ -16,5 +16,3 @@ process = CrawlerProcess(settings={
|
|||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,5 +15,3 @@ process = CrawlerProcess(settings={})
|
|||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,3 @@ process = CrawlerProcess(settings={
|
|||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,3 @@ process = CrawlerProcess(settings={
|
|||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -369,6 +369,30 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
|
|||
yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
|
||||
|
||||
|
||||
class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
|
||||
"""A CrawlSpider with an async def callback"""
|
||||
name = 'crawl_spider_with_async_callback'
|
||||
rules = (
|
||||
Rule(LinkExtractor(), callback='parse_async', follow=True),
|
||||
)
|
||||
|
||||
async def parse_async(self, response, foo=None):
|
||||
self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo)
|
||||
return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
|
||||
|
||||
|
||||
class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
|
||||
"""A CrawlSpider with an async generator callback"""
|
||||
name = 'crawl_spider_with_async_generator_callback'
|
||||
rules = (
|
||||
Rule(LinkExtractor(), callback='parse_async_gen', follow=True),
|
||||
)
|
||||
|
||||
async def parse_async_gen(self, response, foo=None):
|
||||
self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo)
|
||||
yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"})
|
||||
|
||||
|
||||
class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
|
||||
name = 'crawl_spider_with_errback'
|
||||
rules = (
|
||||
|
|
|
|||
|
|
@ -29,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
|
|||
import scrapy
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.spiders import CrawlSpider, Rule
|
||||
from scrapy.utils.test import get_from_asyncio_queue
|
||||
|
||||
class AsyncDefAsyncioSpider(scrapy.Spider):
|
||||
|
||||
name = 'asyncdef{self.spider_name}'
|
||||
|
||||
async def parse(self, response):
|
||||
status = await get_from_asyncio_queue(response.status)
|
||||
return [scrapy.Item(), dict(foo='bar')]
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = '{self.spider_name}'
|
||||
|
|
@ -160,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
|
|||
self.url('/html')])
|
||||
self.assertIn("INFO: It Works!", _textmode(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_asyncio_parse_items(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')]
|
||||
)
|
||||
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_parse_items(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
|
|
|
|||
|
|
@ -689,8 +689,15 @@ class MySpider(scrapy.Spider):
|
|||
])
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
def test_asyncio_enabled_false(self):
|
||||
def test_asyncio_enabled_default(self):
|
||||
log = self.get_log(self.debug_log_spider, args=[])
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
def test_asyncio_enabled_false(self):
|
||||
log = self.get_log(self.debug_log_spider, args=[
|
||||
'-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor'
|
||||
])
|
||||
self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
|
||||
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ from tests.spiders import (
|
|||
BrokenStartRequestsSpider,
|
||||
BytesReceivedCallbackSpider,
|
||||
BytesReceivedErrbackSpider,
|
||||
CrawlSpiderWithAsyncCallback,
|
||||
CrawlSpiderWithAsyncGeneratorCallback,
|
||||
CrawlSpiderWithErrback,
|
||||
CrawlSpiderWithParseMethod,
|
||||
DelaySpider,
|
||||
|
|
@ -348,7 +350,7 @@ with multiples lines
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawl_multiple(self):
|
||||
runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
|
||||
runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
|
||||
runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
|
||||
runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
|
||||
|
||||
|
|
@ -391,6 +393,26 @@ class CrawlSpiderTestCase(TestCase):
|
|||
self.assertIn("[parse] status 201 (foo: None)", str(log))
|
||||
self.assertIn("[parse] status 202 (foo: bar)", str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_with_async_callback(self):
|
||||
crawler = get_crawler(CrawlSpiderWithAsyncCallback)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
|
||||
self.assertIn("[parse_async] status 200 (foo: None)", str(log))
|
||||
self.assertIn("[parse_async] status 201 (foo: None)", str(log))
|
||||
self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_with_async_generator_callback(self):
|
||||
crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
|
||||
self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log))
|
||||
self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log))
|
||||
self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_with_errback(self):
|
||||
crawler = get_crawler(CrawlSpiderWithErrback)
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
|
|||
'LOG_LEVEL': 'INFO',
|
||||
'LOG_FILE': log_file,
|
||||
# settings to avoid extra warnings
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
|
||||
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
|
||||
}
|
||||
|
||||
|
|
@ -235,7 +235,7 @@ class NoRequestsSpider(scrapy.Spider):
|
|||
class CrawlerRunnerHasSpider(unittest.TestCase):
|
||||
|
||||
def _runner(self):
|
||||
return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
|
||||
return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawler_runner_bootstrap_successful(self):
|
||||
|
|
@ -283,14 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
if self.reactor_pytest == 'asyncio':
|
||||
CrawlerRunner(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
|
||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
|
||||
})
|
||||
else:
|
||||
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
|
||||
with self.assertRaisesRegex(Exception, msg):
|
||||
runner = CrawlerRunner(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
|
||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
|
||||
})
|
||||
yield runner.crawl(NoRequestsSpider)
|
||||
|
||||
|
|
@ -327,7 +327,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
|
||||
def test_reactor_default_twisted_reactor_select(self):
|
||||
log = self.run_script('reactor_default_twisted_reactor_select.py')
|
||||
if platform.system() == 'Windows':
|
||||
if platform.system() in ['Windows', 'Darwin']:
|
||||
# The goal of this test function is to test that, when a reactor is
|
||||
# installed (the default one here) and a different reactor is
|
||||
# configured (select here), an error raises.
|
||||
|
|
@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
self.assertIn("async pipeline opened!", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
|
||||
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
|
||||
@mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
|
||||
def test_asyncio_enabled_reactor_same_loop(self):
|
||||
log = self.run_script("asyncio_enabled_reactor_same_loop.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
|
||||
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
|
||||
@mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
|
||||
def test_asyncio_enabled_reactor_different_loop(self):
|
||||
log = self.run_script("asyncio_enabled_reactor_different_loop.py")
|
||||
self.assertNotIn("Spider closed (finished)", log)
|
||||
self.assertIn(
|
||||
(
|
||||
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
|
||||
"setting (uvloop.Loop)"
|
||||
),
|
||||
log,
|
||||
)
|
||||
|
||||
def test_default_loop_asyncio_deferred_signal(self):
|
||||
log = self.run_script("asyncio_deferred_signal.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_df_from_crawler_scheduler(self):
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertTrue(scheduler.df.debug)
|
||||
|
|
@ -60,7 +60,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_df_from_settings_scheduler(self):
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertTrue(scheduler.df.debug)
|
||||
|
|
@ -68,7 +68,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
def test_df_direct_scheduler(self):
|
||||
settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertEqual(scheduler.df.method, 'n/a')
|
||||
|
|
@ -172,7 +172,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': False,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
|
@ -199,7 +199,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
|
@ -233,7 +233,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_log_debug_default_dupefilter(self):
|
||||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
|
|
|||
|
|
@ -2285,6 +2285,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
for expected_batch, got_batch in zip(expected, data[fmt]):
|
||||
self.assertEqual(expected_batch, got_batch)
|
||||
|
||||
@pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output')
|
||||
@defer.inlineCallbacks
|
||||
def test_batch_path_differ(self):
|
||||
"""
|
||||
|
|
@ -2305,7 +2306,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
|
||||
}
|
||||
data = yield self.exported_data(items, settings)
|
||||
self.assertEqual(len(items) + 1, len(data['json']))
|
||||
self.assertEqual(len(items), len([_ for _ in data['json'] if _]))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_stats_batch_file_success(self):
|
||||
|
|
|
|||
|
|
@ -705,7 +705,8 @@ class HtmlResponseTest(TextResponseTest):
|
|||
|
||||
def test_html_encoding(self):
|
||||
|
||||
body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
body = b"""<html><head><title>Some page</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
</head><body>Price: \xa3100</body></html>'
|
||||
"""
|
||||
r1 = self.response_class("http://www.example.com", body=body)
|
||||
|
|
@ -719,7 +720,8 @@ class HtmlResponseTest(TextResponseTest):
|
|||
self._assert_response_values(r2, 'iso-8859-1', body)
|
||||
|
||||
# for conflicting declarations headers must take precedence
|
||||
body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
body = b"""<html><head><title>Some page</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
</head><body>Price: \xa3100</body></html>'
|
||||
"""
|
||||
r3 = self.response_class("http://www.example.com", body=body,
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
|
||||
l.add_css('name', 'div::text')
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
|
||||
|
||||
def test_init_method_with_base_response(self):
|
||||
"""Selector should be None after initialization"""
|
||||
response = Response("https://scrapy.org")
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):
|
|||
self.tmpmediastore = self.mktemp()
|
||||
os.mkdir(self.tmpmediastore)
|
||||
self.settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
|
||||
'ITEM_PIPELINES': {self.pipeline_class: 1},
|
||||
self.store_setting_key: self.tmpmediastore,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class MockCrawler(Crawler):
|
|||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION',
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7',
|
||||
)
|
||||
super().__init__(Spider, settings)
|
||||
self.engine = MockEngine(downloader=MockDownloader())
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class SpiderLoaderTest(unittest.TestCase):
|
|||
module = 'tests.test_spiderloader.test_spiders.spider1'
|
||||
runner = CrawlerRunner({
|
||||
'SPIDER_MODULES': [module],
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
|
||||
})
|
||||
|
||||
self.assertRaisesRegex(KeyError, 'Spider not found',
|
||||
|
|
|
|||
|
|
@ -165,6 +165,89 @@ https://example.org
|
|||
warn_on_generator_with_return_value(None, l2)
|
||||
self.assertEqual(len(w), 0)
|
||||
|
||||
def test_generators_return_none_with_decorator(self):
|
||||
def decorator(func):
|
||||
def inner_func():
|
||||
func()
|
||||
return inner_func
|
||||
|
||||
@decorator
|
||||
def f3():
|
||||
yield 1
|
||||
return None
|
||||
|
||||
@decorator
|
||||
def g3():
|
||||
yield 1
|
||||
return
|
||||
|
||||
@decorator
|
||||
def h3():
|
||||
yield 1
|
||||
|
||||
@decorator
|
||||
def i3():
|
||||
yield 1
|
||||
yield from generator_that_returns_stuff()
|
||||
|
||||
@decorator
|
||||
def j3():
|
||||
yield 1
|
||||
|
||||
def helper():
|
||||
return 0
|
||||
|
||||
yield helper()
|
||||
|
||||
@decorator
|
||||
def k3():
|
||||
"""
|
||||
docstring
|
||||
"""
|
||||
url = """
|
||||
https://example.org
|
||||
"""
|
||||
yield url
|
||||
return
|
||||
|
||||
@decorator
|
||||
def l3():
|
||||
return
|
||||
|
||||
assert not is_generator_with_return_value(top_level_return_none)
|
||||
assert not is_generator_with_return_value(f3)
|
||||
assert not is_generator_with_return_value(g3)
|
||||
assert not is_generator_with_return_value(h3)
|
||||
assert not is_generator_with_return_value(i3)
|
||||
assert not is_generator_with_return_value(j3) # not recursive
|
||||
assert not is_generator_with_return_value(k3) # not recursive
|
||||
assert not is_generator_with_return_value(l3)
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, top_level_return_none)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, f3)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, g3)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, h3)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, i3)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, j3)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, k3)
|
||||
self.assertEqual(len(w), 0)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warn_on_generator_with_return_value(None, l3)
|
||||
self.assertEqual(len(w), 0)
|
||||
|
||||
@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
|
||||
def test_indentation_error(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
|
|
|
|||
|
|
@ -505,7 +505,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
|
|||
|
||||
def test_deprecated_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
|
@ -518,7 +518,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
|
|||
|
||||
def test_recommended_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
|
|
|||
26
tox.ini
26
tox.ini
|
|
@ -38,8 +38,9 @@ install_command =
|
|||
basepython = python3
|
||||
deps =
|
||||
lxml-stubs==0.2.0
|
||||
mypy==0.971
|
||||
types-pyOpenSSL==20.0.3
|
||||
mypy==0.982
|
||||
types-attrs==19.1.0
|
||||
types-pyOpenSSL==21.0.0
|
||||
types-setuptools==57.0.0
|
||||
commands =
|
||||
mypy --show-error-codes {posargs: scrapy tests}
|
||||
|
|
@ -57,31 +58,38 @@ deps =
|
|||
{[testenv]deps}
|
||||
# Twisted[http2] is required to import some files
|
||||
Twisted[http2]>=17.9.0
|
||||
pytest-flake8==1.1.1
|
||||
flake8==4.0.1
|
||||
flake8==5.0.4
|
||||
commands =
|
||||
pytest --flake8 {posargs:docs scrapy tests}
|
||||
flake8 {posargs:docs scrapy tests}
|
||||
|
||||
[testenv:pylint]
|
||||
# reppy does not support Python 3.9+
|
||||
basepython = python3.8
|
||||
deps =
|
||||
{[testenv:extra-deps]deps}
|
||||
pylint==2.14.5
|
||||
pylint==2.15.3
|
||||
commands =
|
||||
pylint conftest.py docs extras scrapy setup.py tests
|
||||
|
||||
[testenv:twinecheck]
|
||||
basepython = python3
|
||||
deps =
|
||||
twine==4.0.1
|
||||
commands =
|
||||
python setup.py sdist
|
||||
twine check dist/*
|
||||
|
||||
[pinned]
|
||||
deps =
|
||||
cryptography==2.8
|
||||
cryptography==3.3
|
||||
cssselect==0.9.1
|
||||
h2==3.0
|
||||
itemadapter==0.1.0
|
||||
parsel==1.5.0
|
||||
Protego==0.1.15
|
||||
pyOpenSSL==19.1.0
|
||||
pyOpenSSL==21.0.0
|
||||
queuelib==1.4.2
|
||||
service_identity==16.0.0
|
||||
service_identity==18.1.0
|
||||
Twisted[http2]==18.9.0
|
||||
w3lib==1.17.0
|
||||
zope.interface==5.1.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue