mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into 4307-use-f-strings
Conflicts resolved: - delete tests/py36/_test_crawl.py
This commit is contained in:
Ammar Najjar
commit
defeaacbc2
|
|
@ -40,7 +40,7 @@ including a list of features.
|
|||
Requirements
|
||||
============
|
||||
|
||||
* Python 3.5.2+
|
||||
* Python 3.6+
|
||||
* Works on Linux, Windows, macOS, BSD
|
||||
|
||||
Install
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ collect_ignore = [
|
|||
*_py_files("tests/CrawlerProcess"),
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
|
||||
*_py_files("tests/CrawlerRunner"),
|
||||
# Py36-only parts of respective tests
|
||||
*_py_files("tests/py36"),
|
||||
]
|
||||
|
||||
for line in open('tests/ignores.txt'):
|
||||
|
|
|
|||
|
|
@ -199,6 +199,17 @@ In any case, if something is covered in a docstring, use the
|
|||
documentation instead of duplicating the docstring in files within the
|
||||
``docs/`` directory.
|
||||
|
||||
Documentation updates that cover new or modified features must use Sphinx’s
|
||||
:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use
|
||||
``VERSION`` as version, we will replace it with the actual version right before
|
||||
the corresponding release. When we release a new major or minor version of
|
||||
Scrapy, we remove these directives if they are older than 3 years.
|
||||
|
||||
Documentation about deprecated features must be removed as those features are
|
||||
deprecated, so that new readers do not run into it. New deprecations and
|
||||
deprecation removals are documented in the :ref:`release notes <news>`.
|
||||
|
||||
|
||||
Tests
|
||||
=====
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ Installation guide
|
|||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
|
||||
the PyPy 5.9+ implementation (see :ref:`python:implementations`).
|
||||
Scrapy requires Python 3.6+, either the CPython implementation (default) or
|
||||
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
|
||||
Installing Scrapy
|
||||
|
|
|
|||
|
|
@ -405,8 +405,6 @@ to get all of them:
|
|||
|
||||
from sys import version_info
|
||||
|
||||
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
|
||||
|
||||
Having figured out how to extract each bit, we can now iterate over all the
|
||||
quotes elements and put them together into a Python dictionary:
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
Core API
|
||||
========
|
||||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
This section documents the Scrapy core API, and it's intended for developers of
|
||||
extensions and middlewares.
|
||||
|
||||
|
|
|
|||
|
|
@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies.
|
|||
AUTOTHROTTLE_TARGET_CONCURRENCY
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 1.1
|
||||
|
||||
Default: ``1.0``
|
||||
|
||||
Average number of requests Scrapy should be sending in parallel to remote
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
Benchmarking
|
||||
============
|
||||
|
||||
.. versionadded:: 0.17
|
||||
|
||||
Scrapy comes with a simple benchmarking suite that spawns a local HTTP server
|
||||
and crawls it at the maximum possible speed. The goal of this benchmarking is
|
||||
to get an idea of how Scrapy performs in your hardware, in order to have a
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@
|
|||
Command line tool
|
||||
=================
|
||||
|
||||
.. versionadded:: 0.10
|
||||
|
||||
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
|
||||
here as the "Scrapy tool" to differentiate it from the sub-commands, which we
|
||||
just call "commands" or "Scrapy commands".
|
||||
|
|
@ -566,8 +564,6 @@ and Platform info, which is useful for bug reports.
|
|||
bench
|
||||
-----
|
||||
|
||||
.. versionadded:: 0.17
|
||||
|
||||
* Syntax: ``scrapy bench``
|
||||
* Requires project: *no*
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
Spiders Contracts
|
||||
=================
|
||||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
Testing spiders can get particularly annoying and while nothing prevents you
|
||||
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
|
||||
integrated way of testing your spiders by the means of contracts.
|
||||
|
|
|
|||
|
|
@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
|||
|
||||
- :class:`~scrapy.http.Request` callbacks.
|
||||
|
||||
The following are known caveats of the current implementation that we aim
|
||||
to address in future versions of Scrapy:
|
||||
|
||||
- The callback output is not processed until the whole callback finishes.
|
||||
.. note:: The callback output is not processed until the whole callback
|
||||
finishes.
|
||||
|
||||
As a side effect, if the callback raises an exception, none of its
|
||||
output is processed.
|
||||
|
||||
- Because `asynchronous generators were introduced in Python 3.6`_, you
|
||||
can only use ``yield`` if you are using Python 3.6 or later.
|
||||
|
||||
If you need to output multiple items or requests and you are using
|
||||
Python 3.5, return an iterable (e.g. a list) instead.
|
||||
This is a known caveat of the current implementation that we aim to
|
||||
address in a future version of Scrapy.
|
||||
|
||||
- The :meth:`process_item` method of
|
||||
:ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
|
@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
|||
|
||||
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
|
||||
|
||||
.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
|
|
|
|||
|
|
@ -217,8 +217,6 @@ The following settings can be used to configure the cookie middleware:
|
|||
Multiple cookie sessions per spider
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
There is support for keeping multiple cookie sessions per spider by using the
|
||||
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
|
||||
(session), but you can pass an identifier to use different ones.
|
||||
|
|
@ -475,8 +473,6 @@ DBM storage backend
|
|||
|
||||
.. class:: DbmCacheStorage
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
A DBM_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
By default, it uses the :mod:`dbm`, but you can change it with the
|
||||
|
|
@ -549,15 +545,10 @@ settings:
|
|||
HTTPCACHE_ENABLED
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.11
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Whether the HTTP cache will be enabled.
|
||||
|
||||
.. versionchanged:: 0.11
|
||||
Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache.
|
||||
|
||||
.. setting:: HTTPCACHE_EXPIRATION_SECS
|
||||
|
||||
HTTPCACHE_EXPIRATION_SECS
|
||||
|
|
@ -570,9 +561,6 @@ Expiration time for cached requests, in seconds.
|
|||
Cached requests older than this time will be re-downloaded. If zero, cached
|
||||
requests will never expire.
|
||||
|
||||
.. versionchanged:: 0.11
|
||||
Before 0.11, zero meant cached requests always expire.
|
||||
|
||||
.. setting:: HTTPCACHE_DIR
|
||||
|
||||
HTTPCACHE_DIR
|
||||
|
|
@ -589,8 +577,6 @@ project data dir. For more info see: :ref:`topics-project-structure`.
|
|||
HTTPCACHE_IGNORE_HTTP_CODES
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.10
|
||||
|
||||
Default: ``[]``
|
||||
|
||||
Don't cache response with these HTTP codes.
|
||||
|
|
@ -609,8 +595,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade
|
|||
HTTPCACHE_IGNORE_SCHEMES
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.10
|
||||
|
||||
Default: ``['file']``
|
||||
|
||||
Don't cache responses with these URI schemes.
|
||||
|
|
@ -629,8 +613,6 @@ The class which implements the cache storage backend.
|
|||
HTTPCACHE_DBM_MODULE
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``'dbm'``
|
||||
|
||||
The database module to use in the :ref:`DBM storage backend
|
||||
|
|
@ -641,8 +623,6 @@ The database module to use in the :ref:`DBM storage backend
|
|||
HTTPCACHE_POLICY
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.18
|
||||
|
||||
Default: ``'scrapy.extensions.httpcache.DummyPolicy'``
|
||||
|
||||
The class which implements the cache policy.
|
||||
|
|
@ -652,8 +632,6 @@ The class which implements the cache policy.
|
|||
HTTPCACHE_GZIP
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 1.0
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If enabled, will compress all cached data with gzip.
|
||||
|
|
@ -664,8 +642,6 @@ This setting is specific to the Filesystem backend.
|
|||
HTTPCACHE_ALWAYS_STORE
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 1.1
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If enabled, will cache pages unconditionally.
|
||||
|
|
@ -684,8 +660,6 @@ responses you feed to the cache middleware.
|
|||
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 1.1
|
||||
|
||||
Default: ``[]``
|
||||
|
||||
List of Cache-Control directives in responses to be ignored.
|
||||
|
|
@ -735,8 +709,6 @@ HttpProxyMiddleware
|
|||
.. module:: scrapy.downloadermiddlewares.httpproxy
|
||||
:synopsis: Http Proxy Middleware
|
||||
|
||||
.. versionadded:: 0.8
|
||||
|
||||
.. reqmeta:: proxy
|
||||
|
||||
.. class:: HttpProxyMiddleware
|
||||
|
|
@ -817,8 +789,6 @@ RedirectMiddleware settings
|
|||
REDIRECT_ENABLED
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether the Redirect middleware will be enabled.
|
||||
|
|
@ -860,8 +830,6 @@ MetaRefreshMiddleware settings
|
|||
METAREFRESH_ENABLED
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.17
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether the Meta Refresh middleware will be enabled.
|
||||
|
|
@ -924,8 +892,6 @@ RetryMiddleware Settings
|
|||
RETRY_ENABLED
|
||||
^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether the Retry middleware will be enabled.
|
||||
|
|
@ -1179,8 +1145,6 @@ AjaxCrawlMiddleware Settings
|
|||
AJAXCRAWL_ENABLED
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.21
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Whether the AjaxCrawlMiddleware will be enabled. You may want to
|
||||
|
|
|
|||
|
|
@ -292,7 +292,7 @@ XmlItemExporter
|
|||
CsvItemExporter
|
||||
---------------
|
||||
|
||||
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
|
||||
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs)
|
||||
|
||||
Exports items in CSV format to the given file-like object. If the
|
||||
:attr:`fields_to_export` attribute is set, it will be used to define the
|
||||
|
|
@ -311,6 +311,11 @@ CsvItemExporter
|
|||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
|
||||
:param errors: The optional string that specifies how encoding and decoding
|
||||
errors are to be handled. For more information see
|
||||
:class:`io.TextIOWrapper`.
|
||||
:type errors: str
|
||||
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||
:func:`csv.writer` function, so you can use any :func:`csv.writer` function
|
||||
|
|
|
|||
|
|
@ -288,8 +288,6 @@ If zero (or non set), spiders won't be closed by number of passed items.
|
|||
CLOSESPIDER_PAGECOUNT
|
||||
"""""""""""""""""""""
|
||||
|
||||
.. versionadded:: 0.11
|
||||
|
||||
Default: ``0``
|
||||
|
||||
An integer which specifies the maximum number of responses to crawl. If the spider
|
||||
|
|
@ -302,8 +300,6 @@ number of crawled responses.
|
|||
CLOSESPIDER_ERRORCOUNT
|
||||
""""""""""""""""""""""
|
||||
|
||||
.. versionadded:: 0.11
|
||||
|
||||
Default: ``0``
|
||||
|
||||
An integer which specifies the maximum number of errors to receive before
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
Feed exports
|
||||
============
|
||||
|
||||
.. versionadded:: 0.10
|
||||
|
||||
One of the most frequently required features when implementing scrapers is
|
||||
being able to store the scraped data properly and, quite often, that means
|
||||
generating an "export file" with the scraped data (commonly called "export
|
||||
|
|
|
|||
|
|
@ -553,18 +553,6 @@ fields with form data from :class:`Response` objects.
|
|||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` ``__init__`` method.
|
||||
|
||||
.. versionadded:: 0.10.3
|
||||
The ``formname`` parameter.
|
||||
|
||||
.. versionadded:: 0.17
|
||||
The ``formxpath`` parameter.
|
||||
|
||||
.. versionadded:: 1.1.0
|
||||
The ``formcss`` parameter.
|
||||
|
||||
.. versionadded:: 1.1.0
|
||||
The ``formid`` parameter.
|
||||
|
||||
Request usage examples
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -1076,8 +1076,6 @@ See :ref:`topics-extensions-ref-memusage`.
|
|||
MEMUSAGE_CHECK_INTERVAL_SECONDS
|
||||
-------------------------------
|
||||
|
||||
.. versionadded:: 1.1
|
||||
|
||||
Default: ``60.0``
|
||||
|
||||
Scope: ``scrapy.extensions.memusage``
|
||||
|
|
@ -1358,8 +1356,6 @@ The class that will be used for loading spiders, which must implement the
|
|||
SPIDER_LOADER_WARN_ONLY
|
||||
-----------------------
|
||||
|
||||
.. versionadded:: 1.3.3
|
||||
|
||||
Default: ``False``
|
||||
|
||||
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
|
||||
|
|
|
|||
|
|
@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
|
||||
.. method:: process_start_requests(start_requests, spider)
|
||||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
This method is called with the start requests of the spider, and works
|
||||
similarly to the :meth:`process_spider_output` method, except that it
|
||||
doesn't have a response associated and must return only requests (not
|
||||
|
|
@ -341,8 +339,6 @@ RefererMiddleware settings
|
|||
REFERER_ENABLED
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether to enable referer middleware.
|
||||
|
|
@ -352,8 +348,6 @@ Whether to enable referer middleware.
|
|||
REFERRER_POLICY
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 1.4
|
||||
|
||||
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
|
||||
|
||||
.. reqmeta:: referrer_policy
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
|||
|
||||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 5, 2):
|
||||
print("Scrapy %s requires Python 3.5.2" % __version__)
|
||||
if sys.version_info < (3, 6):
|
||||
print("Scrapy %s requires Python 3.6+" % __version__)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter):
|
|||
|
||||
class CsvItemExporter(BaseItemExporter):
|
||||
|
||||
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
|
||||
def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
|
||||
super().__init__(dont_fail=True, **kwargs)
|
||||
if not self.encoding:
|
||||
self.encoding = 'utf-8'
|
||||
|
|
@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
line_buffering=False,
|
||||
write_through=True,
|
||||
encoding=self.encoding,
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
errors=errors,
|
||||
)
|
||||
self.csv_writer = csv.writer(self.stream, **self._kwargs)
|
||||
self._headers_not_written = True
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst
|
|||
"""
|
||||
import logging
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request
|
||||
|
|
@ -18,8 +19,8 @@ class Spider(object_ref):
|
|||
class.
|
||||
"""
|
||||
|
||||
name = None
|
||||
custom_settings = None
|
||||
name: Optional[str] = None
|
||||
custom_settings: Optional[dict] = None
|
||||
|
||||
def __init__(self, name=None, **kwargs):
|
||||
if name is not None:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst
|
|||
|
||||
import copy
|
||||
import warnings
|
||||
from typing import Sequence
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, HtmlResponse
|
||||
|
|
@ -72,7 +73,7 @@ class Rule:
|
|||
|
||||
class CrawlSpider(Spider):
|
||||
|
||||
rules = ()
|
||||
rules: Sequence[Rule] = ()
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super().__init__(*a, **kw)
|
||||
|
|
|
|||
|
|
@ -124,18 +124,11 @@ def iter_errback(iterable, errback, *a, **kw):
|
|||
errback(failure.Failure(), *a, **kw)
|
||||
|
||||
|
||||
def _isfuture(o):
|
||||
# workaround for Python before 3.5.3 not having asyncio.isfuture
|
||||
if hasattr(asyncio, 'isfuture'):
|
||||
return asyncio.isfuture(o)
|
||||
return isinstance(o, asyncio.Future)
|
||||
|
||||
|
||||
def deferred_from_coro(o):
|
||||
"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
|
||||
if isinstance(o, defer.Deferred):
|
||||
return o
|
||||
if _isfuture(o) or inspect.isawaitable(o):
|
||||
if asyncio.isfuture(o) or inspect.isawaitable(o):
|
||||
if not is_asyncio_reactor_installed():
|
||||
# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
|
||||
# that use asyncio, e.g. "await asyncio.sleep(1)"
|
||||
|
|
@ -167,7 +160,7 @@ def maybeDeferred_coro(f, *args, **kw):
|
|||
|
||||
if isinstance(result, defer.Deferred):
|
||||
return result
|
||||
elif _isfuture(result) or inspect.isawaitable(result):
|
||||
elif asyncio.isfuture(result) or inspect.isawaitable(result):
|
||||
return deferred_from_coro(result)
|
||||
elif isinstance(result, failure.Failure):
|
||||
return defer.fail(result)
|
||||
|
|
|
|||
|
|
@ -6,11 +6,10 @@ import struct
|
|||
from scrapy.utils.decorators import deprecated
|
||||
|
||||
|
||||
# - Python>=3.5 GzipFile's read() has issues returning leftover
|
||||
# uncompressed data when input is corrupted
|
||||
# (regression or bug-fix compared to Python 3.4)
|
||||
# - GzipFile's read() has issues returning leftover uncompressed data when
|
||||
# input is corrupted
|
||||
# - read1(), which fetches data before raising EOFError on next call
|
||||
# works here but is only available from Python>=3.3
|
||||
# works here
|
||||
@deprecated('GzipFile.read1')
|
||||
def read1(gzf, size=-1):
|
||||
return gzf.read1(size)
|
||||
|
|
|
|||
|
|
@ -12,10 +12,12 @@ def render_templatefile(path, **kwargs):
|
|||
content = string.Template(raw).substitute(**kwargs)
|
||||
|
||||
render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
|
||||
|
||||
if path.endswith('.tmpl'):
|
||||
os.rename(path, render_path)
|
||||
|
||||
with open(render_path, 'wb') as fp:
|
||||
fp.write(content.encode('utf8'))
|
||||
if path.endswith('.tmpl'):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
|
||||
|
|
|
|||
51
setup.cfg
51
setup.cfg
|
|
@ -16,9 +16,6 @@ ignore_errors = True
|
|||
[mypy-scrapy.commands]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.commands.bench]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.commands.parse]
|
||||
ignore_errors = True
|
||||
|
||||
|
|
@ -28,9 +25,6 @@ ignore_errors = True
|
|||
[mypy-scrapy.contracts]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.core.spidermw]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.interfaces]
|
||||
ignore_errors = True
|
||||
|
||||
|
|
@ -70,15 +64,6 @@ ignore_errors = True
|
|||
[mypy-tests.mocks.dummydbm]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.spiders]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_command_fetch]
|
||||
ignore_errors = True
|
||||
|
||||
|
|
@ -94,9 +79,6 @@ ignore_errors = True
|
|||
[mypy-tests.test_contracts]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_crawler]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_downloader_handlers]
|
||||
ignore_errors = True
|
||||
|
||||
|
|
@ -127,53 +109,20 @@ ignore_errors = True
|
|||
[mypy-tests.test_pipeline_images]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_pipelines]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_attribute_binding]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_cb_kwargs]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_left]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_scheduler]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_signals]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.spider1]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.spider2]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.spider3]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spidermiddleware_httperror]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spidermiddleware_output_chain]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spidermiddleware_referer]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_reqser]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_serialize]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_spider]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_url]
|
||||
ignore_errors = True
|
||||
|
|
|
|||
3
setup.py
3
setup.py
|
|
@ -82,7 +82,6 @@ setup(
|
|||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
|
|
@ -92,7 +91,7 @@ setup(
|
|||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=3.5.2',
|
||||
python_requires='>=3.6',
|
||||
install_requires=install_requires,
|
||||
extras_require=extras_require,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
import asyncio
|
||||
|
||||
from scrapy import Request
|
||||
from tests.spiders import SimpleSpider
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen'
|
||||
|
||||
async def parse(self, response):
|
||||
await asyncio.sleep(0.2)
|
||||
yield {'foo': 42}
|
||||
self.logger.info("Got response %d" % response.status)
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen_loop'
|
||||
|
||||
async def parse(self, response):
|
||||
for i in range(10):
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'foo': i}
|
||||
self.logger.info("Got response %d" % response.status)
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen_complex'
|
||||
initial_reqs = 4
|
||||
following_reqs = 3
|
||||
depth = 2
|
||||
|
||||
def _get_req(self, index, cb=None):
|
||||
return Request(self.mockserver.url(f"/status?n=200&request={index}"),
|
||||
meta={'index': index},
|
||||
dont_filter=True,
|
||||
callback=cb)
|
||||
|
||||
def start_requests(self):
|
||||
for i in range(1, self.initial_reqs + 1):
|
||||
yield self._get_req(i)
|
||||
|
||||
async def parse(self, response):
|
||||
index = response.meta['index']
|
||||
yield {'index': index}
|
||||
if index < 10 ** self.depth:
|
||||
for new_index in range(10 * index, 10 * index + self.following_reqs):
|
||||
yield self._get_req(new_index)
|
||||
yield self._get_req(index, cb=self.parse2)
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'index': index + 5}
|
||||
|
||||
async def parse2(self, response):
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'index2': response.meta['index']}
|
||||
|
|
@ -2,8 +2,7 @@
|
|||
attrs
|
||||
dataclasses; python_version == '3.6'
|
||||
mitmproxy; python_version >= '3.7'
|
||||
mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'
|
||||
mitmproxy < 4; python_version < '3.6'
|
||||
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7'
|
||||
pyftpdlib
|
||||
# https://github.com/pytest-dev/pytest-twisted/issues/93
|
||||
pytest != 5.4, != 5.4.1
|
||||
|
|
|
|||
|
|
@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
|
|||
return reqs
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen'
|
||||
|
||||
async def parse(self, response):
|
||||
await asyncio.sleep(0.2)
|
||||
yield {'foo': 42}
|
||||
self.logger.info("Got response %d" % response.status)
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen_loop'
|
||||
|
||||
async def parse(self, response):
|
||||
for i in range(10):
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'foo': i}
|
||||
self.logger.info("Got response %d" % response.status)
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen_complex'
|
||||
initial_reqs = 4
|
||||
following_reqs = 3
|
||||
depth = 2
|
||||
|
||||
def _get_req(self, index, cb=None):
|
||||
return Request(self.mockserver.url("/status?n=200&request=%d" % index),
|
||||
meta={'index': index},
|
||||
dont_filter=True,
|
||||
callback=cb)
|
||||
|
||||
def start_requests(self):
|
||||
for i in range(1, self.initial_reqs + 1):
|
||||
yield self._get_req(i)
|
||||
|
||||
async def parse(self, response):
|
||||
index = response.meta['index']
|
||||
yield {'index': index}
|
||||
if index < 10 ** self.depth:
|
||||
for new_index in range(10 * index, 10 * index + self.following_reqs):
|
||||
yield self._get_req(new_index)
|
||||
yield self._get_req(index, cb=self.parse2)
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'index': index + 5}
|
||||
|
||||
async def parse2(self, response):
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'index2': response.meta['index']}
|
||||
|
||||
|
||||
class ItemSpider(FollowAllSpider):
|
||||
|
||||
name = 'item'
|
||||
|
|
@ -255,7 +308,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
|
|||
A CrawlSpider which overrides the 'parse' method
|
||||
"""
|
||||
name = 'crawl_spider_with_parse_method'
|
||||
custom_settings = {
|
||||
custom_settings: dict = {
|
||||
'RETRY_HTTP_CODES': [], # no need to retry
|
||||
}
|
||||
rules = (
|
||||
|
|
|
|||
|
|
@ -128,9 +128,13 @@ class StartprojectTest(ProjectTest):
|
|||
|
||||
|
||||
def get_permissions_dict(path, renamings=None, ignore=None):
|
||||
|
||||
def get_permissions(path):
|
||||
return oct(os.stat(path).st_mode)
|
||||
|
||||
renamings = renamings or tuple()
|
||||
permissions_dict = {
|
||||
'.': os.stat(path).st_mode,
|
||||
'.': get_permissions(path),
|
||||
}
|
||||
for root, dirs, files in os.walk(path):
|
||||
nodes = list(chain(dirs, files))
|
||||
|
|
@ -145,13 +149,15 @@ def get_permissions_dict(path, renamings=None, ignore=None):
|
|||
search_string,
|
||||
replacement
|
||||
)
|
||||
permissions = os.stat(absolute_path).st_mode
|
||||
permissions = get_permissions(absolute_path)
|
||||
permissions_dict[relative_path] = permissions
|
||||
return permissions_dict
|
||||
|
||||
|
||||
class StartprojectTemplatesTest(ProjectTest):
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.tmpl = join(self.temp_path, 'templates')
|
||||
|
|
@ -293,7 +299,7 @@ class StartprojectTemplatesTest(ProjectTest):
|
|||
path.mkdir(mode=permissions)
|
||||
else:
|
||||
path.touch(mode=permissions)
|
||||
expected_permissions[node] = path.stat().st_mode
|
||||
expected_permissions[node] = oct(path.stat().st_mode)
|
||||
|
||||
process = subprocess.Popen(
|
||||
(
|
||||
|
|
@ -313,6 +319,53 @@ class StartprojectTemplatesTest(ProjectTest):
|
|||
|
||||
self.assertEqual(actual_permissions, expected_permissions)
|
||||
|
||||
def test_startproject_permissions_umask_022(self):
|
||||
"""Check that generated files have the right permissions when the
|
||||
system uses a umask value that causes new files to have different
|
||||
permissions than those from the template folder."""
|
||||
@contextmanager
|
||||
def umask(new_mask):
|
||||
cur_mask = os.umask(new_mask)
|
||||
yield
|
||||
os.umask(cur_mask)
|
||||
|
||||
scrapy_path = scrapy.__path__[0]
|
||||
project_template = os.path.join(
|
||||
scrapy_path,
|
||||
'templates',
|
||||
'project'
|
||||
)
|
||||
project_name = 'umaskproject'
|
||||
renamings = (
|
||||
('module', project_name),
|
||||
('.tmpl', ''),
|
||||
)
|
||||
expected_permissions = get_permissions_dict(
|
||||
project_template,
|
||||
renamings,
|
||||
IGNORE,
|
||||
)
|
||||
|
||||
with umask(0o002):
|
||||
destination = mkdtemp()
|
||||
process = subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
'-m',
|
||||
'scrapy.cmdline',
|
||||
'startproject',
|
||||
project_name,
|
||||
),
|
||||
cwd=destination,
|
||||
env=self.env,
|
||||
)
|
||||
process.wait()
|
||||
|
||||
project_dir = os.path.join(destination, project_name)
|
||||
actual_permissions = get_permissions_dict(project_dir)
|
||||
|
||||
self.assertEqual(actual_permissions, expected_permissions)
|
||||
|
||||
|
||||
class CommandTest(ProjectTest):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import json
|
||||
import logging
|
||||
import sys
|
||||
from ipaddress import IPv4Address
|
||||
from socket import gethostbyname
|
||||
from urllib.parse import urlparse
|
||||
|
|
@ -20,6 +19,9 @@ from scrapy.http.response import Response
|
|||
from scrapy.utils.python import to_unicode
|
||||
from tests.mockserver import MockServer
|
||||
from tests.spiders import (
|
||||
AsyncDefAsyncioGenComplexSpider,
|
||||
AsyncDefAsyncioGenLoopSpider,
|
||||
AsyncDefAsyncioGenSpider,
|
||||
AsyncDefAsyncioReqsReturnSpider,
|
||||
AsyncDefAsyncioReturnSingleElementSpider,
|
||||
AsyncDefAsyncioReturnSpider,
|
||||
|
|
@ -405,11 +407,9 @@ class CrawlSpiderTestCase(TestCase):
|
|||
self.assertIn("Got response 200", str(log))
|
||||
self.assertIn({"foo": 42}, items)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncgen_parse(self):
|
||||
from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
|
||||
crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
|
||||
|
|
@ -417,7 +417,6 @@ class CrawlSpiderTestCase(TestCase):
|
|||
itemcount = crawler.stats.get_value('item_scraped_count')
|
||||
self.assertEqual(itemcount, 1)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_loop(self):
|
||||
|
|
@ -426,7 +425,6 @@ class CrawlSpiderTestCase(TestCase):
|
|||
def _on_item_scraped(item):
|
||||
items.append(item)
|
||||
|
||||
from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
|
||||
crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
|
||||
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
|
||||
with LogCapture() as log:
|
||||
|
|
@ -437,7 +435,6 @@ class CrawlSpiderTestCase(TestCase):
|
|||
for i in range(10):
|
||||
self.assertIn({'foo': i}, items)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_complex(self):
|
||||
|
|
@ -446,7 +443,6 @@ class CrawlSpiderTestCase(TestCase):
|
|||
def _on_item_scraped(item):
|
||||
items.append(item)
|
||||
|
||||
from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
|
||||
crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
|
||||
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
|
|
|
|||
|
|
@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
expected='22,False,3.14,2015-01-01 01:01:01\r\n'
|
||||
)
|
||||
|
||||
def test_errors_default(self):
|
||||
with self.assertRaises(UnicodeEncodeError):
|
||||
self.assertExportResult(
|
||||
item=dict(text=u'W\u0275\u200Brd'),
|
||||
expected=None,
|
||||
encoding='windows-1251',
|
||||
)
|
||||
|
||||
def test_errors_xmlcharrefreplace(self):
|
||||
self.assertExportResult(
|
||||
item=dict(text=u'W\u0275\u200Brd'),
|
||||
include_headers_line=False,
|
||||
expected='Wɵ​rd\r\n',
|
||||
encoding='windows-1251',
|
||||
errors='xmlcharrefreplace',
|
||||
)
|
||||
|
||||
|
||||
class CsvItemExporterDataclassTest(CsvItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
from warnings import catch_warnings
|
||||
|
|
@ -7,9 +6,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
|
|||
from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
|
||||
|
||||
|
||||
PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
|
||||
|
||||
|
||||
class ItemTest(unittest.TestCase):
|
||||
|
||||
def assertSortedEqual(self, first, second, msg=None):
|
||||
|
|
@ -280,14 +276,6 @@ class ItemMetaTest(unittest.TestCase):
|
|||
with mock.patch.object(base, '__new__', new_mock):
|
||||
|
||||
class MyItem(Item):
|
||||
if not PY36_PLUS:
|
||||
# This attribute is an internal attribute in Python 3.6+
|
||||
# and must be propagated properly. See
|
||||
# https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object
|
||||
# In <3.6, we add a dummy attribute just to ensure the
|
||||
# __new__ method propagates it correctly.
|
||||
__classcell__ = object()
|
||||
|
||||
def f(self):
|
||||
# For rationale of this see:
|
||||
# https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from subprocess import Popen, PIPE
|
|||
from urllib.parse import urlsplit, urlunsplit
|
||||
from unittest import skipIf
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
|
@ -58,8 +57,6 @@ def _wrong_credentials(proxy_url):
|
|||
return urlunsplit(bad_auth_proxy)
|
||||
|
||||
|
||||
@skipIf(sys.version_info < (3, 5, 4),
|
||||
"requires mitmproxy < 3.0.0, which these tests do not support")
|
||||
@skipIf("pypy" in sys.executable,
|
||||
"mitmproxy does not support PyPy")
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
|
||||
|
|
@ -88,14 +85,6 @@ class ProxyConnectTestCase(TestCase):
|
|||
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
|
||||
self._assert_got_response_code(200, log)
|
||||
|
||||
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6))
|
||||
@defer.inlineCallbacks
|
||||
def test_https_connect_tunnel_error(self):
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl("https://localhost:99999/status?n=200")
|
||||
self._assert_got_tunnel_error(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_https_tunnel_auth_error(self):
|
||||
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
|
||||
|
|
|
|||
2
tox.ini
2
tox.ini
|
|
@ -89,7 +89,7 @@ deps =
|
|||
basepython = python3
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
# First lxml version that includes a Windows wheel for Python 3.5, so we do
|
||||
# First lxml version that includes a Windows wheel for Python 3.6, so we do
|
||||
# not need to build lxml from sources in a CI Windows job:
|
||||
lxml==3.8.0
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue