mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' of https://github.com/scrapy/scrapy into h2-client-protocol
This commit is contained in:
Aditya
commit
d17417b03a
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.1.0
|
||||
current_version = 2.2.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
|
|||
removal, etc) in separate commits from functional changes. This will make pull
|
||||
requests easier to review and more likely to get merged.
|
||||
|
||||
|
||||
.. _coding-style:
|
||||
|
||||
Coding style
|
||||
============
|
||||
|
||||
|
|
@ -163,7 +166,7 @@ Scrapy:
|
|||
|
||||
* Unless otherwise specified, follow :pep:`8`.
|
||||
|
||||
* It's OK to use lines longer than 80 chars if it improves the code
|
||||
* It's OK to use lines longer than 79 chars if it improves the code
|
||||
readability.
|
||||
|
||||
* Don't put your name in the code you contribute; git provides enough
|
||||
|
|
|
|||
195
docs/news.rst
195
docs/news.rst
|
|
@ -3,6 +3,201 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.2.0:
|
||||
|
||||
Scrapy 2.2.0 (2020-06-24)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* Python 3.5.2+ is required now
|
||||
* :ref:`dataclass objects <dataclass-items>` and
|
||||
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
|
||||
<item-types>`
|
||||
* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
|
||||
* New :signal:`bytes_received` signal that allows canceling response download
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to
|
||||
run with a Python version lower than 3.5.2, which introduced
|
||||
:class:`typing.Type` (:issue:`4615`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* :meth:`TextResponse.body_as_unicode
|
||||
<scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use
|
||||
:attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead
|
||||
(:issue:`4546`, :issue:`4555`, :issue:`4579`)
|
||||
|
||||
* :class:`scrapy.item.BaseItem` is now deprecated, use
|
||||
:class:`scrapy.item.Item` instead (:issue:`4534`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* :ref:`dataclass objects <dataclass-items>` and
|
||||
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
|
||||
<item-types>`, and a new itemadapter_ library makes it easy to
|
||||
write code that :ref:`supports any item type <supporting-item-types>`
|
||||
(:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`)
|
||||
|
||||
* A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
|
||||
allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`,
|
||||
:issue:`4574`)
|
||||
|
||||
* A new :signal:`bytes_received` signal allows monitoring response download
|
||||
progress and :ref:`stopping downloads <topics-stop-response-download>`
|
||||
(:issue:`4205`, :issue:`4559`)
|
||||
|
||||
* The dictionaries in the result list of a :ref:`media pipeline
|
||||
<topics-media-pipeline>` now include a new key, ``status``, which indicates
|
||||
if the file was downloaded or, if the file was not downloaded, why it was
|
||||
not downloaded; see :meth:`FilesPipeline.get_media_requests
|
||||
<scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more
|
||||
information (:issue:`2893`, :issue:`4486`)
|
||||
|
||||
* When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for
|
||||
a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if
|
||||
the configured credentials do not grant the required permissions
|
||||
(:issue:`4346`, :issue:`4508`)
|
||||
|
||||
* :ref:`Link extractors <topics-link-extractors>` are now serializable,
|
||||
as long as you do not use :ref:`lambdas <lambda>` for parameters; for
|
||||
example, you can now pass link extractors in :attr:`Request.cb_kwargs
|
||||
<scrapy.http.Request.cb_kwargs>` or
|
||||
:attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting
|
||||
scheduled requests <topics-jobs>` (:issue:`4554`)
|
||||
|
||||
* Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses
|
||||
from protocol 2 to protocol 4, improving serialization capabilities and
|
||||
performance (:issue:`4135`, :issue:`4541`)
|
||||
|
||||
* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError`
|
||||
exception if the resulting instance is ``None`` (:issue:`4528`,
|
||||
:issue:`4532`)
|
||||
|
||||
.. _itemadapter: https://github.com/scrapy/itemadapter
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
|
||||
discards cookies defined in :attr:`Request.headers
|
||||
<scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`)
|
||||
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
|
||||
re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter
|
||||
of the ``__init__`` method of :class:`~scrapy.http.Request`
|
||||
(:issue:`2400`, :issue:`3575`)
|
||||
|
||||
* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is
|
||||
``False`` and the crawl yields no items, Scrapy no longer stops feed
|
||||
exports after the first URI (:issue:`4621`, :issue:`4626`)
|
||||
|
||||
* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine
|
||||
syntax <topics/coroutines>` no longer need to return an iterable, and may
|
||||
instead return a :class:`~scrapy.http.Request` object, an
|
||||
:ref:`item <topics-items>`, or ``None`` (:issue:`4609`)
|
||||
|
||||
* The :command:`startproject` command now ensures that the generated project
|
||||
folders and files have the right permissions (:issue:`4604`)
|
||||
|
||||
* Fix a :exc:`KeyError` exception being sometimes raised from
|
||||
:class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`,
|
||||
:issue:`4599`)
|
||||
|
||||
* When :setting:`FEEDS` defines multiple URIs, log messages about items being
|
||||
stored now contain information from the corresponding feed, instead of
|
||||
always containing information about only one of the feeds (:issue:`4619`,
|
||||
:issue:`4629`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Added a new section about :ref:`accessing cb_kwargs from errbacks
|
||||
<errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`)
|
||||
|
||||
* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`,
|
||||
:issue:`4562`)
|
||||
|
||||
* Removed from :doc:`topics/coroutines` the warning about the API being
|
||||
experimental (:issue:`4511`, :issue:`4513`)
|
||||
|
||||
* Removed references to unsupported versions of :doc:`Twisted
|
||||
<twisted:index>` (:issue:`4533`)
|
||||
|
||||
* Updated the description of the :ref:`screenshot pipeline example
|
||||
<ScreenshotPipeline>`, which now uses :doc:`coroutine syntax
|
||||
<topics/coroutines>` instead of returning a
|
||||
:class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`)
|
||||
|
||||
* Removed a misleading import line from the
|
||||
:func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`,
|
||||
:issue:`4587`)
|
||||
|
||||
* The display-on-hover behavior of internal documentation references now also
|
||||
covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta
|
||||
<scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and
|
||||
:ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`)
|
||||
|
||||
* It is again possible to download the documentation for offline reading
|
||||
(:issue:`4578`, :issue:`4585`)
|
||||
|
||||
* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function
|
||||
and method signatures (:issue:`4592`, :issue:`4596`)
|
||||
|
||||
.. _chompjs: https://github.com/Nykakin/chompjs
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Adjusted the code base further to our :ref:`style guidelines
|
||||
<coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`,
|
||||
:issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`,
|
||||
:issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`,
|
||||
:issue:`4572`)
|
||||
|
||||
* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`,
|
||||
:issue:`4568`)
|
||||
|
||||
* Improved code sharing between the :command:`crawl` and :command:`runspider`
|
||||
commands (:issue:`4548`, :issue:`4552`)
|
||||
|
||||
* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)``
|
||||
(:issue:`4635`)
|
||||
|
||||
* You may now run the :mod:`asyncio` tests with Tox on any Python version
|
||||
(:issue:`4521`)
|
||||
|
||||
* Updated test requirements to reflect an incompatibility with pytest 5.4 and
|
||||
5.4.1 (:issue:`4588`)
|
||||
|
||||
* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for
|
||||
scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`)
|
||||
|
||||
* Configured Travis CI to also run the tests with Python 3.5.2
|
||||
(:issue:`4518`, :issue:`4615`)
|
||||
|
||||
* Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI
|
||||
(:issue:`3727`)
|
||||
|
||||
* Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`)
|
||||
|
||||
* Made use of set literals in tests (:issue:`4573`)
|
||||
|
||||
* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`,
|
||||
:issue:`4522`, :issue:`4537`)
|
||||
|
||||
|
||||
.. _release-2.1.0:
|
||||
|
||||
Scrapy 2.1.0 (2020-04-24)
|
||||
|
|
|
|||
|
|
@ -201,6 +201,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
|
|||
.. _s3.scality: https://s3.scality.com/
|
||||
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
||||
|
||||
.. _media-pipeline-gcs:
|
||||
|
||||
Google Cloud Storage
|
||||
---------------------
|
||||
|
||||
|
|
@ -475,7 +478,11 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
* ``checksum`` - a `MD5 hash`_ of the image contents
|
||||
|
||||
* ``status`` - the file status indication. It can be one of the following:
|
||||
* ``status`` - the file status indication.
|
||||
|
||||
.. versionadded:: 2.2
|
||||
|
||||
It can be one of the following:
|
||||
|
||||
* ``downloaded`` - file was downloaded.
|
||||
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ Request objects
|
|||
|
||||
In case of a failure to process the request, this dict can be accessed as
|
||||
``failure.request.cb_kwargs`` in the request's errback. For more information,
|
||||
see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
|
||||
see :ref:`errback-cb_kwargs`.
|
||||
|
||||
.. method:: Request.copy()
|
||||
|
||||
|
|
@ -316,7 +316,7 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
|
||||
.. _errback-cb_kwargs:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
----------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.1.0
|
||||
2.2.0
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ class TextResponse(Response):
|
|||
|
||||
def json(self):
|
||||
"""
|
||||
.. versionadded:: 2.2
|
||||
|
||||
Deserialize a JSON document to a Python object.
|
||||
"""
|
||||
if self._cached_decoded_json is _NONE:
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser):
|
|||
curl_parser = CurlParser()
|
||||
curl_parser.add_argument('url')
|
||||
curl_parser.add_argument('-H', '--header', dest='headers', action='append')
|
||||
curl_parser.add_argument('-X', '--request', dest='method', default='get')
|
||||
curl_parser.add_argument('-d', '--data', dest='data')
|
||||
curl_parser.add_argument('-X', '--request', dest='method')
|
||||
curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
|
||||
curl_parser.add_argument('-u', '--user', dest='auth')
|
||||
|
||||
|
||||
|
|
@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
|
|||
if not parsed_url.scheme:
|
||||
url = 'http://' + url
|
||||
|
||||
result = {'method': parsed_args.method.upper(), 'url': url}
|
||||
method = parsed_args.method or 'GET'
|
||||
|
||||
result = {'method': method.upper(), 'url': url}
|
||||
|
||||
headers = []
|
||||
cookies = {}
|
||||
|
|
@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
|
|||
result['cookies'] = cookies
|
||||
if parsed_args.data:
|
||||
result['body'] = parsed_args.data
|
||||
if not parsed_args.method:
|
||||
# if the "data" is specified but the "method" is not specified,
|
||||
# the default method is 'POST'
|
||||
result['method'] = 'POST'
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -138,8 +138,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
|
||||
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
||||
|
||||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
||||
extension has not been implemented correctly).
|
||||
.. versionchanged:: 2.2
|
||||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
||||
extension has not been implemented correctly).
|
||||
"""
|
||||
if settings is None:
|
||||
if crawler is None:
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
|
||||
BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
|
||||
NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
|
||||
VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
|
||||
BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
|
||||
o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
|
||||
njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
|
||||
ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
|
||||
8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
|
||||
gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
|
||||
BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
|
||||
xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
|
||||
9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
|
||||
jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
|
||||
TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
|
||||
tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
|
||||
am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
|
||||
Mjybr/hQK3mnVdE=
|
||||
MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL
|
||||
BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j
|
||||
YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE
|
||||
BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN
|
||||
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK
|
||||
2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe
|
||||
pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH
|
||||
wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7
|
||||
56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN
|
||||
cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB
|
||||
o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU
|
||||
VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
|
||||
AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ
|
||||
VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX
|
||||
FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf
|
||||
2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE
|
||||
W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb
|
||||
dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA==
|
||||
-----END CERTIFICATE-----
|
||||
|
|
|
|||
|
|
@ -1,28 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
|
||||
ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
|
||||
JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
|
||||
vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
|
||||
RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
|
||||
b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
|
||||
XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
|
||||
gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
|
||||
i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
|
||||
Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
|
||||
ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
|
||||
hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
|
||||
7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
|
||||
EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
|
||||
uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
|
||||
HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
|
||||
UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
|
||||
j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
|
||||
fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
|
||||
6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
|
||||
xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
|
||||
DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
|
||||
npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
|
||||
8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
|
||||
J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
|
||||
Z+pMCFLbwAbFZ9Qb/NFlUv0=
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w
|
||||
mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM
|
||||
39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT
|
||||
yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2
|
||||
XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM
|
||||
DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc
|
||||
niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT
|
||||
S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi
|
||||
X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1
|
||||
QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN
|
||||
HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W
|
||||
birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p
|
||||
hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis
|
||||
vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o
|
||||
8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq
|
||||
12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi
|
||||
M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo
|
||||
gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8
|
||||
kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM
|
||||
kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy
|
||||
kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl
|
||||
xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l
|
||||
VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX
|
||||
abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV
|
||||
SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv
|
||||
PWuPk5VX+qF78JhUhPO3nfnu
|
||||
-----END PRIVATE KEY-----
|
||||
|
|
|
|||
|
|
@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
|
|||
|
||||
def __init__(self, start_urls):
|
||||
self.start_urls = start_urls
|
||||
super(StartUrlsSpider, self).__init__(start_urls)
|
||||
super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
|
||||
|
||||
def parse(self, response):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase):
|
|||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_post_data_raw(self):
|
||||
curl_command = (
|
||||
"curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"
|
||||
"bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"
|
||||
"ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"
|
||||
)
|
||||
expected_result = {
|
||||
"method": "POST",
|
||||
"url": "https://www.example.org/",
|
||||
"body": (
|
||||
"excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
|
||||
"2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
|
||||
"Functions=%5B%5D")
|
||||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_explicit_get_with_data(self):
|
||||
curl_command = 'curl httpbin.org/anything -X GET --data asdf'
|
||||
expected_result = {
|
||||
"method": "GET",
|
||||
"url": "http://httpbin.org/anything",
|
||||
"body": "asdf"
|
||||
}
|
||||
self._test_command(curl_command, expected_result)
|
||||
|
||||
def test_patch(self):
|
||||
curl_command = (
|
||||
'curl "https://example.com/api/fake" -u "username:password" -H "Ac'
|
||||
|
|
|
|||
Loading…
Reference in New Issue