mirror of https://github.com/scrapy/scrapy.git
Daniel Graña
GitHub
commit
dff8349cd9
|
|
@ -1,6 +1,6 @@
|
|||
from docutils.parsers.rst.roles import set_classes
|
||||
from docutils import nodes
|
||||
from sphinx.util.compat import Directive
|
||||
from docutils.parsers.rst import Directive
|
||||
from sphinx.util.nodes import make_refnode
|
||||
from operator import itemgetter
|
||||
|
||||
|
|
@ -110,24 +110,28 @@ def setup(app):
|
|||
app.connect('doctree-read', collect_scrapy_settings_refs)
|
||||
app.connect('doctree-resolved', replace_settingslist_nodes)
|
||||
|
||||
|
||||
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
|
||||
def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/issues/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
|
||||
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/commit/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
|
||||
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
|
||||
set_classes(options)
|
||||
|
|
|
|||
|
|
@ -144,10 +144,6 @@ html_static_path = ['_static']
|
|||
# using the given strftime format.
|
||||
html_last_updated_fmt = '%b %d, %Y'
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
#html_sidebars = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Contributing to Scrapy
|
|||
|
||||
.. important::
|
||||
|
||||
Double check you are reading the most recent version of this document at
|
||||
Double check that you are reading the most recent version of this document at
|
||||
https://doc.scrapy.org/en/master/contributing.html
|
||||
|
||||
There are many ways to contribute to Scrapy. Here are some of them:
|
||||
|
|
@ -18,7 +18,7 @@ There are many ways to contribute to Scrapy. Here are some of them:
|
|||
* Report bugs and request features in the `issue tracker`_, trying to follow
|
||||
the guidelines detailed in `Reporting bugs`_ below.
|
||||
|
||||
* Submit patches for new functionality and/or bug fixes. Please read
|
||||
* Submit patches for new functionalities and/or bug fixes. Please read
|
||||
:ref:`writing-patches` and `Submitting patches`_ below for details on how to
|
||||
write and submit a patch.
|
||||
|
||||
|
|
@ -48,9 +48,9 @@ guidelines when reporting a new bug.
|
|||
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
|
||||
(use "scrapy" tag).
|
||||
|
||||
* check the `open issues`_ to see if it has already been reported. If it has,
|
||||
don't dismiss the report, but check the ticket history and comments. If you
|
||||
have additional useful information, please leave a comment, or consider
|
||||
* check the `open issues`_ to see if the issue has already been reported. If it
|
||||
has, don't dismiss the report, but check the ticket history and comments. If
|
||||
you have additional useful information, please leave a comment, or consider
|
||||
:ref:`sending a pull request <writing-patches>` with a fix.
|
||||
|
||||
* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
|
||||
|
|
@ -80,8 +80,7 @@ guidelines when reporting a new bug.
|
|||
Writing patches
|
||||
===============
|
||||
|
||||
The better written a patch is, the higher chance that it'll get accepted and
|
||||
the sooner that will be merged.
|
||||
The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged.
|
||||
|
||||
Well-written patches should:
|
||||
|
||||
|
|
@ -123,7 +122,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first.
|
|||
Sometimes there is an existing pull request for the problem you'd like to
|
||||
solve, which is stalled for some reason. Often the pull request is in a
|
||||
right direction, but changes are requested by Scrapy maintainers, and the
|
||||
original pull request author haven't had time to address them.
|
||||
original pull request author hasn't had time to address them.
|
||||
In this case consider picking up this pull request: open
|
||||
a new pull request with all commits from the original pull request, as well as
|
||||
additional changes to address the raised issues. Doing so helps a lot; it is
|
||||
|
|
@ -144,7 +143,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through
|
|||
the issue tracker.
|
||||
|
||||
Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
|
||||
removal, etc) in separate commits than functional changes. This will make pull
|
||||
removal, etc) in separate commits from functional changes. This will make pull
|
||||
requests easier to review and more likely to get merged.
|
||||
|
||||
Coding style
|
||||
|
|
@ -171,7 +170,7 @@ Documentation policies
|
|||
**do** provide a docstring, but make sure sphinx documentation uses
|
||||
autodoc_ extension to pull the docstring. For example, the
|
||||
:meth:`ItemLoader.add_value` method should be either
|
||||
documented only in the sphinx documentation (not it a docstring), or
|
||||
documented only in the sphinx documentation (not as a docstring), or
|
||||
it should have a docstring which is pulled to sphinx documentation using
|
||||
autodoc_ extension.
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ Docs
|
|||
- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example
|
||||
(:issue:`2781`)
|
||||
- Spelling mistake and typos
|
||||
(:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`)
|
||||
(:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`)
|
||||
- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`)
|
||||
- Document ``DontCloseSpider`` exception and clarify ``spider_idle``
|
||||
(:issue:`2791`)
|
||||
|
|
@ -156,7 +156,7 @@ attributes with ``FormRequest``.
|
|||
|
||||
**Please also note that link extractors do not canonicalize URLs by default
|
||||
anymore.** This was puzzling users every now and then, and it's not what
|
||||
browsers do in fact, so we removed that extra transformation on extractred
|
||||
browsers do in fact, so we removed that extra transformation on extracted
|
||||
links.
|
||||
|
||||
For those of you wanting more control on the ``Referer:`` header that Scrapy
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
Sphinx>=1.3
|
||||
Sphinx>=1.6
|
||||
sphinx_rtd_theme
|
||||
|
|
@ -158,13 +158,13 @@ more of the following methods:
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
|
||||
If present, this classmethod is called to create a middleware instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the middleware. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for middleware to
|
||||
access them and hook its functionality into Scrapy.
|
||||
|
||||
|
||||
:param crawler: crawler that uses this middleware
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
|
@ -237,16 +237,14 @@ Default: ``True``
|
|||
Whether to enable the cookies middleware. If disabled, no cookies will be sent
|
||||
to web servers.
|
||||
|
||||
Notice that if the :class:`~scrapy.http.Request`
|
||||
has ``meta['dont_merge_cookies']`` evaluated to ``True``.
|
||||
despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be
|
||||
sent to web servers and received cookies in
|
||||
:class:`~scrapy.http.Response` will **not** be merged with the existing
|
||||
cookies.
|
||||
|
||||
For more detailed information see the ``cookies`` parameter in
|
||||
:class:`~scrapy.http.Request`
|
||||
Notice that despite the value of :setting:`COOKIES_ENABLED` setting if
|
||||
``Request.``:reqmeta:`meta['dont_merge_cookies'] <dont_merge_cookies>`
|
||||
evaluates to ``True`` the request cookies will **not** be sent to the
|
||||
web server and received cookies in :class:`~scrapy.http.Response` will
|
||||
**not** be merged with the existing cookies.
|
||||
|
||||
For more detailed information see the ``cookies`` parameter in
|
||||
:class:`~scrapy.http.Request`.
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_.
|
|||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `botocore`_ or `boto`_
|
||||
* Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are
|
|||
other cases where the memory leaks could come from other (more or less obscure)
|
||||
objects. If this is your case, and you can't find your leaks using ``trackref``,
|
||||
you still have another resource: the `Guppy library`_.
|
||||
If you're using Python3, see :ref:`topics-leaks-muppy`.
|
||||
|
||||
.. _Guppy library: https://pypi.python.org/pypi/guppy
|
||||
|
||||
|
|
@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the
|
|||
|
||||
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
|
||||
|
||||
.. _topics-leaks-muppy:
|
||||
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
If you're using Python 3, you can use muppy from `Pympler`_.
|
||||
|
||||
.. _Pympler: https://pypi.org/project/Pympler/
|
||||
|
||||
If you use ``pip``, you can install muppy with the following command::
|
||||
|
||||
pip install Pympler
|
||||
|
||||
Here's an example to view all Python objects available in
|
||||
the heap using muppy::
|
||||
|
||||
>>> from pympler import muppy
|
||||
>>> all_objects = muppy.get_objects()
|
||||
>>> len(all_objects)
|
||||
28667
|
||||
>>> from pympler import summary
|
||||
>>> suml = summary.summarize(all_objects)
|
||||
>>> summary.print_(suml)
|
||||
types | # objects | total size
|
||||
==================================== | =========== | ============
|
||||
<class 'str | 9822 | 1.10 MB
|
||||
<class 'dict | 1658 | 856.62 KB
|
||||
<class 'type | 436 | 443.60 KB
|
||||
<class 'code | 2974 | 419.56 KB
|
||||
<class '_io.BufferedWriter | 2 | 256.34 KB
|
||||
<class 'set | 420 | 159.88 KB
|
||||
<class '_io.BufferedReader | 1 | 128.17 KB
|
||||
<class 'wrapper_descriptor | 1130 | 88.28 KB
|
||||
<class 'tuple | 1304 | 86.57 KB
|
||||
<class 'weakref | 1013 | 79.14 KB
|
||||
<class 'builtin_function_or_method | 958 | 67.36 KB
|
||||
<class 'method_descriptor | 865 | 60.82 KB
|
||||
<class 'abc.ABCMeta | 62 | 59.96 KB
|
||||
<class 'list | 446 | 58.52 KB
|
||||
<class 'int | 1425 | 43.20 KB
|
||||
|
||||
For more info about muppy, refer to the `muppy documentation`_.
|
||||
|
||||
.. _muppy documentation: https://pythonhosted.org/Pympler/muppy.html
|
||||
|
||||
.. _topics-leaks-without-leaks:
|
||||
|
||||
Leaks without leaks
|
||||
|
|
|
|||
|
|
@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator.
|
|||
containing the collected values (for that field). The result of the output
|
||||
processors is the value that will be finally assigned to the item.
|
||||
|
||||
If you want to use a plain function as a processor, make sure it receives
|
||||
``self`` as the first argument::
|
||||
|
||||
def lowercase_processor(self, values):
|
||||
for v in values:
|
||||
yield v.lower()
|
||||
|
||||
class MyItemLoader(ItemLoader):
|
||||
name_in = lowercase_processor
|
||||
|
||||
This is because whenever a function is assigned as a class variable, it becomes
|
||||
a method and would be passed the instance as the the first argument when being
|
||||
called. See `this answer on stackoverflow`_ for more details.
|
||||
|
||||
The other thing you need to keep in mind is that the values returned by input
|
||||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
|
@ -143,6 +157,7 @@ processors to populate the fields.
|
|||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
|
||||
.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
|
||||
|
||||
Declaring Item Loaders
|
||||
======================
|
||||
|
|
|
|||
|
|
@ -80,6 +80,8 @@ Request objects
|
|||
attributes of the cookie. This is only useful if the cookies are saved
|
||||
for later requests.
|
||||
|
||||
.. reqmeta:: dont_merge_cookies
|
||||
|
||||
When some site returns cookies (in a response) those are stored in the
|
||||
cookies for that domain and will be sent again in future requests. That's
|
||||
the typical behaviour of any regular web browser. However, if, for some
|
||||
|
|
@ -294,7 +296,7 @@ Those are:
|
|||
* :reqmeta:`dont_retry`
|
||||
* :reqmeta:`handle_httpstatus_list`
|
||||
* :reqmeta:`handle_httpstatus_all`
|
||||
* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
|
||||
* :reqmeta:`dont_merge_cookies`
|
||||
* :reqmeta:`cookiejar`
|
||||
* :reqmeta:`dont_cache`
|
||||
* :reqmeta:`redirect_urls`
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ following methods:
|
|||
method (from other spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
iterable of :class:`~scrapy.http.Response`, dict or
|
||||
iterable of :class:`~scrapy.http.Request`, dict or
|
||||
:class:`~scrapy.item.Item` objects.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ scrapy.Spider
|
|||
|
||||
A list of URLs where the spider will begin to crawl from, when no
|
||||
particular URLs are specified. So, the first pages downloaded will be those
|
||||
listed here. The subsequent URLs will be generated successively from data
|
||||
listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
|
||||
contained in the start URLs.
|
||||
|
||||
.. attribute:: custom_settings
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ class ScrapyAgent(object):
|
|||
{'size': expected_size, 'warnsize': warnsize, 'request': request})
|
||||
|
||||
def _cancel(_):
|
||||
# Abort connection inmediately.
|
||||
# Abort connection immediately.
|
||||
txresponse._transport._producer.abortConnection()
|
||||
|
||||
d = defer.Deferred(_cancel)
|
||||
|
|
|
|||
|
|
@ -40,6 +40,14 @@ if twisted_version >= (14, 0, 0):
|
|||
from twisted.internet._sslverify import (ClientTLSOptions,
|
||||
verifyHostname,
|
||||
VerificationError)
|
||||
try:
|
||||
# XXX: this import would fail on Debian jessie with system installed
|
||||
# service_identity library, due to lack of cryptography.x509 dependency
|
||||
# See https://github.com/pyca/service_identity/issues/21
|
||||
from service_identity.exceptions import CertificateError
|
||||
verification_errors = (CertificateError, VerificationError)
|
||||
except ImportError:
|
||||
verification_errors = VerificationError
|
||||
|
||||
if twisted_version < (17, 0, 0):
|
||||
from twisted.internet._sslverify import _maybeSetHostNameIndication
|
||||
|
|
@ -55,8 +63,9 @@ if twisted_version >= (14, 0, 0):
|
|||
(for genuinely invalid certificates or bugs in verification code).
|
||||
|
||||
Same as Twisted's private _sslverify.ClientTLSOptions,
|
||||
except that VerificationError and ValueError exceptions are caught,
|
||||
so that the connection is not closed, only logging warnings.
|
||||
except that VerificationError, CertificateError and ValueError
|
||||
exceptions are caught, so that the connection is not closed, only
|
||||
logging warnings.
|
||||
"""
|
||||
|
||||
def _identityVerifyingInfoCallback(self, connection, where, ret):
|
||||
|
|
@ -65,7 +74,7 @@ if twisted_version >= (14, 0, 0):
|
|||
elif where & SSL_CB_HANDSHAKE_DONE:
|
||||
try:
|
||||
verifyHostname(connection, self._hostnameASCII)
|
||||
except VerificationError as e:
|
||||
except verification_errors as e:
|
||||
logger.warning(
|
||||
'Remote certificate is not valid for hostname "{}"; {}'.format(
|
||||
self._hostnameASCII, e))
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ def defer_fail(_failure):
|
|||
"""Same as twisted.internet.defer.fail but delay calling errback until
|
||||
next reactor loop
|
||||
|
||||
It delays by 100ms so reactor has a chance to go trough readers and writers
|
||||
It delays by 100ms so reactor has a chance to go through readers and writers
|
||||
before attending pending delayed calls, so do not set delay to zero.
|
||||
"""
|
||||
d = defer.Deferred()
|
||||
|
|
|
|||
|
|
@ -30,25 +30,25 @@ def gunzip(data):
|
|||
This is resilient to CRC checksum errors.
|
||||
"""
|
||||
f = GzipFile(fileobj=BytesIO(data))
|
||||
output = b''
|
||||
output_list = []
|
||||
chunk = b'.'
|
||||
while chunk:
|
||||
try:
|
||||
chunk = read1(f, 8196)
|
||||
output += chunk
|
||||
output_list.append(chunk)
|
||||
except (IOError, EOFError, struct.error):
|
||||
# complete only if there is some data, otherwise re-raise
|
||||
# see issue 87 about catching struct.error
|
||||
# some pages are quite small so output is '' and f.extrabuf
|
||||
# some pages are quite small so output_list is empty and f.extrabuf
|
||||
# contains the whole page content
|
||||
if output or getattr(f, 'extrabuf', None):
|
||||
if output_list or getattr(f, 'extrabuf', None):
|
||||
try:
|
||||
output += f.extrabuf[-f.extrasize:]
|
||||
output_list.append(f.extrabuf[-f.extrasize:])
|
||||
finally:
|
||||
break
|
||||
else:
|
||||
raise
|
||||
return output
|
||||
return b''.join(output_list)
|
||||
|
||||
_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
|
||||
_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
|
||||
|
|
|
|||
|
|
@ -1,17 +1,18 @@
|
|||
|
||||
from __future__ import absolute_import
|
||||
import signal
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
import signal
|
||||
|
||||
signal_names = {}
|
||||
for signame in dir(signal):
|
||||
if signame.startswith("SIG"):
|
||||
if signame.startswith('SIG') and not signame.startswith('SIG_'):
|
||||
signum = getattr(signal, signame)
|
||||
if isinstance(signum, int):
|
||||
signal_names[signum] = signame
|
||||
|
||||
|
||||
def install_shutdown_handlers(function, override_sigint=True):
|
||||
"""Install the given function as a signal handler for all common shutdown
|
||||
signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
|
||||
|
|
@ -24,5 +25,5 @@ def install_shutdown_handlers(function, override_sigint=True):
|
|||
override_sigint:
|
||||
signal.signal(signal.SIGINT, function)
|
||||
# Catch Ctrl-Break in windows
|
||||
if hasattr(signal, "SIGBREAK"):
|
||||
if hasattr(signal, 'SIGBREAK'):
|
||||
signal.signal(signal.SIGBREAK, function)
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Twisted!=18.4.0
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt
|
||||
Generating a 2048 bit RSA private key
|
||||
...................................................................................................+++
|
||||
.....+++
|
||||
writing new private key to 'localhost.key'
|
||||
-----
|
||||
You are about to be asked to enter information that will be incorporated
|
||||
into your certificate request.
|
||||
What you are about to enter is what is called a Distinguished Name or a DN.
|
||||
There are quite a few fields but you can leave some blank
|
||||
For some fields there will be a default value,
|
||||
If you enter '.', the field will be left blank.
|
||||
-----
|
||||
Country Name (2 letter code) [AU]:IE
|
||||
State or Province Name (full name) [Some-State]:.
|
||||
Locality Name (eg, city) []:.
|
||||
Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy
|
||||
Organizational Unit Name (eg, section) []:.
|
||||
Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1
|
||||
Email Address []:.
|
||||
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
|
||||
BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x
|
||||
ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
|
||||
VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB
|
||||
BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL
|
||||
JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev
|
||||
y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV
|
||||
ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR
|
||||
v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4
|
||||
JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O
|
||||
BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ
|
||||
dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr
|
||||
idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP
|
||||
8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X
|
||||
kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ
|
||||
FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb
|
||||
smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO
|
||||
zh6AcsnjkLumOkM=
|
||||
-----END CERTIFICATE-----
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo
|
||||
+g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC
|
||||
T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb
|
||||
rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR
|
||||
3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi
|
||||
JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj
|
||||
uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES
|
||||
FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc
|
||||
BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud
|
||||
JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD
|
||||
bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT
|
||||
ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx
|
||||
KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N
|
||||
yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA
|
||||
hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+
|
||||
32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie
|
||||
PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY
|
||||
r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2
|
||||
X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz
|
||||
ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ
|
||||
Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ
|
||||
g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5
|
||||
WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+
|
||||
inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u
|
||||
BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz
|
||||
m1Q+9xJdtRxlYc1YGTK1YGCq
|
||||
-----END PRIVATE KEY-----
|
||||
|
|
@ -505,6 +505,19 @@ class Https11InvalidDNSId(Https11TestCase):
|
|||
super(Https11InvalidDNSId, self).setUp()
|
||||
self.host = '127.0.0.1'
|
||||
|
||||
class Https11InvalidDNSPattern(Https11TestCase):
|
||||
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
|
||||
|
||||
keyfile = 'keys/localhost.ip.key'
|
||||
certfile = 'keys/localhost.ip.crt'
|
||||
|
||||
def setUp(self):
|
||||
try:
|
||||
from service_identity.exceptions import CertificateError
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("cryptography lib is too old")
|
||||
super(Https11InvalidDNSPattern, self).setUp()
|
||||
|
||||
|
||||
class Http11MockServerTestCase(unittest.TestCase):
|
||||
"""HTTP 1.1 test case with MockServer"""
|
||||
|
|
|
|||
|
|
@ -272,7 +272,10 @@ class TextResponseTest(BaseResponseTest):
|
|||
headers={"Content-type": ["text/html; charset=utf-8"]},
|
||||
body=b"\xef\xbb\xbfWORD\xe3\xab")
|
||||
self.assertEqual(r6.encoding, 'utf-8')
|
||||
self.assertEqual(r6.text, u'WORD\ufffd\ufffd')
|
||||
self.assertIn(r6.text, {
|
||||
u'WORD\ufffd\ufffd', # w3lib < 1.19.0
|
||||
u'WORD\ufffd', # w3lib >= 1.19.0
|
||||
})
|
||||
|
||||
def test_bom_is_removed_from_body(self):
|
||||
# Inferring encoding from body also cache decoded body as sideeffect,
|
||||
|
|
|
|||
Loading…
Reference in New Issue