From 02eed9f496fe2432a8b443647abab20f094589a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Mon, 1 Jan 2018 15:59:38 +0000 Subject: [PATCH 01/23] Fix link in news.rst --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 36ead3aba..1629510b2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -98,7 +98,7 @@ Docs - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) - Spelling mistake and typos - (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`) + (:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`) - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) - Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) From 037c6b21834ff2ea77cd40acfd5cdb8b9cbd5a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Mon, 1 Jan 2018 16:03:55 +0000 Subject: [PATCH 02/23] Fix typo in comment --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8e9727093..038db7b47 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -379,7 +379,7 @@ class ScrapyAgent(object): {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): - # Abort connection inmediately. + # Abort connection immediately. txresponse._transport._producer.abortConnection() d = defer.Deferred(_cancel) From f1a9d81c887acf6b229155b40f03d0e2131fd1f2 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Fri, 26 Jan 2018 01:42:17 +0530 Subject: [PATCH 03/23] Changed some documentations (#3089) DOC typo fix in defer_fail docstring --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index aa6dcffda..bcf209511 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -11,7 +11,7 @@ def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() From 117e4657a3919ad07b5b2894cdfeda629cbce74f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jan 2018 02:11:49 +0500 Subject: [PATCH 04/23] TST fix tests to account for changes in w3lib 1.19 --- tests/test_http_response.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index a36ec3af6..b228344b5 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -272,7 +272,10 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.text, u'WORD\ufffd\ufffd') + self.assertIn(r6.text, { + u'WORD\ufffd\ufffd', # w3lib < 1.19.0 + u'WORD\ufffd', # w3lib >= 1.19.0 + }) def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, From bfcba6252bdd43653fdb6078c5fa4a44f244be8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Sat, 27 Jan 2018 21:24:15 +0000 Subject: [PATCH 05/23] Fix OS signal names --- scrapy/utils/ossignal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index df4eee5ec..f87d5a803 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,17 +1,18 @@ from __future__ import absolute_import +import signal from twisted.internet import reactor -import signal signal_names = {} for signame in dir(signal): - if signame.startswith("SIG"): + if signame.startswith('SIG') and not signame.startswith('SIG_'): signum = getattr(signal, signame) if isinstance(signum, int): signal_names[signum] = signame + def install_shutdown_handlers(function, override_sigint=True): """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the @@ -24,5 +25,5 @@ def install_shutdown_handlers(function, override_sigint=True): override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows - if hasattr(signal, "SIGBREAK"): + if hasattr(signal, 'SIGBREAK'): signal.signal(signal.SIGBREAK, function) From 72a80d97efa4d4436e16fb25ba76452d547d7b8c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 13 Feb 2018 19:47:41 +0500 Subject: [PATCH 06/23] fix docs building with recent sphinx: don't use deprecated sphinx options and imports --- docs/_ext/scrapydocs.py | 6 +++++- docs/conf.py | 4 ---- docs/requirements.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 83b0d2cc6..192123473 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,6 +1,6 @@ from docutils.parsers.rst.roles import set_classes from docutils import nodes -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode from operator import itemgetter @@ -110,24 +110,28 @@ def setup(app): app.connect('doctree-read', collect_scrapy_settings_refs) app.connect('doctree-resolved', replace_settingslist_nodes) + def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/blob/master/' + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] + def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/issues/' + text set_classes(options) node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options) return [node], [] + def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/commit/' + text set_classes(options) node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) return [node], [] + def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'http://hg.scrapy.org/scrapy/changeset/' + text set_classes(options) diff --git a/docs/conf.py b/docs/conf.py index 007dc2788..594740f39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,10 +144,6 @@ html_static_path = ['_static'] # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -html_use_smartypants = True - # Custom sidebar templates, maps document names to template names. #html_sidebars = {} diff --git a/docs/requirements.txt b/docs/requirements.txt index d3dcb97be..8e7611d21 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.3 +Sphinx>=1.6 sphinx_rtd_theme \ No newline at end of file From 64d2da6cf351d42be0ab444790f2a8ed59de7c9f Mon Sep 17 00:00:00 2001 From: Anjali Jain Date: Thu, 15 Feb 2018 23:27:40 +0530 Subject: [PATCH 07/23] Updated contributing.rst Rectified grammatical errors --- docs/contributing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9a02634cb..44068baa9 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,7 +6,7 @@ Contributing to Scrapy .. important:: - Double check you are reading the most recent version of this document at + Double check that you are reading the most recent version of this document at https://doc.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: @@ -18,7 +18,7 @@ There are many ways to contribute to Scrapy. Here are some of them: * Report bugs and request features in the `issue tracker`_, trying to follow the guidelines detailed in `Reporting bugs`_ below. -* Submit patches for new functionality and/or bug fixes. Please read +* Submit patches for new functionalities and/or bug fixes. Please read :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. @@ -80,8 +80,8 @@ guidelines when reporting a new bug. Writing patches =============== -The better written a patch is, the higher chance that it'll get accepted and -the sooner that will be merged. +The better written a patch is, higher is the chance that it'll get accepted and +sooner it will be merged. Well-written patches should: From 2ee03e3d72a7090ddf25f6db928c465962affdaf Mon Sep 17 00:00:00 2001 From: Anjali Jain Date: Fri, 16 Feb 2018 23:33:10 +0530 Subject: [PATCH 08/23] further edited --- docs/contributing.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 44068baa9..f4f9e393f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -80,8 +80,7 @@ guidelines when reporting a new bug. Writing patches =============== -The better written a patch is, higher is the chance that it'll get accepted and -sooner it will be merged. +The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: From 780c98b04c0511a974a878c0964aa7cb2f12f80c Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 12 Mar 2018 22:25:19 +0800 Subject: [PATCH 09/23] catch CertificateError in tls verification --- scrapy/core/downloader/tls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 498e3d60f..e1c4f4908 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,6 +40,7 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) + from service_identity.exceptions import CertificateError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -65,7 +66,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except VerificationError as e: + except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) From 90170af46212e28e4115e66feb7b5ca300fbbe91 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 08:59:03 +0800 Subject: [PATCH 10/23] add a test case --- tests/keys/localhost-ip.gen.README | 21 ++++++++++++ tests/keys/localhost.crt | 36 ++++++++++----------- tests/keys/localhost.ip.crt | 20 ++++++++++++ tests/keys/localhost.ip.key | 28 ++++++++++++++++ tests/keys/localhost.key | 52 +++++++++++++++--------------- tests/test_downloader_handlers.py | 9 ++++++ 6 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 tests/keys/localhost-ip.gen.README create mode 100644 tests/keys/localhost.ip.crt create mode 100644 tests/keys/localhost.ip.key diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README new file mode 100644 index 000000000..8e94e1217 --- /dev/null +++ b/tests/keys/localhost-ip.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1 +Email Address []:. + diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 13c5b5bd6..48d7bd9a3 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x -NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K -o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y -njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY -ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ -8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 -gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O -BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j -xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES -9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i -jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ -TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own -tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM -am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U -Mjybr/hQK3mnVdE= +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.crt b/tests/keys/localhost.ip.crt new file mode 100644 index 000000000..48d7bd9a3 --- /dev/null +++ b/tests/keys/localhost.ip.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.key b/tests/keys/localhost.ip.key new file mode 100644 index 000000000..1e12c1255 --- /dev/null +++ b/tests/keys/localhost.ip.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq +-----END PRIVATE KEY----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index da975e6d3..1e12c1255 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 -ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 -JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz -vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 -RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY -b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ -XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 -gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk -i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu -Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss -ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 -hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD -7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa -EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf -uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG -HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR -UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO -j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri -fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ -6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI -xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g -DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f -npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM -8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 -J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj -Z+pMCFLbwAbFZ9Qb/NFlUv0= +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq -----END PRIVATE KEY----- diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bd2c86292..ceb03f945 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -505,6 +505,15 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' +class Https11InvalidDNSPattern(Https11TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + super(Https11InvalidDNSPattern, self).setUp() + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" From a69b8529b99b5340803b4e2df479f8eb308868b0 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 13:05:37 +0800 Subject: [PATCH 11/23] revert wrong changes --- tests/keys/localhost.crt | 36 ++++++++++++++-------------- tests/keys/localhost.key | 52 ++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 48d7bd9a3..13c5b5bd6 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x -ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL -JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev -y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV -ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR -v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 -JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O -BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ -dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr -idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP -8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X -kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ -FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb -smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO -zh6AcsnjkLumOkM= +MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x +NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K +o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y +njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY +ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ +8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 +gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O +BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j +xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES +9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i +jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ +TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own +tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM +am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U +Mjybr/hQK3mnVdE= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index 1e12c1255..da975e6d3 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo -+g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC -T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb -rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR -3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi -JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj -uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES -FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc -BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud -JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD -bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT -ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx -KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N -yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA -hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ -32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie -PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY -r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 -X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz -ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ -Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ -g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 -WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ -inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u -BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz -m1Q+9xJdtRxlYc1YGTK1YGCq +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 +ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 +JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz +vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 +RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY +b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ +XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 +gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk +i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu +Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss +ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 +hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD +7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa +EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf +uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG +HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR +UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO +j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri +fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ +6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI +xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g +DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f +npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM +8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 +J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj +Z+pMCFLbwAbFZ9Qb/NFlUv0= -----END PRIVATE KEY----- From 1eb146d0df1ec47d05405c40af453b5988cea3e7 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 19:14:52 +0800 Subject: [PATCH 12/23] fix tests on jessie --- scrapy/core/downloader/tls.py | 11 +++++++++-- tests/test_downloader_handlers.py | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e1c4f4908..c97c6a9a9 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,7 +40,14 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) - from service_identity.exceptions import CertificateError + try: + # XXX: this import would fail on Debian jessie with system installed + # service_identity library, due to lack of cryptography.x509 dependency + # See https://github.com/pyca/service_identity/issues/21 + from service_identity.exceptions import CertificateError + verification_errors = (CertificateError, VerificationError) + except ImportError: + verification_errors = VerificationError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -66,7 +73,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except (CertificateError, VerificationError) as e: + except verification_errors as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ceb03f945..b34faa7e7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -512,6 +512,10 @@ class Https11InvalidDNSPattern(Https11TestCase): certfile = 'keys/localhost.ip.crt' def setUp(self): + try: + from service_identity.exceptions import CertificateError + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") super(Https11InvalidDNSPattern, self).setUp() From 572d43fcd6bd67a43192f205e81829a4ef9d3ff6 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 14 Mar 2018 09:27:59 +0800 Subject: [PATCH 13/23] update docstring of ScrapyClientTLSOptions --- scrapy/core/downloader/tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index c97c6a9a9..df8051182 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -63,8 +63,9 @@ if twisted_version >= (14, 0, 0): (for genuinely invalid certificates or bugs in verification code). Same as Twisted's private _sslverify.ClientTLSOptions, - except that VerificationError and ValueError exceptions are caught, - so that the connection is not closed, only logging warnings. + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. """ def _identityVerifyingInfoCallback(self, connection, where, ret): From 7f1918163c030212e9aa46b1c4c32b98354c5f19 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Tue, 27 Mar 2018 16:21:07 -0400 Subject: [PATCH 14/23] Doc: update wording for COOKIES_ENABLED --- docs/topics/downloader-middleware.rst | 20 +++++++++----------- docs/topics/request-response.rst | 4 +++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 863620900..dfe4c13b4 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -158,13 +158,13 @@ more of the following methods: :type spider: :class:`~scrapy.spiders.Spider` object .. method:: from_crawler(cls, crawler) - + If present, this classmethod is called to create a middleware instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the middleware. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for middleware to access them and hook its functionality into Scrapy. - + :param crawler: crawler that uses this middleware :type crawler: :class:`~scrapy.crawler.Crawler` object @@ -237,16 +237,14 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. -Notice that if the :class:`~scrapy.http.Request` -has ``meta['dont_merge_cookies']`` evaluated to ``True``. -despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be -sent to web servers and received cookies in -:class:`~scrapy.http.Response` will **not** be merged with the existing -cookies. - -For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request` +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. .. setting:: COOKIES_DEBUG diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 121abe6b5..e29914dbf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -80,6 +80,8 @@ Request objects attributes of the cookie. This is only useful if the cookies are saved for later requests. + .. reqmeta:: dont_merge_cookies + When some site returns cookies (in a response) those are stored in the cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. However, if, for some @@ -294,7 +296,7 @@ Those are: * :reqmeta:`dont_retry` * :reqmeta:`handle_httpstatus_list` * :reqmeta:`handle_httpstatus_all` -* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor) +* :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` * :reqmeta:`redirect_urls` From 6630622652813c919fb31d4e4cbb15f8ba831529 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Thu, 19 Apr 2018 13:35:46 +0800 Subject: [PATCH 15/23] improve document about functions as processors --- docs/topics/loaders.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 0849090b4..a895b535c 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. +If you want to use a plain function as a processor, make sure it receives +``self`` as the first argument:: + + def lowercase_processor(self, values): + for v in values: + yield v.lower() + + class MyItemLoader(ItemLoader): + name_in = lowercase_processor + +This is because whenever a function is assigned as a class variable, it becomes +a method and would be passed the instance as the the first argument when being +called. See `this answer on stackoverflow`_ for more details. + The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -143,6 +157,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. +.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 Declaring Item Loaders ====================== From 3bc110a9c6ddfdd0b863ae648ed04a5651fe06b2 Mon Sep 17 00:00:00 2001 From: Kevin Tewouda Date: Wed, 30 May 2018 06:33:18 +0200 Subject: [PATCH 16/23] Update spiders.rst I changed URLs to :class:`~scrapy.http.Request` in start_urls explanation of the default spider --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c2c271245..697732b47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -88,7 +88,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data + listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings From 0da43609c3d8d27827ab0e7e9d640f2bff674fca Mon Sep 17 00:00:00 2001 From: Chris Slothouber Date: Fri, 1 Jun 2018 09:25:34 -0400 Subject: [PATCH 17/23] Minor edits to contributing.rst Corrected minor grammatical issues and increased clarity of instructions. --- docs/contributing.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f4f9e393f..6615840f7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -48,9 +48,9 @@ guidelines when reporting a new bug. `Stack Overflow `__ (use "scrapy" tag). -* check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report, but check the ticket history and comments. If you - have additional useful information, please leave a comment, or consider +* check the `open issues`_ to see if the issue has already been reported. If it + has, don't dismiss the report, but check the ticket history and comments. If + you have additional useful information, please leave a comment, or consider :ref:`sending a pull request ` with a fix. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has @@ -122,7 +122,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first. Sometimes there is an existing pull request for the problem you'd like to solve, which is stalled for some reason. Often the pull request is in a right direction, but changes are requested by Scrapy maintainers, and the -original pull request author haven't had time to address them. +original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is @@ -143,7 +143,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports -removal, etc) in separate commits than functional changes. This will make pull +removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. Coding style @@ -170,7 +170,7 @@ Documentation policies **do** provide a docstring, but make sure sphinx documentation uses autodoc_ extension to pull the docstring. For example, the :meth:`ItemLoader.add_value` method should be either - documented only in the sphinx documentation (not it a docstring), or + documented only in the sphinx documentation (not as a docstring), or it should have a docstring which is pulled to sphinx documentation using autodoc_ extension. From f86841c5c786f55127d97347f5ef6170283bd318 Mon Sep 17 00:00:00 2001 From: grammy-jiang Date: Wed, 4 Apr 2018 05:56:05 -0400 Subject: [PATCH 18/23] fix a mistake in topic spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..1d451af21 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -116,7 +116,7 @@ following methods: method (from other spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Response`, dict or + iterable of :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. If it returns ``None``, Scrapy will continue processing this exception, From 07ae8a037e390bd8798af8c45c6e35c8485e15cc Mon Sep 17 00:00:00 2001 From: Fredrik Bergenlid Date: Fri, 1 Jun 2018 21:38:07 +0200 Subject: [PATCH 19/23] Improve gunzip performance for big files --- scrapy/utils/gz.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 16c9ce539..ec3949651 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -30,25 +30,25 @@ def gunzip(data): This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output = b'' + output_list = [] chunk = b'.' while chunk: try: chunk = read1(f, 8196) - output += chunk + output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output is '' and f.extrabuf + # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output or getattr(f, 'extrabuf', None): + if output_list or getattr(f, 'extrabuf', None): try: - output += f.extrabuf[-f.extrasize:] + output_list.append(f.extrabuf[-f.extrasize:]) finally: break else: raise - return output + return b''.join(output_list) _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search From a188c389505596d62a0a5961dd44853adf2704dd Mon Sep 17 00:00:00 2001 From: mugayoshi Date: Sat, 9 Jun 2018 18:17:11 +0900 Subject: [PATCH 20/23] Update debugging memory leaks section in the docs Add Python3 tools description. --- docs/topics/leaks.rst | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 92590c180..af14d14e8 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, you still have another resource: the `Guppy library`_. +If you're using Python3, see :ref:`topics-leaks-muppy`. .. _Guppy library: https://pypi.python.org/pypi/guppy @@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the .. _Guppy documentation: http://guppy-pe.sourceforge.net/ +.. _topics-leaks-muppy: + +Debugging memory leaks with muppy +================================= +If you're using Python 3, you can use muppy from `Pympler`_. + +.. _Pympler: https://pypi.org/project/Pympler/ + +If you use ``pip``, you can install muppy with the following command:: + + pip install Pympler + +Here's an example to view all Python objects available in +the heap using muppy:: + + >>> from pympler import muppy + >>> all_objects = muppy.get_objects() + >>> len(all_objects) + 28667 + >>> from pympler import summary + >>> suml = summary.summarize(all_objects) + >>> summary.print_(suml) + types | # objects | total size + ==================================== | =========== | ============ + Date: Tue, 19 Jun 2018 10:51:55 +0200 Subject: [PATCH 21/23] fix typo extractred --> extracted --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 1629510b2..1b8d121a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -156,7 +156,7 @@ attributes with ``FormRequest``. **Please also note that link extractors do not canonicalize URLs by default anymore.** This was puzzling users every now and then, and it's not what -browsers do in fact, so we removed that extra transformation on extractred +browsers do in fact, so we removed that extra transformation on extracted links. For those of you wanting more control on the ``Referer:`` header that Scrapy From 8387b383184e49c1f58fa2c5d620b1bf369a24fa Mon Sep 17 00:00:00 2001 From: Renne Rocha Date: Wed, 13 Jun 2018 18:11:43 -0300 Subject: [PATCH 22/23] Include Python version indication to each required library used in S3 storage --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 135d05c93..b64dbfbfd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ or `boto`_ + * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only) The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: From 45d2325ef7abac4318275a5c7f0d7a5f906a1abb Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 17 May 2018 08:53:42 +0000 Subject: [PATCH 23/23] blacklist twisted version with regression in constraints file --- tests/constraints.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/constraints.txt diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..3bc30de15 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1 @@ +Twisted!=18.4.0 diff --git a/tox.ini b/tox.ini index 60ff8c15e..c2fa9af28 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27 [testenv] deps = + -ctests/constraints.txt -rrequirements.txt # Extras botocore