Merge remote-tracking branch 'upstream/remove-py2-tests' into remove-six-code

This commit is contained in:
Eugenio Lacuesta 2019-11-03 01:20:08 -03:00
commit cca1959c07
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
7 changed files with 57 additions and 306 deletions

View File

@ -6,8 +6,8 @@ collect_ignore = [
"scrapy/utils/testsite.py",
]
# FIXME: fix or delete these tests
for line in open('tests/py3-ignores.txt'):
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)

View File

@ -1,6 +1,3 @@
tests/test_linkextractors_deprecated.py
tests/test_proxy_connect.py
scrapy/linkextractors/sgml.py
scrapy/linkextractors/regex.py
scrapy/linkextractors/htmlparser.py

View File

@ -1,5 +1,7 @@
# Tests requirements
jmespath
mitmproxy; python_version >= '3.6'
mitmproxy==3.0.4; python_version < '3.6'
pytest
pytest-cov
pytest-twisted

View File

@ -1,5 +1,4 @@
import os
import six
import shutil
import tempfile
from unittest import mock
@ -630,18 +629,16 @@ class Http11MockServerTestCase(unittest.TestCase):
# download_maxsize < 100, hence the CancelledError
self.assertIsInstance(failure.value, defer.CancelledError)
if six.PY2:
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
request = request.replace(url=self.mockserver.url('/xpayload'))
yield crawler.crawl(seed=request)
# download_maxsize = 50 is enough for the gzipped response
failure = crawler.spider.meta.get('failure')
self.assertTrue(failure == None)
reason = crawler.spider.meta['close_reason']
self.assertTrue(reason, 'finished')
else:
# See issue https://twistedmatrix.com/trac/ticket/8175
raise unittest.SkipTest("xpayload only enabled for PY2")
# See issue https://twistedmatrix.com/trac/ticket/8175
raise unittest.SkipTest("xpayload fails on PY3")
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
request = request.replace(url=self.mockserver.url('/xpayload'))
yield crawler.crawl(seed=request)
# download_maxsize = 50 is enough for the gzipped response
failure = crawler.spider.meta.get('failure')
self.assertTrue(failure == None)
reason = crawler.spider.meta['close_reason']
self.assertTrue(reason, 'finished')
class UriResource(resource.Resource):

View File

@ -1,233 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
from scrapy.linkextractors.regex import RegexLinkExtractor
from scrapy.http import HtmlResponse
from scrapy.link import Link
from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
from tests import get_testdata
from tests.test_linkextractors import Base
class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
# XXX: should we move some of these tests to base link extractor tests?
def test_basic(self):
html = """<html><head><title>Page title<title>
<body><p><a href="item/12.html">Item 12</a></p>
<p><a href="/about.html">About us</a></p>
<img src="/logo.png" alt="Company logo (not a link)" />
<p><a href="../othercat.html">Other category</a></p>
<p><a href="/">&gt;&gt;</a></p>
<p><a href="/" /></p>
</body></html>"""
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
Link(url='http://example.org/about.html', text='About us'),
Link(url='http://example.org/othercat.html', text='Other category'),
Link(url='http://example.org/', text='>>'),
Link(url='http://example.org/', text='')])
def test_base_url(self):
html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
<body><p><a href="item/12.html">Item 12</a></p>
</body></html>"""
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
self.assertEqual(lx.extract_links(response),
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
# base url is an absolute path and relative to host
html = """<html><head><title>Page title<title><base href="/" />
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
self.assertEqual(lx.extract_links(response),
[Link(url='https://example.org/item/12.html', text='Item 12')])
# base url has no scheme
html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
self.assertEqual(lx.extract_links(response),
[Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
def test_link_text_wrong_encoding(self):
html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
lx = BaseSgmlLinkExtractor()
self.assertEqual(lx.extract_links(response), [
Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
])
def test_extraction_encoding(self):
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
body = get_testdata('link_extractor', 'linkextractor_latin1.html')
response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
lx = BaseSgmlLinkExtractor()
self.assertEqual(lx.extract_links(response_utf8), [
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
])
self.assertEqual(lx.extract_links(response_noenc), [
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
])
# document encoding does not affect URL path component, only query part
# >>> u'sample_ñ.html'.encode('utf8')
# b'sample_\xc3\xb1.html'
# >>> u"sample_á.html".encode('utf8')
# b'sample_\xc3\xa1.html'
# >>> u"sample_ö.html".encode('utf8')
# b'sample_\xc3\xb6.html'
# >>> u"£32".encode('latin1')
# b'\xa332'
# >>> u"µ".encode('latin1')
# b'\xb5'
self.assertEqual(lx.extract_links(response_latin1), [
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
])
def test_matches(self):
url1 = 'http://lotsofstuff.com/stuff1/index'
url2 = 'http://evenmorestuff.com/uglystuff/index'
lx = BaseSgmlLinkExtractor()
self.assertEqual(lx.matches(url1), True)
self.assertEqual(lx.matches(url2), True)
class HtmlParserLinkExtractorTestCase(unittest.TestCase):
def setUp(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
self.response = HtmlResponse(url='http://example.com/index', body=body)
def test_extraction(self):
# Default arguments
lx = HtmlParserLinkExtractor()
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
def test_link_wrong_href(self):
html = """
<a href="http://example.org/item1.html">Item 1</a>
<a href="http://[example.org/item2.html">Item 2</a>
<a href="http://example.org/item3.html">Item 3</a>
"""
response = HtmlResponse("http://example.org/index.html", body=html)
lx = HtmlParserLinkExtractor()
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
])
class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
extractor_cls = SgmlLinkExtractor
escapes_whitespace = True
def test_deny_extensions(self):
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
response = HtmlResponse("http://example.org/", body=html)
lx = SgmlLinkExtractor(deny_extensions="jpg")
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.org/page.html', text=u'asd'),
])
def test_attrs_sgml(self):
html = """<html><area href="sample1.html"></area>
<a ref="sample2.html">sample text 2</a></html>"""
response = HtmlResponse("http://example.com/index.html", body=html)
lx = SgmlLinkExtractor(attrs="href")
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/sample1.html', text=u''),
])
def test_link_nofollow(self):
html = """
<a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
<a href="about.html">About us</a>
<a href="http://google.com/something" rel="external nofollow">Something</a>
"""
response = HtmlResponse("http://example.org/page.html", body=html)
lx = SgmlLinkExtractor()
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
Link(url='http://google.com/something', text=u'Something', nofollow=True),
])
class RegexLinkExtractorTestCase(unittest.TestCase):
# XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
# not to depend on SgmlLinkExractor. Its speed is also much worse
# than it should be.
def setUp(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
self.response = HtmlResponse(url='http://example.com/index', body=body)
def test_extraction(self):
# Default arguments
lx = RegexLinkExtractor()
self.assertEqual(lx.extract_links(self.response),
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
def test_link_wrong_href(self):
html = """
<a href="http://example.org/item1.html">Item 1</a>
<a href="http://[example.org/item2.html">Item 2</a>
<a href="http://example.org/item3.html">Item 3</a>
"""
response = HtmlResponse("http://example.org/index.html", body=html)
lx = RegexLinkExtractor()
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
])
def test_html_base_href(self):
html = """
<html>
<head>
<base href="http://b.com/">
</head>
<body>
<a href="test.html"></a>
</body>
</html>
"""
response = HtmlResponse("http://a.com/", body=html)
lx = RegexLinkExtractor()
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://b.com/test.html', text=u'', nofollow=False),
])
@unittest.expectedFailure
def test_extraction(self):
# RegexLinkExtractor doesn't parse URLs with leading/trailing
# whitespaces correctly.
super(RegexLinkExtractorTestCase, self).test_extraction()

View File

@ -1,11 +1,11 @@
import json
import os
import time
import re
import sys
from urllib.parse import urlsplit, urlunsplit
from threading import Thread
from subprocess import Popen, PIPE
from libmproxy import controller, proxy
from netlib import http_auth
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
@ -16,23 +16,37 @@ from tests.spiders import SimpleSpider, SingleRequestSpider
from tests.mockserver import MockServer
class HTTPSProxy(controller.Master, Thread):
class MitmProxy:
auth_user = 'scrapy'
auth_pass = 'scrapy'
def __init__(self):
password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
def start(self):
from scrapy.utils.test import get_testenv
script = """
import sys
from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'keys', 'mitmproxy-ca.pem')
server = proxy.ProxyServer(proxy.ProxyConfig(
authenticator = authenticator,
cacert = cert_path),
0)
self.server = server
Thread.__init__(self)
controller.Master.__init__(self, server)
self.proc = Popen([sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
'--certs', cert_path,
'--ssl-insecure',
],
stdout=PIPE, env=get_testenv())
line = self.proc.stdout.readline().decode('utf-8')
host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
return address
def http_address(self):
return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
def stop(self):
self.proc.kill()
self.proc.communicate()
def _wrong_credentials(proxy_url):
@ -40,6 +54,7 @@ def _wrong_credentials(proxy_url):
bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
return urlunsplit(bad_auth_proxy)
class ProxyConnectTestCase(TestCase):
def setUp(self):
@ -47,17 +62,14 @@ class ProxyConnectTestCase(TestCase):
self.mockserver.__enter__()
self._oldenv = os.environ.copy()
self._proxy = HTTPSProxy()
self._proxy.start()
# Wait for the proxy to start.
time.sleep(1.0)
os.environ['https_proxy'] = self._proxy.http_address()
os.environ['http_proxy'] = self._proxy.http_address()
self._proxy = MitmProxy()
proxy_url = self._proxy.start()
os.environ['https_proxy'] = proxy_url
os.environ['http_proxy'] = proxy_url
def tearDown(self):
self.mockserver.__exit__(None, None, None)
self._proxy.shutdown()
self._proxy.stop()
os.environ = self._oldenv
@defer.inlineCallbacks
@ -67,6 +79,7 @@ class ProxyConnectTestCase(TestCase):
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
def test_https_noconnect(self):
proxy = os.environ['https_proxy']
@ -76,6 +89,7 @@ class ProxyConnectTestCase(TestCase):
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
crawler = get_crawler(SimpleSpider)
@ -100,9 +114,10 @@ class ProxyConnectTestCase(TestCase):
with LogCapture() as l:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, l)
echo = json.loads(crawler.spider.meta['responses'][0].body)
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
def test_https_noconnect_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'

View File

@ -2,15 +2,15 @@ import gc
import functools
import operator
import unittest
from itertools import count
import platform
import six
from itertools import count
from scrapy.utils.python import (
memoizemethod_noargs, binary_is_text, equal_attributes,
WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
without_none_values, MutableChain)
__doctests__ = ['scrapy.utils.python']
@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase):
gc.collect()
self.assertFalse(len(wk._weakdict))
@unittest.skipUnless(six.PY2, "deprecated function")
def test_stringify_dict(self):
d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
d2 = stringify_dict(d, keys_only=False)
self.assertEqual(d, d2)
self.assertIsNot(d, d2) # shouldn't modify in place
self.assertFalse(any(isinstance(x, str) for x in d2.keys()))
self.assertFalse(any(isinstance(x, str) for x in d2.values()))
@unittest.skipUnless(six.PY2, "deprecated function")
def test_stringify_dict_tuples(self):
tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
d = dict(tuples)
d2 = stringify_dict(tuples, keys_only=False)
self.assertEqual(d, d2)
self.assertIsNot(d, d2) # shouldn't modify in place
self.assertFalse(any(isinstance(x, str) for x in d2.keys()), d2.keys())
self.assertFalse(any(isinstance(x, str) for x in d2.values()))
@unittest.skipUnless(six.PY2, "deprecated function")
def test_stringify_dict_keys_only(self):
d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
d2 = stringify_dict(d)
self.assertEqual(d, d2)
self.assertIsNot(d, d2) # shouldn't modify in place
self.assertFalse(any(isinstance(x, str) for x in d2.keys()))
def test_get_func_args(self):
def f1(a, b, c):
pass