From 1bc9d35a878a4028b4dd802707c7b513d4c3ef86 Mon Sep 17 00:00:00 2001 From: Matvei Nazaruk Date: Thu, 19 May 2016 22:24:37 +0300 Subject: [PATCH 01/22] Fixed choosing of response class based on body. --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 88c6b9480..348b39e28 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -343,7 +343,7 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url) + respcls = responsetypes.from_args(headers=headers, url=url, body=body) return respcls(url=url, status=status, headers=headers, body=body, flags=flags) From 1aec5200bc81493623f2a4e077b4e80e104e47d5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 8 Jun 2016 16:49:33 +0200 Subject: [PATCH 02/22] Do not fail on canonicalizing URLs with wrong netlocs Fixes #2010 --- scrapy/utils/url.py | 9 ++++++++- tests/test_utils_url.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c80fc6e70..406eb5843 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions): def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'): + # IDNA encoding can fail for too long labels (>63 characters) + # or missing labels (e.g. http://.example.com) + try: + netloc = parts.netloc.encode('idna') + except UnicodeError: + netloc = parts.netloc + return ( to_native_str(parts.scheme), - to_native_str(parts.netloc.encode('idna')), + to_native_str(netloc), # default encoding for path component SHOULD be UTF-8 quote(to_bytes(parts.path, path_encoding), _safe_chars), diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1fc3a3510..b4819874d 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase): # without encoding, already canonicalized URL is canonicalized identically self.assertEqual(canonicalize_url(canonicalized), canonicalized) + def test_canonicalize_url_idna_exceptions(self): + # missing DNS label + self.assertEqual( + canonicalize_url(u"http://.example.com/résumé?q=résumé"), + "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9") + + # DNS label too long + self.assertEqual( + canonicalize_url( + u"http://www.{label}.com/résumé?q=résumé".format( + label=u"example"*11)), + "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format( + label=u"example"*11)) + class AddHttpIfNoScheme(unittest.TestCase): From 989f6b8843c949aa2ce3839a37399fcada442ac7 Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Sun, 12 Jun 2016 01:38:01 +0200 Subject: [PATCH 03/22] Test to show bug with is_gzipped and Content-Type: application/gzip;charset. --- tests/test_utils_gz.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 8fb1e414d..e107615f3 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,7 +1,8 @@ import unittest from os.path import join -from scrapy.utils.gz import gunzip +from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.http import Response, Headers from tests import tests_datadir SAMPLEDIR = join(tests_datadir, 'compressed') @@ -27,3 +28,16 @@ class GunzipTest(unittest.TestCase): with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: text = gunzip(f.read()) assert text.endswith(b'') + + def test_is_gzipped(self): + hdrs = Headers({"Content-Type": "application/x-gzip"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + r2 = Response("http://www.example.com") + self.assertTrue(not is_gzipped(r2)) + hdrs = Headers({"Content-Type": "application/javascript"}) + r3 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(not is_gzipped(r3)) + hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) From db729f5b304212518e3208995f44e66e55af420e Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Sun, 12 Jun 2016 02:26:16 +0200 Subject: [PATCH 04/22] Suggested fix for is_gzipped --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index d035f9fdf..f174950a4 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -54,4 +54,4 @@ def gunzip(data): def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') - return ctype in (b'application/x-gzip', b'application/gzip') + return b'application/x-gzip' in ctype or b'application/gzip' in ctype From 2c98a88a0e584fff64eb70f48b869494eca1d7ae Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Sun, 12 Jun 2016 10:49:34 +0200 Subject: [PATCH 05/22] Separated tests based on case --- tests/test_utils_gz.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index e107615f3..3648d5c43 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -29,15 +29,24 @@ class GunzipTest(unittest.TestCase): text = gunzip(f.read()) assert text.endswith(b'') - def test_is_gzipped(self): + def test_is_gzipped_right(self): hdrs = Headers({"Content-Type": "application/x-gzip"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) - r2 = Response("http://www.example.com") - self.assertTrue(not is_gzipped(r2)) + hdrs = Headers({"Content-Type": "application/gzip"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + + def test_is_gzipped_empty(self): + r1 = Response("http://www.example.com") + self.assertTrue(not is_gzipped(r1)) + + def test_is_gzipped_wrong(self): hdrs = Headers({"Content-Type": "application/javascript"}) - r3 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(not is_gzipped(r3)) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(not is_gzipped(r1)) + + def test_is_gzipped_with_charset(self): hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) From b76b374648bc7736e37574c6e800f0428f014dfa Mon Sep 17 00:00:00 2001 From: Matvei Nazaruk Date: Mon, 13 Jun 2016 22:36:13 +0300 Subject: [PATCH 06/22] Added test for http11 choosing response type without content-type header. --- tests/test_downloader_handlers.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45a806f2e..09a0950e8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -27,6 +27,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request +from scrapy.http.response.text import TextResponse from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes @@ -114,6 +115,16 @@ class ContentLengthHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"content-length")[0] +class EmptyContentTypeHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of request body + without content-type header in response. + """ + def render(self, request): + request.setHeader("content-type", "") + return request.content.read() + + class HttpTestCase(unittest.TestCase): scheme = 'http' @@ -136,6 +147,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) + r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' @@ -243,11 +255,6 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) - d = self.download_request(request, Spider('foo')) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'0') - return d - def test_payload(self): body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) @@ -284,6 +291,20 @@ class Http11TestCase(HttpTestCase): d.addCallback(self.assertEquals, b"0123456789") return d + def test_response_class_choosing_request(self): + """Tests choosing of correct response type + in case of Content-Type is empty but body contains text. + """ + body = b'Some plain text\ndata with tabs\t and null bytes\0' + + def _test_type(response): + self.assertEquals(type(response), TextResponse) + + request = Request(self.getURL('nocontenttype'), body=body) + d = self.download_request(request, Spider('foo')) + d.addCallback(_test_type) + return d + @defer.inlineCallbacks def test_download_with_maxsize(self): request = Request(self.getURL('file')) From 124e218a3b3d8c5c4f924da5eb8399b205349d19 Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Tue, 14 Jun 2016 14:22:18 +0200 Subject: [PATCH 07/22] Added new testcases suggested by @redapple. --- tests/test_utils_gz.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 3648d5c43..a9bd29bae 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -37,14 +37,28 @@ class GunzipTest(unittest.TestCase): r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) + def test_is_gzipped_not_quite(self): + hdrs = Headers({"Content-Type": "application/gzippppp"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertFalse(is_gzipped(r1)) + + def test_is_gzipped_case_insensitive(self): + hdrs = Headers({"Content-Type": "Application/X-Gzip"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + + hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) + r1 = Response("http://www.example.com", headers=hdrs) + self.assertTrue(is_gzipped(r1)) + def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertTrue(not is_gzipped(r1)) + self.assertFalse(is_gzipped(r1)) def test_is_gzipped_wrong(self): hdrs = Headers({"Content-Type": "application/javascript"}) r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(not is_gzipped(r1)) + self.assertFalse(is_gzipped(r1)) def test_is_gzipped_with_charset(self): hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) From 259426ec9995da9a5415de9d851febf788160cf3 Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Tue, 14 Jun 2016 14:39:16 +0200 Subject: [PATCH 08/22] is_gzipped: Changed to regex to check the content-type header. Also suggested by @redapple. --- scrapy/utils/gz.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index f174950a4..f2a9555b1 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,7 +7,7 @@ except ImportError: from gzip import GzipFile import six - +import re # - Python>=3.5 GzipFile's read() has issues returning leftover # uncompressed data when input is corrupted @@ -50,8 +50,9 @@ def gunzip(data): raise return output +_is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I) def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') - return b'application/x-gzip' in ctype or b'application/gzip' in ctype + return not _is_gzipped_re.search(ctype) is None From 36928d897ca44b0a62ebcb1c3fb358cbfd07440f Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Tue, 14 Jun 2016 15:40:20 +0200 Subject: [PATCH 09/22] is_gzipped: improved readability --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index f2a9555b1..cfb652143 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -55,4 +55,4 @@ _is_gzipped_re = re.compile(br'^application/(x-)?gzip\b', re.I) def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') - return not _is_gzipped_re.search(ctype) is None + return _is_gzipped_re.search(ctype) is not None From 23f99e98c4cf97891202847f3384a35b6fc57e7c Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Tue, 14 Jun 2016 21:33:51 +0200 Subject: [PATCH 10/22] is_gzipped: Separated tests again. --- tests/test_utils_gz.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index a9bd29bae..2b47bf8da 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -29,10 +29,12 @@ class GunzipTest(unittest.TestCase): text = gunzip(f.read()) assert text.endswith(b'') - def test_is_gzipped_right(self): + def test_is_x_gzipped_right(self): hdrs = Headers({"Content-Type": "application/x-gzip"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) + + def test_is_gzipped_right(self): hdrs = Headers({"Content-Type": "application/gzip"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) From 73cc066caad75e3dc7aa3e8343d0bc89baab4cae Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 17 Jun 2016 09:21:58 +0200 Subject: [PATCH 11/22] [docs] add note about windows + python3 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index cbcd4d613..ac87b449d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -72,7 +72,7 @@ features are still missing (and some may never be ported). Almost all builtin extensions/middlewares are expected to work. However, we are aware of some limitations in Python 3: -- Scrapy has not been tested on Windows with Python 3 +- Scrapy does not work on Windows with Python 3 - Sending emails is not supported - FTP download handler is not supported - Telnet console is not supported From 07d1605586d36a65538a4e9a8b3432fb9ffc0a97 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 17 Jun 2016 13:28:51 +0200 Subject: [PATCH 12/22] [docs] warnings about windows + python 3 in faq and install --- docs/faq.rst | 2 +- docs/intro/install.rst | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 82e1f3422..35551d3cc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -71,7 +71,7 @@ What Python versions does Scrapy support? Scrapy is supported under Python 2.7 and Python 3.3+. Python 2.6 support was dropped starting at Scrapy 0.20. -Python 3 support was added in Scrapy 1.1. +Python 3 support was added in Scrapy 1.1. Python 3 is not yet supported on Windows. Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 25520b4b9..16b8761c2 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -11,7 +11,7 @@ Installing Scrapy The installation steps assume that you have the following things installed: -* `Python`_ 2.7 +* `Python`_ 2.7 or above 3.3 * `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and installs `setuptools`_ if not installed. Python 2.7.9 and later include @@ -85,6 +85,10 @@ Windows pip install Scrapy +.. note:: + Python 3 is not supported on Windows. Installation of Scrapy on Windows + with Python 3 will fail. + Ubuntu 9.10 or above -------------------- From d9343463cb69d01a43ed12d8931bbcd86d2aae09 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Jun 2016 13:26:32 +0200 Subject: [PATCH 13/22] Add "Host" header in CONNECT requests to HTTPS proxies --- scrapy/core/downloader/handlers/http11.py | 33 +++++++++++++---------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 88c6b9480..d02027dd1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile(b'HTTP/1\.. 200') + _responseMatcher = re.compile(b'HTTP/1\.. (?P\d{3})(?P.{,32})') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): @@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocol = protocol return protocol - def processProxyResponse(self, bytes): + def processProxyResponse(self, rcvd_bytes): """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ self._protocol.dataReceived = self._protocolDataReceived - if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): + respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes) + if respm and int(respm.group('status')) == 200: try: # this sets proper Server Name Indication extension # but is only available for Twisted>=14.0 @@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + if respm: + extra = {'status': int(respm.group('status')), + 'reason': respm.group('reason').strip()} + else: + extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % ( - self._host, self._port))) + TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( + self._host, self._port, extra))) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -151,23 +157,22 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None): +def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True): r""" Return binary content of a CONNECT request. >>> from scrapy.utils.python import to_native_str as s >>> s(tunnel_request_data("example.com", 8080)) - 'CONNECT example.com:8080 HTTP/1.1\r\n\r\n' + 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' >>> s(tunnel_request_data("example.com", 8080, b"123")) - 'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n' + 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' >>> s(tunnel_request_data(b"example.com", "8090")) - 'CONNECT example.com:8090 HTTP/1.1\r\n\r\n' + 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ - tunnel_req = ( - b'CONNECT ' + - to_bytes(host, encoding='ascii') + b':' + - to_bytes(str(port)) + - b' HTTP/1.1\r\n') + host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port)) + tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n' + if host_header: + tunnel_req += b'Host: ' + host_value + b'\r\n' if proxy_auth_header: tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n' tunnel_req += b'\r\n' From 6539277f995bcfc9310ea92dce180cfa11d982c9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Jun 2016 17:14:41 +0200 Subject: [PATCH 14/22] Fix CONNECT request timeout (with an ugly hack) --- tests/test_downloader_handlers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45a806f2e..c63c64d86 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -401,7 +401,13 @@ class UriResource(resource.Resource): return self def render(self, request): - return request.uri + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b'CONNECT': + return request.uri + else: + return b'' class HttpProxyTestCase(unittest.TestCase): From b67440dec01f27e5a2cb5984d809c840b53afcf6 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 4 Jul 2016 16:35:59 +0200 Subject: [PATCH 15/22] docs on Scrapy on Windows + Python 3 --- docs/faq.rst | 5 ++++- docs/intro/install.rst | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 35551d3cc..415331515 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -71,7 +71,10 @@ What Python versions does Scrapy support? Scrapy is supported under Python 2.7 and Python 3.3+. Python 2.6 support was dropped starting at Scrapy 0.20. -Python 3 support was added in Scrapy 1.1. Python 3 is not yet supported on Windows. +Python 3 support was added in Scrapy 1.1. + +.. note:: + Python 3 is not yet supported on Windows. Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 16b8761c2..3364c3b31 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -86,8 +86,8 @@ Windows pip install Scrapy .. note:: - Python 3 is not supported on Windows. Installation of Scrapy on Windows - with Python 3 will fail. + Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support + Python 3 on Windows. Ubuntu 9.10 or above -------------------- From 2a92ffb409b5f58659fce796a5b051a011bba984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Sat, 4 Jun 2016 10:32:29 +0300 Subject: [PATCH 16/22] Encourage descriptive PR titles --- docs/contributing.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4e8330b3c..b0a435ad2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -100,6 +100,11 @@ starting point is to send a pull request on GitHub. It can be simple enough to illustrate your idea, and leave documentation/tests for later, after the idea has been validated and proven useful. Alternatively, you can send an email to `scrapy-users`_ to discuss your idea first. +When writing GitHub pull requests, try to keep titles short but descriptive. +E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" +prefer "Fix hanging when exception occurs in start_requests (#411)" +instead of "Fix for #411". +Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits than functional changes. This will make pull From 49ac7de23162bd92dc0f2ba88967aa53bbc4af2b Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Tue, 5 Jul 2016 15:38:17 -0500 Subject: [PATCH 17/22] prioritize default headers over user agent --- docs/topics/settings.rst | 6 +++--- scrapy/settings/default_settings.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7f49aacdb..c845c59b9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -459,9 +459,9 @@ Default:: 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8f064f81e..e563e56aa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -93,9 +93,9 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, From 4273734744bc8083713b63c3aaff6ef6368c1727 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 6 Jul 2016 18:29:49 +0500 Subject: [PATCH 18/22] TST pin pytest-cov to 2.2.1; upgrade pytest --- tests/requirements-py3.txt | 4 ++-- tests/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2a89763a5..ed189c66c 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,6 @@ -pytest==2.7.3 +pytest==2.9.2 pytest-twisted -pytest-cov +pytest-cov==2.2.1 testfixtures jmespath leveldb diff --git a/tests/requirements.txt b/tests/requirements.txt index 8901fe16b..9d0c3c996 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,9 +2,9 @@ mock mitmproxy==0.10.1 netlib==0.10.1 -pytest==2.7.3 +pytest==2.9.2 pytest-twisted -pytest-cov +pytest-cov==2.2.1 jmespath testfixtures # optional for shell wrapper tests From 15d0c89159851ae9916e49a27cbcb36335b1bc2e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 6 Jul 2016 17:15:21 +0200 Subject: [PATCH 19/22] Cleanup unused argument --- scrapy/core/downloader/handlers/http11.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index d02027dd1..f07397a4f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -157,7 +157,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True): +def tunnel_request_data(host, port, proxy_auth_header=None): r""" Return binary content of a CONNECT request. @@ -171,8 +171,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True): """ host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port)) tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n' - if host_header: - tunnel_req += b'Host: ' + host_value + b'\r\n' + tunnel_req += b'Host: ' + host_value + b'\r\n' if proxy_auth_header: tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n' tunnel_req += b'\r\n' From 1779f5fecacb49d59461799115fb02b6f80e974f Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Sat, 11 Jun 2016 20:44:08 -0300 Subject: [PATCH 20/22] enable genspider command outside projects --- scrapy/commands/genspider.py | 17 +++++++++++------ tests/test_commands.py | 7 +++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 58bdb9156..d5498bb5c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -25,7 +25,7 @@ def sanitize_module_name(module_name): class Command(ScrapyCommand): - requires_project = True + requires_project = False default_settings = {'LOG_ENABLED': False} def syntax(self): @@ -94,14 +94,19 @@ class Command(ScrapyCommand): 'classname': '%sSpider' % ''.join(s.capitalize() \ for s in module.split('_')) } - spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = abspath(dirname(spiders_module.__file__)) + if self.settings.get('NEWSPIDER_MODULE'): + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = abspath(dirname(spiders_module.__file__)) + else: + spiders_module = None + spiders_dir = "." spider_file = "%s.py" % join(spiders_dir, module) shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r in module:" % (name, \ - template_name)) - print(" %s.%s" % (spiders_module.__name__, module)) + print("Created spider %r using template %r " % (name, \ + template_name), end=('' if spiders_module else '\n')) + if spiders_module: + print("in module:\n %s.%s" % (spiders_module.__name__, module)) def _find_template(self, template): template_file = join(self.templates_dir, '%s.tmpl' % template) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e47160d7..cf415a388 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -146,6 +146,13 @@ class GenspiderCommandTest(CommandTest): assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) +class GenspiderStandaloneCommandTest(ProjectTest): + + def test_generate_standalone_spider(self): + self.call('genspider', 'example', 'example.com') + assert exists(join(self.temp_path, 'example.py')) + + class MiscCommandsTest(CommandTest): def test_list(self): From 8987b17730622708a5103b5e4cc024904870ed32 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 4 Jul 2016 18:59:53 -0300 Subject: [PATCH 21/22] remove references to Item classes in templates --- scrapy/templates/spiders/crawl.tmpl | 4 +--- scrapy/templates/spiders/csvfeed.tmpl | 4 +--- scrapy/templates/spiders/xmlfeed.tmpl | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index a179d16ff..154237d9c 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -3,8 +3,6 @@ import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule -from $project_name.items import ${ProjectName}Item - class $classname(CrawlSpider): name = '$name' @@ -16,7 +14,7 @@ class $classname(CrawlSpider): ) def parse_item(self, response): - i = ${ProjectName}Item() + i = {} #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() #i['name'] = response.xpath('//div[@id="name"]').extract() #i['description'] = response.xpath('//div[@id="description"]').extract() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 69c606538..0544e0ae7 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider -from $project_name.items import ${ProjectName}Item - class $classname(CSVFeedSpider): name = '$name' @@ -16,7 +14,7 @@ class $classname(CSVFeedSpider): # return response def parse_row(self, response, row): - i = ${ProjectName}Item() + i = {} #i['url'] = row['url'] #i['name'] = row['name'] #i['description'] = row['description'] diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 9c0910d23..d8ff61f6e 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider -from $project_name.items import ${ProjectName}Item - class $classname(XMLFeedSpider): name = '$name' @@ -12,7 +10,7 @@ class $classname(XMLFeedSpider): itertag = 'item' # change it accordingly def parse_node(self, response, selector): - i = ${ProjectName}Item() + i = {} #i['url'] = selector.select('url').extract() #i['name'] = selector.select('name').extract() #i['description'] = selector.select('description').extract() From 081595a2e49d17abf9c9d4d3763fa239f79a83e6 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Tue, 5 Jul 2016 22:48:18 -0300 Subject: [PATCH 22/22] document new genspider behavior --- docs/topics/commands.rst | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9a40a2c29..d7999900b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -159,6 +159,7 @@ settings). Global commands: * :command:`startproject` +* :command:`genspider` * :command:`settings` * :command:`runspider` * :command:`shell` @@ -173,7 +174,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`genspider` * :command:`bench` .. command:: startproject @@ -197,14 +197,9 @@ genspider --------- * Syntax: ``scrapy genspider [-t template] `` -* Requires project: *yes* +* Requires project: *no* -Create a new spider in the current project. - -This is just a convenience shortcut command for creating spiders based on -pre-defined templates, but certainly not the only way to create spiders. You -can just create the spider source code files yourself, instead of using this -command. +Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -215,22 +210,16 @@ Usage example:: csvfeed xmlfeed - $ scrapy genspider -d basic - import scrapy + $ scrapy genspider example example.com + Created spider 'example' using template 'basic' - class $classname(scrapy.Spider): - name = "$name" - allowed_domains = ["$domain"] - start_urls = ( - 'http://www.$domain/', - ) + $ scrapy genspider -t crawl scrapyorg scrapy.org + Created spider 'scrapyorg' using template 'crawl' - def parse(self, response): - pass - - $ scrapy genspider -t basic example example.com - Created spider 'example' using template 'basic' in module: - mybot.spiders.example +This is just a convenience shortcut command for creating spiders based on +pre-defined templates, but certainly not the only way to create spiders. You +can just create the spider source code files yourself, instead of using this +command. .. command:: crawl