From 1ce6662a9d7115348788972afce62a5c45199021 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sat, 24 Nov 2018 20:02:00 +0530 Subject: [PATCH 01/67] Implement Request subclass for json requests --- scrapy/http/__init__.py | 1 + scrapy/http/request/json_request.py | 28 +++++++++++++++ tests/test_http_request.py | 55 ++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 scrapy/http/request/json_request.py diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index f04a9d3e5..4b2f7b33f 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -10,6 +10,7 @@ from scrapy.http.headers import Headers from scrapy.http.request import Request from scrapy.http.request.form import FormRequest from scrapy.http.request.rpc import XmlRpcRequest +from scrapy.http.request.json_request import JSONRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py new file mode 100644 index 000000000..0fdd2ddf1 --- /dev/null +++ b/scrapy/http/request/json_request.py @@ -0,0 +1,28 @@ +""" +This module implements the JSONRequest class which is a more convenient class +(than Request) to generate JSON Requests. + +See documentation in docs/topics/request-response.rst +""" + +import json + +from scrapy.http.request import Request + + +class JSONRequest(Request): + def __init__(self, *args, **kwargs): + if 'method' not in kwargs: + kwargs['method'] = 'POST' + + data = kwargs.pop('data', {}) + kwargs['body'] = json.dumps(data) + super(JSONRequest, self).__init__(*args, **kwargs) + self.headers.setdefault(b'Content-Type', b'application/json') + + def replace(self, *args, **kwargs): + """ Create a new Request with the same attributes except for those + given new values. """ + + kwargs.pop('body', None) + return super(JSONRequest, self).replace(*args, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..3f2e4f521 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -2,6 +2,7 @@ import cgi import unittest import re +import json import six from six.moves import xmlrpc_client as xmlrpclib @@ -9,7 +10,7 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote if six.PY3: from urllib.parse import unquote_to_bytes -from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse +from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str @@ -1147,5 +1148,57 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=(u'pas£',), encoding='latin1') +class JSONRequestTest(RequestTest): + request_class = JSONRequest + default_method = 'POST' + default_headers = {b'Content-Type': [b'application/json']} + + def test_body(self): + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.body, '{}') + + r2 = self.request_class(url="http://www.example.com/", body=b"") + self.assertEqual(r2.body, '{}') + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.body, json.dumps(data)) + + r4 = self.request_class(url="http://www.example.com/", body='body1', data=data) + self.assertEqual(r3.body, json.dumps(data)) + + def test_replace(self): + """Test Request.replace() method""" + r1 = self.request_class("http://www.example.com") + hdrs = Headers(r1.headers) + hdrs[b'key'] = b'value' + r2 = r1.replace(body="New body", headers=hdrs) + + # body will not be replaced + self.assertEqual(r1.body, r2.body) + self.assertEqual(r1.url, r2.url) + self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) + + # Empty attributes (which may fail if not compared properly) + r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) + r4 = r3.replace(url="http://www.example.com/2", meta={}, dont_filter=False) + self.assertEqual(r4.url, "http://www.example.com/2") + self.assertEqual(r4.meta, {}) + assert r4.dont_filter is False + + data1 = { + 'name': 'value1', + } + data2 = { + 'name': 'value2', + } + r5 = self.request_class("http://www.example.com", data=data1) + r6 = r5.replace(url="http://www.example.com/2", data=data2) + self.assertNotEqual(r5.body, r6.body) + self.assertEqual((r5.body, r6.body), (json.dumps(data1), json.dumps(data2))) + + if __name__ == "__main__": unittest.main() From 1b2b8b4bf0c73b4ad143f943584545702d66cbb7 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Tue, 27 Nov 2018 08:57:44 +0530 Subject: [PATCH 02/67] fix tests under py3 --- tests/test_http_request.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 3f2e4f521..a2021bd65 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1155,19 +1155,19 @@ class JSONRequestTest(RequestTest): def test_body(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, '{}') + self.assertEqual(r1.body, b'{}') r2 = self.request_class(url="http://www.example.com/", body=b"") - self.assertEqual(r2.body, '{}') + self.assertEqual(r2.body, b'{}') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.body, json.dumps(data)) + self.assertEqual(r3.body, to_bytes(json.dumps(data))) r4 = self.request_class(url="http://www.example.com/", body='body1', data=data) - self.assertEqual(r3.body, json.dumps(data)) + self.assertEqual(r3.body, to_bytes(json.dumps(data))) def test_replace(self): """Test Request.replace() method""" @@ -1197,7 +1197,7 @@ class JSONRequestTest(RequestTest): r5 = self.request_class("http://www.example.com", data=data1) r6 = r5.replace(url="http://www.example.com/2", data=data2) self.assertNotEqual(r5.body, r6.body) - self.assertEqual((r5.body, r6.body), (json.dumps(data1), json.dumps(data2))) + self.assertEqual((r5.body, r6.body), (to_bytes(json.dumps(data1)), to_bytes(json.dumps(data2)))) if __name__ == "__main__": From cd619c1d4f3810c96af0ff5c5735c1856dfac95a Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sat, 8 Dec 2018 22:10:45 +0530 Subject: [PATCH 03/67] removed overriden replace method --- scrapy/http/request/json_request.py | 20 +++++------ tests/test_http_request.py | 51 ++++++++--------------------- 2 files changed, 21 insertions(+), 50 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 0fdd2ddf1..03a0ab061 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -12,17 +12,13 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): - if 'method' not in kwargs: - kwargs['method'] = 'POST' + data = kwargs.pop('data', None) + if data: + kwargs['body'] = json.dumps(data) + + if 'method' not in kwargs: + kwargs['method'] = 'POST' - data = kwargs.pop('data', {}) - kwargs['body'] = json.dumps(data) super(JSONRequest, self).__init__(*args, **kwargs) - self.headers.setdefault(b'Content-Type', b'application/json') - - def replace(self, *args, **kwargs): - """ Create a new Request with the same attributes except for those - given new values. """ - - kwargs.pop('body', None) - return super(JSONRequest, self).replace(*args, **kwargs) + self.headers.setdefault('Content-Type', 'application/json') + self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a2021bd65..793a583bc 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1150,54 +1150,29 @@ class XmlRpcRequestTest(RequestTest): class JSONRequestTest(RequestTest): request_class = JSONRequest - default_method = 'POST' - default_headers = {b'Content-Type': [b'application/json']} + default_method = 'GET' + default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} - def test_body(self): + def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b'{}') + self.assertEqual(r1.body, b'') + self.assertEqual(r1.method, 'GET') - r2 = self.request_class(url="http://www.example.com/", body=b"") - self.assertEqual(r2.body, b'{}') + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.body, body) + self.assertEqual(r2.method, 'GET') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) + self.assertEqual(r3.method, 'POST') - r4 = self.request_class(url="http://www.example.com/", body='body1', data=data) - self.assertEqual(r3.body, to_bytes(json.dumps(data))) - - def test_replace(self): - """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com") - hdrs = Headers(r1.headers) - hdrs[b'key'] = b'value' - r2 = r1.replace(body="New body", headers=hdrs) - - # body will not be replaced - self.assertEqual(r1.body, r2.body) - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) - - # Empty attributes (which may fail if not compared properly) - r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) - r4 = r3.replace(url="http://www.example.com/2", meta={}, dont_filter=False) - self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.meta, {}) - assert r4.dont_filter is False - - data1 = { - 'name': 'value1', - } - data2 = { - 'name': 'value2', - } - r5 = self.request_class("http://www.example.com", data=data1) - r6 = r5.replace(url="http://www.example.com/2", data=data2) - self.assertNotEqual(r5.body, r6.body) - self.assertEqual((r5.body, r6.body), (to_bytes(json.dumps(data1)), to_bytes(json.dumps(data2)))) + r4 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r4.body, to_bytes(json.dumps(data))) + self.assertEqual(r4.method, 'POST') if __name__ == "__main__": From c347acbff6545c428aa2c965cd03f03db6bae1bf Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sun, 9 Dec 2018 11:27:09 +0530 Subject: [PATCH 04/67] warning if body and data are provided --- scrapy/http/request/json_request.py | 7 ++++++- tests/test_http_request.py | 25 ++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 03a0ab061..3b791eda3 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -6,14 +6,19 @@ See documentation in docs/topics/request-response.rst """ import json +import warnings from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): + body_passed = 'body' in kwargs data = kwargs.pop('data', None) - if data: + if body_passed and data: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data: kwargs['body'] = json.dumps(data) if 'method' not in kwargs: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 793a583bc..e5a85e6fc 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import cgi import unittest import re import json +import warnings import six from six.moves import xmlrpc_client as xmlrpclib @@ -1153,6 +1154,10 @@ class JSONRequestTest(RequestTest): default_method = 'GET' default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} + def setUp(self): + warnings.simplefilter("always") + super(JSONRequestTest, self).setUp() + def test_data(self): r1 = self.request_class(url="http://www.example.com/") self.assertEqual(r1.body, b'') @@ -1170,9 +1175,23 @@ class JSONRequestTest(RequestTest): self.assertEqual(r3.body, to_bytes(json.dumps(data))) self.assertEqual(r3.method, 'POST') - r4 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r4.body, to_bytes(json.dumps(data))) - self.assertEqual(r4.method, 'POST') + with warnings.catch_warnings(record=True) as _warnings: + r4 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r4.body, body) + self.assertEqual(r4.method, 'GET') + self.assertEqual(len(_warnings), 1) + self.assertIn('data will be ignored', str(_warnings[0].message)) + + with warnings.catch_warnings(record=True) as _warnings: + r5 = self.request_class(url="http://www.example.com/", body=b'', data=data) + self.assertEqual(r5.body, b'') + self.assertEqual(r5.method, 'GET') + self.assertEqual(len(_warnings), 1) + self.assertIn('data will be ignored', str(_warnings[0].message)) + + def tearDown(self): + warnings.resetwarnings() + super(JSONRequestTest, self).tearDown() if __name__ == "__main__": From 3c981bf204c739fa77e205b9747d2aff446c99d5 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sun, 9 Dec 2018 12:56:12 +0530 Subject: [PATCH 05/67] add documentation --- docs/topics/request-response.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e29914dbf..d957915e7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -508,6 +508,38 @@ method for this job. Here's an example spider which uses it:: # continue scraping with authenticated session... +JSONRequest +----------- + +The JSONRequest class extends the base :class:`Request` class with functionality for +dealing with JSON requests. + +.. class:: JSONRequest(url, [data, ...]) + + The :class:`JSONRequest` class adds a new argument to the constructor called data. The + remaining arguments are the same as for the :class:`Request` class and are + not documented here. + + Using the :class:`JSONRequest` will set the `Content-Type` header to `application/json` + and `Accept` header to `application/json, text/javascript, */*; q=0.01` + + :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. + if :attr:`Request.body` argument is provided this parameter will be ignored. + if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + set to POST automatically. + :type data: JSON serializable object + +JSONRequest usage example +------------------------- + +Sending a JSON POST request with a JSON payload:: + + data = { + 'name1': 'value1', + 'name2': 'value2', + } + yield JSONRequest(url='http://www.example.com/post/action', data=data) + Response objects ================ From ecda69130e97629b15d3b09b1e588cb6777ee94d Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 10 Dec 2018 22:34:49 +0530 Subject: [PATCH 06/67] allow to send empty data values and docs changes --- docs/topics/request-response.rst | 6 +++--- scrapy/http/request/json_request.py | 8 +++++--- tests/test_http_request.py | 27 +++++++++++++++++++++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d957915e7..02b853fc0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -520,13 +520,13 @@ dealing with JSON requests. remaining arguments are the same as for the :class:`Request` class and are not documented here. - Using the :class:`JSONRequest` will set the `Content-Type` header to `application/json` - and `Accept` header to `application/json, text/javascript, */*; q=0.01` + Using the :class:`JSONRequest` will set the ``Content-Type`` header to ``application/json`` + and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01`` :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be - set to POST automatically. + set to ``'POST'`` automatically. :type data: JSON serializable object JSONRequest usage example diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 3b791eda3..593dfdcb0 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -13,12 +13,14 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): - body_passed = 'body' in kwargs + body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) - if body_passed and data: + data_passed = data is not None + + if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data: + elif not body_passed and data_passed: kwargs['body'] = json.dumps(data) if 'method' not in kwargs: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e5a85e6fc..5eb655c12 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1175,20 +1175,35 @@ class JSONRequestTest(RequestTest): self.assertEqual(r3.body, to_bytes(json.dumps(data))) self.assertEqual(r3.method, 'POST') + r4 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r4.body, to_bytes(json.dumps([]))) + self.assertEqual(r4.method, 'POST') + with warnings.catch_warnings(record=True) as _warnings: - r4 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r4.body, body) - self.assertEqual(r4.method, 'GET') + r5 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r5.body, body) + self.assertEqual(r5.method, 'GET') self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) with warnings.catch_warnings(record=True) as _warnings: - r5 = self.request_class(url="http://www.example.com/", body=b'', data=data) - self.assertEqual(r5.body, b'') - self.assertEqual(r5.method, 'GET') + r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) + self.assertEqual(r6.body, b'') + self.assertEqual(r6.method, 'GET') self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + with warnings.catch_warnings(record=True) as _warnings: + r7 = self.request_class(url="http://www.example.com/", body=None, data=data) + self.assertEqual(r7.body, to_bytes(json.dumps(data))) + self.assertEqual(r7.method, 'POST') + self.assertEqual(len(_warnings), 0) + + with warnings.catch_warnings(record=True) as _warnings: + r8 = self.request_class(url="http://www.example.com/", body=None, data=None) + self.assertEqual(r8.method, 'GET') + self.assertEqual(len(_warnings), 0) + def tearDown(self): warnings.resetwarnings() super(JSONRequestTest, self).tearDown() From 71ef321b68d2fd202de145d0c580387ee59cd2e2 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Wed, 12 Dec 2018 11:12:48 +0530 Subject: [PATCH 07/67] sort_keys while serializing to json --- scrapy/http/request/json_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 593dfdcb0..afc4356a3 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -21,7 +21,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = json.dumps(data) + kwargs['body'] = json.dumps(data, sort_keys=True) if 'method' not in kwargs: kwargs['method'] = 'POST' From 8f1507a4a5de2ed55cb0fda198265845a047fedb Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 17 Dec 2018 23:14:06 +0530 Subject: [PATCH 08/67] dumps_kwargs --- docs/topics/request-response.rst | 10 ++- scrapy/http/request/json_request.py | 21 ++++- tests/test_http_request.py | 114 +++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 7 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 02b853fc0..4e6f00bb0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -514,9 +514,9 @@ JSONRequest The JSONRequest class extends the base :class:`Request` class with functionality for dealing with JSON requests. -.. class:: JSONRequest(url, [data, ...]) +.. class:: JSONRequest(url, [... data]) - The :class:`JSONRequest` class adds a new argument to the constructor called data. The + The :class:`JSONRequest` class adds two new argument to the constructor. The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -529,6 +529,12 @@ dealing with JSON requests. set to ``'POST'`` automatically. :type data: JSON serializable object + :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data + into JSON format. + :type dumps_kwargs: dict + +.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps + JSONRequest usage example ------------------------- diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index afc4356a3..7499610b9 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -13,6 +13,7 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): + dumps_kwargs = kwargs.pop('dumps_kwargs', {}) body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None @@ -21,7 +22,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = json.dumps(data, sort_keys=True) + kwargs['body'] = self.dump(data, **dumps_kwargs) if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -29,3 +30,21 @@ class JSONRequest(Request): super(JSONRequest, self).__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + self._dumps_kwargs = dumps_kwargs + + def replace(self, *args, **kwargs): + body_passed = kwargs.get('body', None) is not None + data = kwargs.pop('data', None) + data_passed = data is not None + + if body_passed and data_passed: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data_passed: + kwargs['body'] = self.dump(data, **self._dumps_kwargs) + + return super(JSONRequest, self).replace(*args, **kwargs) + + def dump(self, data, **kwargs): + """Convert to JSON """ + return json.dumps(data, sort_keys=True, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5eb655c12..6dcfa25da 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -14,6 +14,8 @@ if six.PY3: from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str +from tests import mock + class RequestTest(unittest.TestCase): @@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest): def test_data(self): r1 = self.request_class(url="http://www.example.com/") self.assertEqual(r1.body, b'') - self.assertEqual(r1.method, 'GET') body = b'body' r2 = self.request_class(url="http://www.example.com/", body=body) self.assertEqual(r2.body, body) - self.assertEqual(r2.method, 'GET') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) - self.assertEqual(r3.method, 'POST') + # empty data r4 = self.request_class(url="http://www.example.com/", data=[]) self.assertEqual(r4.body, to_bytes(json.dumps([]))) - self.assertEqual(r4.method, 'POST') + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.method, 'GET') + + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.method, 'GET') + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.method, 'POST') + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method='GET') + self.assertEqual(r4.method, 'GET') + + r5 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r5.method, 'POST') + + def test_body_data(self): + """ passing both body and data should result a warning """ + body = b'body' + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) self.assertEqual(r5.body, body) @@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest): self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + def test_empty_body_data(self): + """ passing any body value and data should result a warning """ + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) self.assertEqual(r6.body, b'') @@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest): self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + def test_body_none_data(self): + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) self.assertEqual(r7.body, to_bytes(json.dumps(data))) self.assertEqual(r7.method, 'POST') self.assertEqual(len(_warnings), 0) + def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) self.assertEqual(r8.method, 'GET') self.assertEqual(len(_warnings), 0) + def test_dumps_sort_keys(self): + """ Test that sort_keys=True is passed to json.dumps by default """ + data = { + 'name': 'value', + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_dumps_kwargs(self): + """ Test that dumps_kwargs are passed to json.dumps """ + data = { + 'name': 'value', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + + def test_replace_data(self): + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + + def test_replace_sort_keys(self): + """ Test that replace provides sort_keys=True to json.dumps """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_replace_dumps_kwargs(self): + """ Test that dumps_kwargs are provided json.dumps when replace is called """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + def tearDown(self): warnings.resetwarnings() super(JSONRequestTest, self).tearDown() From 12ad06b7ac57dd022a4add16259ee8fd64d5ede2 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 17 Dec 2018 23:17:13 +0530 Subject: [PATCH 09/67] docs change --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4e6f00bb0..6758269b1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -529,8 +529,8 @@ dealing with JSON requests. set to ``'POST'`` automatically. :type data: JSON serializable object - :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data - into JSON format. + :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize + data into JSON format. :type dumps_kwargs: dict .. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps From 24acc50d1894b6566e427f1dfea14e2aa647077e Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Tue, 18 Dec 2018 23:16:14 +0530 Subject: [PATCH 10/67] dumps_kwargs parameter in docs --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6758269b1..37b73edd1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -514,7 +514,7 @@ JSONRequest The JSONRequest class extends the base :class:`Request` class with functionality for dealing with JSON requests. -.. class:: JSONRequest(url, [... data]) +.. class:: JSONRequest(url, [... data, dumps_kwargs]) The :class:`JSONRequest` class adds two new argument to the constructor. The remaining arguments are the same as for the :class:`Request` class and are From f85c915872cf70bb87a05cecd6ef5a6534d2c4ed Mon Sep 17 00:00:00 2001 From: Joaquin Garmendia Cabrera Date: Sun, 23 Dec 2018 00:26:58 -0500 Subject: [PATCH 11/67] Update item-pipeline example --- docs/topics/item-pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 38265b474..1c2c51e05 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -87,8 +87,8 @@ contain a price:: vat_factor = 1.15 def process_item(self, item, spider): - if item['price']: - if item['price_excludes_vat']: + if 'price' in item and item['price']: + if 'price_excludes_vat' in item and item['price_excludes_vat']: item['price'] = item['price'] * self.vat_factor return item else: From e1f8b55ba0a132ed28c71661e2df3c5bc27feb75 Mon Sep 17 00:00:00 2001 From: Joaquin Garmendia Cabrera Date: Fri, 28 Dec 2018 16:53:12 -0500 Subject: [PATCH 12/67] Improve syntax for readability --- docs/topics/item-pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 1c2c51e05..fae18200a 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -87,8 +87,8 @@ contain a price:: vat_factor = 1.15 def process_item(self, item, spider): - if 'price' in item and item['price']: - if 'price_excludes_vat' in item and item['price_excludes_vat']: + if item.get('price'): + if item.get('price_excludes_vat'): item['price'] = item['price'] * self.vat_factor return item else: From 3f914f6d8c369a18e1f856c01b7d1ad2a63f6e49 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 14 Jan 2019 23:03:14 +0530 Subject: [PATCH 13/67] made jsonrequest dump into private method --- scrapy/http/request/json_request.py | 15 +++++++++------ tests/test_http_request.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 7499610b9..1e2c6b0c6 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,6 +5,7 @@ This module implements the JSONRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +import copy import json import warnings @@ -13,7 +14,10 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): - dumps_kwargs = kwargs.pop('dumps_kwargs', {}) + dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + dumps_kwargs['sort_keys'] = True + self._dumps_kwargs = dumps_kwargs + body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None @@ -22,7 +26,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = self.dump(data, **dumps_kwargs) + kwargs['body'] = self._dumps(data) if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -30,7 +34,6 @@ class JSONRequest(Request): super(JSONRequest, self).__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') - self._dumps_kwargs = dumps_kwargs def replace(self, *args, **kwargs): body_passed = kwargs.get('body', None) is not None @@ -41,10 +44,10 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = self.dump(data, **self._dumps_kwargs) + kwargs['body'] = self._dumps(data) return super(JSONRequest, self).replace(*args, **kwargs) - def dump(self, data, **kwargs): + def _dumps(self, data): """Convert to JSON """ - return json.dumps(data, sort_keys=True, **kwargs) + return json.dumps(data, **self._dumps_kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dcfa25da..49f148016 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1292,7 +1292,7 @@ class JSONRequestTest(RequestTest): self.assertEqual(kwargs['sort_keys'], True) def test_replace_dumps_kwargs(self): - """ Test that dumps_kwargs are provided json.dumps when replace is called """ + """ Test that dumps_kwargs are provided to json.dumps when replace is called """ data1 = { 'name1': 'value1', } From bdf12f775062fda8aa8bf03f7b4faade4faac16d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 18 Jan 2019 11:38:59 -0200 Subject: [PATCH 14/67] Logging the request referer when DUPEFILTER_DEBUG is active --- scrapy/dupefilters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 9d8966b9c..0bcdd3495 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -3,8 +3,7 @@ import os import logging from scrapy.utils.job import job_dir -from scrapy.utils.request import request_fingerprint - +from scrapy.utils.request import referer_str, request_fingerprint class BaseDupeFilter(object): @@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter): def log(self, request, spider): if self.debug: - msg = "Filtered duplicate request: %(request)s" - self.logger.debug(msg, {'request': request}, extra={'spider': spider}) + msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" + args = {'request': request, 'referer': referer_str(request) } + self.logger.debug(msg, args, extra={'spider': spider}) elif self.logdupes: msg = ("Filtered duplicate request: %(request)s" " - no more duplicates will be shown" From 8eade7d8640e112faf8677f4666bbe3ab10c7234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 18 Jan 2019 11:39:35 -0200 Subject: [PATCH 15/67] Testing stats and log messages from RFPDupeFilter --- tests/test_dupefilters.py | 57 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index db69597a2..d7eb98c97 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,6 +2,7 @@ import hashlib import tempfile import unittest import shutil +from testfixtures import LogCapture from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request @@ -9,7 +10,7 @@ from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler - +from tests.spiders import SimpleSpider class FromCrawlerRFPDupeFilter(RFPDupeFilter): @@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase): assert case_insensitive_dupefilter.request_seen(r2) case_insensitive_dupefilter.close('finished') + + def test_log(self): + with LogCapture() as l: + settings = {'DUPEFILTER_DEBUG': False, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + spider = SimpleSpider.from_crawler(crawler) + + dupefilter = scheduler.df + dupefilter.open() + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html') + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' - no more duplicates will be shown' + ' (see DUPEFILTER_DEBUG to show all duplicates)'))) + + dupefilter.close('finished') + + def test_log_debug(self): + with LogCapture() as l: + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + spider = SimpleSpider.from_crawler(crawler) + + dupefilter = scheduler.df + dupefilter.open() + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html', + headers={'Referer': 'http://scrapytest.org/INDEX.html'} + ) + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' (referer: None)'))) + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' (referer: http://scrapytest.org/INDEX.html)'))) + + dupefilter.close('finished') From 71743a6546e96b5d99bd3c068a7ec5b71dca1659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 19 Jan 2019 18:43:58 +0000 Subject: [PATCH 16/67] Add release notes for v1.5.2 --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 01016e2e6..adf679ded 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,6 +12,22 @@ Cleanups * Remove deprecated ``CrawlerSettings`` class. * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. +Scrapy 1.5.2 (2019-01-22) +------------------------- + +* *Security bugfix*: Telnet console extension can be easily exploited by rogue + websites POSTing content to http://localhost:6023, we haven't found a way to + exploit it from Scrapy, but it is very easy to trick a browser to do so and + elevates the risk for local development environment. + + *The fix is backwards incompatible*, it enables telnet user-password + authentication by default with a random generated password. If you can't + upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT` + out of its default value. + + See :ref:`telnet console ` documentation for more info + +* Backport CI build failure under GCE environemnt due to boto import error. Scrapy 1.5.1 (2018-07-12) ------------------------- From d9aa5391327dd34f8d840e7ce2bca1eb8583d932 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Fri, 25 Jan 2019 21:26:28 +0530 Subject: [PATCH 17/67] enabled sort keys only if not provided --- scrapy/http/request/json_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1e2c6b0c6..8f7a61a6d 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -15,7 +15,7 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) - dumps_kwargs['sort_keys'] = True + dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs body_passed = kwargs.get('body', None) is not None From b828b5f8c8650a30aef382af661b6c7b9ea57186 Mon Sep 17 00:00:00 2001 From: Harry Moreno Date: Sat, 26 Jan 2019 18:39:05 -0500 Subject: [PATCH 18/67] fix grammar --- docs/topics/jobs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 8e1574376..ea684b4cf 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -30,7 +30,7 @@ a *single* job. How to use it ============= -To start a spider with persistence supported enabled, run it like this:: +To start a spider with persistence support enabled, run it like this:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 From 8fca98616a90d8452ff2e488bfea93e5c89caf08 Mon Sep 17 00:00:00 2001 From: Harry Moreno Date: Sat, 26 Jan 2019 16:47:10 -0500 Subject: [PATCH 19/67] fix grammar --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index a1f518cbd..c60b55391 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -310,7 +310,7 @@ images. .. setting:: IMAGES_THUMBS -In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary +In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary where the keys are the thumbnail names and the values are their dimensions. For example:: From 706910790b6ee755bafa828606e215e668af3eee Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 18:28:24 +0500 Subject: [PATCH 20/67] [wip] draft 1.6 release notes --- docs/news.rst | 153 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index adf679ded..99a339cea 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,11 +6,156 @@ Release notes Scrapy 1.6.0 (unreleased) ------------------------- +Highlights for this release: + +* better Windows compatibility; +* Python 3.7 compatibility; +* big documentation improvements, including a switch + from ``.extract() / .extract_first()`` API to ``.get() / .getall()`` API; +* Feed exports, FilePipeline and MediaPipeline improvements; +* ``scrapy.contracts`` fixes and new features; +* large clean-up of deprecated code +* TODO + +parsel 1.5 +~~~~~~~~~~ + +TODO +While this is not a change in Scrapy itself, a new version of ``parsel`` +is released; Scrapy now depends on ``parsel >= 1.5``. + +Feed export improvements +~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``from_crawler`` support is added to feed exporters and feed storages. This, + among other things, allow to access Scrapy settings from custom storages + and exporters (:issue:`1605`, :issue:`3348`). +* fixed issue with extra blank lines in .csv exports under Windows + (:issue:`3039`); +* better error message when an exporter is disabled (:issue:`3358`); + +FilePipeline and MediaPipeline improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`, + :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`. + For example, this allows to use alternative or self-hosted + AWS-compatible providers (:issue:`2609`). +* ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and + :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`). + +``scrapy.contracts`` improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Exceptions in contracts code are handled better (:issue:`3377`); +* ``dont_filter=True`` is used for contract requests, which allows to test + different callbacks with the same URL (:issue:`3381`); +* ``request_cls`` attribute in Contract subclasses allow to use different + Request classes in contracts, for example FormRequest (:issue:`3383`). +* Fixed errback handling in contracts, e.g. for cases where a contract + is executed for URL which returns non-200 response (:issue:`3371`). + +Documentation improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Docs are re-written to suggest .get/.getall API instead of + .extract/.extract_first. Also, :ref:`topics-selectors` docs are updated + and re-structured to match latest parsel docs; they now contain more topics, + such as :ref:`selecting-attributes` or :ref:`topics-selectors-css-extensions` + (:issue:`3390`). +* :ref:`topics-developer-tools` is a new tutorial which replaces + old Firefox and Firebug tutorials (:issue:`3400`). +* SCRAPY_PROJECT environment variable is documented (:issue:`3518`); +* troubleshooting section is added to install instructions (:issue:`3517`); +* improved links to beginner resources in the tutorial + (:issue:`3367`, :issue:`3468`); +* fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); +* remove unused `DEPTH_STATS` option from docs (:issue:`3245`); +* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`). + +Better Windows support +~~~~~~~~~~~~~~~~~~~~~~ + +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Scrapy used to produce unnecessary blank lines in .csv exports on Windows, + this is fixed (:issue:`3039`). + +Testing fixes +~~~~~~~~~~~~~ + +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`) +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +Compatibility shims for pre-1.0 Scrapy module names are removed +(:issue:`3318`): + +* ``scrapy.command`` +* ``scrapy.contrib`` (with all submodules) +* ``scrapy.contrib_exp`` (with all submodules) +* ``scrapy.dupefilter`` +* ``scrapy.linkextractor`` +* ``scrapy.project`` +* ``scrapy.spider`` +* ``scrapy.spidermanager`` +* ``scrapy.squeue`` +* ``scrapy.stats`` +* ``scrapy.statscol`` +* ``scrapy.utils.decorator`` + +See :ref:`module_relocations` for more information, or use suggestions +from Scrapy 1.5.x deprecation warnings to update your code. + +Other deprecation removals: + +* Deprecated scrapy.interfaces.ISpiderManager is removed; please use + scrapy.interfaces.ISpiderLoader. +* Deprecated ``CrawlerSettings`` class is removed (:issue:`3327`). +* Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes + are removed (:issue:`3327`, :issue:`3359`). + +Internal improvements +~~~~~~~~~~~~~~~~~~~~~ + +* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows + to access e.g. settings or a spider from a dupefilter. +* :signal:`item_error` is fired when an error happens in a pipeline + (:issue:`3256`); +* :signal:`request_reached_downloader` is fired when Downloader gets + a new Request; this signal can be useful e.g. for custom Schedulers + (:issue:`3393`). +* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" + optional arguments (:issue:`3231`). + +Usability improvements +~~~~~~~~~~~~~~~~~~~~~~ + +* more stats for RobotsTxtMiddleware (:issue:`3100`) +* INFO log level is used to show telnet host/port (:issue:`3115`) +* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) +* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) +* non-zero exit code is returned from Scrapy commands when error happens + on spider inititalization (:issue:`3226`) +* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + "flv" is added to common video extensions (:issue:`3165`) + +Bug fixes +~~~~~~~~~ +* proper handling of pickling errors in Python 3 when serializing objects + for disk queues (:issue:`3082`) +* flags are now preserved when copying Requests (:issue:`3342`); +* FormRequest.from_response clickdata shouldn't ignore elements with + ``input[type=image]`` (:issue:`3153`). +* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) + Cleanups ~~~~~~~~ - -* Remove deprecated ``CrawlerSettings`` class. -* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. +* additional files are included to sdist (:issue:`3495`); +* code style fixes (:issue:`3405`, :issue:`3304`) Scrapy 1.5.2 (2019-01-22) ------------------------- @@ -1080,6 +1225,8 @@ until it reaches a stable status. See more examples for scripts running Scrapy: :ref:`topics-practices` +.. _module_relocations: + Module Relocations ~~~~~~~~~~~~~~~~~~ From e479f5aa15809e7f75a7dbc20d0629f57be46b5d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 27 Dec 2018 00:48:10 +0500 Subject: [PATCH 21/67] DOC update changelog * changes from recently merged pull requests * more highlights * re-organized headers * Selector API changes --- docs/news.rst | 142 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 105 insertions(+), 37 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 99a339cea..bf469a350 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,41 +6,83 @@ Release notes Scrapy 1.6.0 (unreleased) ------------------------- -Highlights for this release: +Highlights: -* better Windows compatibility; +* better Windows support; * Python 3.7 compatibility; * big documentation improvements, including a switch - from ``.extract() / .extract_first()`` API to ``.get() / .getall()`` API; -* Feed exports, FilePipeline and MediaPipeline improvements; + from ``.extract()`` + ``.extract_first()`` API to ``.get()`` + ``.getall()`` + API; +* feed exports, FilePipeline and MediaPipeline improvements; +* better extensibility: :signal:`item_error` and + :signal:`request_reached_downloader` signals; ``from_crawler`` support + for feed exporters, feed storages and dupefilters. * ``scrapy.contracts`` fixes and new features; -* large clean-up of deprecated code -* TODO +* telnet console security improvements; +* clean-up of the deprecated code; +* various bug fixes, small new features and usability improvements across + the codebase. -parsel 1.5 -~~~~~~~~~~ +Selector API changes +~~~~~~~~~~~~~~~~~~~~ -TODO -While this is not a change in Scrapy itself, a new version of ``parsel`` -is released; Scrapy now depends on ``parsel >= 1.5``. +While these are not changes in Scrapy itself, but rather in the parsel_ +library which Scrapy uses for xpath/css selectors, these changes are +worth mentioning here. Scrapy now depends on parsel >= 1.5, and +Scrapy documentation is updated to follow recent ``parsel`` API conventions. -Feed export improvements -~~~~~~~~~~~~~~~~~~~~~~~~ +Most visible change is that ``.get()`` and ``.getall()`` selector +methods are now preferred over ``.extract()`` and ``.extract_first()``. +We feel that these new methods result in a more concise and readable code. +See :ref:`old-extraction-api` for more details. + +.. note:: + There are currently **no plans** to deprecate ``.extract()`` + and ``.extract_first()`` methods. + +Another useful new feature is the introduction of ``Selector.attrib`` and +``SelectorList.attrib`` properties, which make it easier to get +attributes of HTML elements. See :ref:`selecting-attributes`. + +CSS selectors are cached in parsel >= 1.5, which makes them faster +when the same CSS path is used many times. This is very common in +case of Scrapy spiders: callbacks are usually called several times, +on different pages. + +If you're using custom ``Selector`` or ``SelectorList`` subclasses, +a **backwards incompatible** change in parsel may affect your code. +See `parsel changelog`_ for a detailed description, as well as for the +full list of improvements. + +.. _parsel changelog: https://parsel.readthedocs.io/en/latest/history.html + +Telnet console +~~~~~~~~~~~~~~ + +**Backwards incompatible**: Scrapy's telnet console now requires username +and password. See :ref:`topics-telnetconsole` for more details. + +New extensibility features +~~~~~~~~~~~~~~~~~~~~~~~~~~ * ``from_crawler`` support is added to feed exporters and feed storages. This, - among other things, allow to access Scrapy settings from custom storages - and exporters (:issue:`1605`, :issue:`3348`). -* fixed issue with extra blank lines in .csv exports under Windows - (:issue:`3039`); -* better error message when an exporter is disabled (:issue:`3358`); + among other things, allows to access Scrapy settings from custom feed + storages and exporters (:issue:`1605`, :issue:`3348`). +* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows + to access e.g. settings or a spider from a dupefilter. +* :signal:`item_error` is fired when an error happens in a pipeline + (:issue:`3256`); +* :signal:`request_reached_downloader` is fired when Downloader gets + a new Request; this signal can be useful e.g. for custom Schedulers + (:issue:`3393`). -FilePipeline and MediaPipeline improvements +New FilePipeline and MediaPipeline features ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`. For example, this allows to use alternative or self-hosted - AWS-compatible providers (:issue:`2609`). + AWS-compatible providers (:issue:`2609`, :issue:`3548`). * ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`). @@ -55,6 +97,47 @@ FilePipeline and MediaPipeline improvements * Fixed errback handling in contracts, e.g. for cases where a contract is executed for URL which returns non-200 response (:issue:`3371`). +Usability and other improvements, cleanups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). +* Lazy loading of Downloader Handlers is now optional; this enables better + initialization error handling in custom Downloader Handlers (:issue:`3394`). +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) +* better error message when an exporter is disabled (:issue:`3358`); +* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" + optional arguments (:issue:`3231`). +* more stats for RobotsTxtMiddleware (:issue:`3100`) +* INFO log level is used to show telnet host/port (:issue:`3115`) +* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) +* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) +* non-zero exit code is returned from Scrapy commands when error happens + on spider inititalization (:issue:`3226`); +* link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + "flv" is added to common video extensions (:issue:`3165`) +* `scrapy shell --help` mentions syntax required for local files + (``./file.html``) - :issue:`3496`. +* additional files are included to sdist (:issue:`3495`); +* code style fixes (:issue:`3405`, :issue:`3304`); +* unneeded .strip() call is removed (:issue:`3519`); +* collections.deque is used to store MiddlewareManager methods instead + of a list (:issue:`3476`) + +Bug fixes +~~~~~~~~~ + +* fixed issue with extra blank lines in .csv exports under Windows + (:issue:`3039`); +* proper handling of pickling errors in Python 3 when serializing objects + for disk queues (:issue:`3082`) +* flags are now preserved when copying Requests (:issue:`3342`); +* FormRequest.from_response clickdata shouldn't ignore elements with + ``input[type=image]`` (:issue:`3153`). +* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) + Documentation improvements ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -73,21 +156,6 @@ Documentation improvements * remove unused `DEPTH_STATS` option from docs (:issue:`3245`); * other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`). -Better Windows support -~~~~~~~~~~~~~~~~~~~~~~ - -* All Scrapy tests now pass on Windows; Scrapy testing suite is executed - in a Windows environment on CI (:issue:`3315`). -* Scrapy used to produce unnecessary blank lines in .csv exports on Windows, - this is fixed (:issue:`3039`). - -Testing fixes -~~~~~~~~~~~~~ - -* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`) -* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, - :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) - Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -107,7 +175,7 @@ Compatibility shims for pre-1.0 Scrapy module names are removed * ``scrapy.statscol`` * ``scrapy.utils.decorator`` -See :ref:`module_relocations` for more information, or use suggestions +See :ref:`module-relocations` for more information, or use suggestions from Scrapy 1.5.x deprecation warnings to update your code. Other deprecation removals: @@ -1225,7 +1293,7 @@ until it reaches a stable status. See more examples for scripts running Scrapy: :ref:`topics-practices` -.. _module_relocations: +.. _module-relocations: Module Relocations ~~~~~~~~~~~~~~~~~~ From 638469f9efdcc104f7b1a1c1a9890694e0d41c68 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 28 Dec 2018 01:13:01 +0500 Subject: [PATCH 22/67] DOC extract_first/extract matches get/getall better Thanks @Gallaecio! --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index bf469a350..4a236f1b9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,7 +11,7 @@ Highlights: * better Windows support; * Python 3.7 compatibility; * big documentation improvements, including a switch - from ``.extract()`` + ``.extract_first()`` API to ``.get()`` + ``.getall()`` + from ``.extract_first()`` + ``.extract()`` API to ``.get()`` + ``.getall()`` API; * feed exports, FilePipeline and MediaPipeline improvements; * better extensibility: :signal:`item_error` and @@ -32,7 +32,7 @@ worth mentioning here. Scrapy now depends on parsel >= 1.5, and Scrapy documentation is updated to follow recent ``parsel`` API conventions. Most visible change is that ``.get()`` and ``.getall()`` selector -methods are now preferred over ``.extract()`` and ``.extract_first()``. +methods are now preferred over ``.extract_first()`` and ``.extract()``. We feel that these new methods result in a more concise and readable code. See :ref:`old-extraction-api` for more details. From 4cf4dd1d3e068e0df32f700c89d833cc7cd79b85 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 03:08:17 +0500 Subject: [PATCH 23/67] DOC add recent changes to changelog --- docs/news.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 4a236f1b9..1a08f93ec 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -75,6 +75,9 @@ New extensibility features * :signal:`request_reached_downloader` is fired when Downloader gets a new Request; this signal can be useful e.g. for custom Schedulers (:issue:`3393`). +* new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows + to select sitemap entries based on their attributes in SitemapSpider + subclasses (:issue:`3512`). New FilePipeline and MediaPipeline features ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -125,6 +128,7 @@ Usability and other improvements, cleanups * unneeded .strip() call is removed (:issue:`3519`); * collections.deque is used to store MiddlewareManager methods instead of a list (:issue:`3476`) +* Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) Bug fixes ~~~~~~~~~ @@ -154,7 +158,8 @@ Documentation improvements (:issue:`3367`, :issue:`3468`); * fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); * remove unused `DEPTH_STATS` option from docs (:issue:`3245`); -* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`). +* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`, + :issue:`3605`). Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 0fc9d705c271f5d87174143c09f95993e5a45797 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 03:28:19 +0500 Subject: [PATCH 24/67] DOC mention that telnet security improvements happened in 1.5.2 --- docs/news.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 1a08f93ec..a4f07efad 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,8 @@ Release notes ============= +.. _release-1.6.0: + Scrapy 1.6.0 (unreleased) ------------------------- @@ -18,11 +20,13 @@ Highlights: :signal:`request_reached_downloader` signals; ``from_crawler`` support for feed exporters, feed storages and dupefilters. * ``scrapy.contracts`` fixes and new features; -* telnet console security improvements; +* telnet console security improvements, first released as a + backport in :ref:`release-1.5.2`; * clean-up of the deprecated code; * various bug fixes, small new features and usability improvements across the codebase. + Selector API changes ~~~~~~~~~~~~~~~~~~~~ @@ -230,6 +234,8 @@ Cleanups * additional files are included to sdist (:issue:`3495`); * code style fixes (:issue:`3405`, :issue:`3304`) +.. _release-1.5.2: + Scrapy 1.5.2 (2019-01-22) ------------------------- @@ -247,6 +253,8 @@ Scrapy 1.5.2 (2019-01-22) * Backport CI build failure under GCE environemnt due to boto import error. +.. _release-1.5.1: + Scrapy 1.5.1 (2018-07-12) ------------------------- @@ -262,6 +270,9 @@ This is a maintenance release with important bug fixes, but no new features: :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`, :issue:`3294`). + +.. _release-1.5.0: + Scrapy 1.5.0 (2017-12-29) ------------------------- @@ -373,6 +384,7 @@ Docs - Document ``from_crawler`` methods for spider and downloader middlewares (:issue:`3019`) +.. _release-1.4.0: Scrapy 1.4.0 (2017-05-18) ------------------------- @@ -559,6 +571,8 @@ Documentation - Clarify ``allowed_domains`` example (:issue:`2670`) +.. _release-1.3.3: + Scrapy 1.3.3 (2017-03-10) ------------------------- @@ -571,6 +585,7 @@ Bug fixes A new setting is introduced to toggle between warning or exception if needed ; see :setting:`SPIDER_LOADER_WARN_ONLY` for details. +.. _release-1.3.2: Scrapy 1.3.2 (2017-02-13) ------------------------- @@ -582,6 +597,8 @@ Bug fixes - Use consistent selectors for author field in tutorial (:issue:`2551`). - Fix TLS compatibility in Twisted 17+ (:issue:`2558`) +.. _release-1.3.1: + Scrapy 1.3.1 (2017-02-08) ------------------------- @@ -630,6 +647,8 @@ Cleanups - Remove dead code supporting old Twisted versions (:issue:`2544`). +.. _release-1.3.0: + Scrapy 1.3.0 (2016-12-21) ------------------------- @@ -669,6 +688,7 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. +.. _release-1.2.3: Scrapy 1.2.3 (2017-03-03) ------------------------- @@ -676,6 +696,8 @@ Scrapy 1.2.3 (2017-03-03) - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.2.2: + Scrapy 1.2.2 (2016-12-06) ------------------------- @@ -711,6 +733,8 @@ Other changes .. _conda-forge: https://anaconda.org/conda-forge/scrapy +.. _release-1.2.1: + Scrapy 1.2.1 (2016-10-21) ------------------------- @@ -735,6 +759,8 @@ Other changes - Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`). +.. _release-1.2.0: + Scrapy 1.2.0 (2016-10-03) ------------------------- @@ -803,12 +829,14 @@ Documentation - Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`). - Add StackOverflow as a support channel (:issue:`2257`). +.. _release-1.1.4: Scrapy 1.1.4 (2017-03-03) ------------------------- - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.1.3: Scrapy 1.1.3 (2016-09-22) ------------------------- @@ -826,6 +854,7 @@ Documentation rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`, :issue:`2252`). +.. _release-1.1.2: Scrapy 1.1.2 (2016-08-18) ------------------------- @@ -840,6 +869,7 @@ Bug fixes - :setting:`IMAGES_EXPIRES` default value set back to 90 (the regression was introduced in 1.1.1) +.. _release-1.1.1: Scrapy 1.1.1 (2016-07-13) ------------------------- @@ -892,6 +922,7 @@ Tests - Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`) +.. _release-1.1.0: Scrapy 1.1.0 (2016-05-11) ------------------------- @@ -1081,12 +1112,14 @@ Bugfixes - HTTPS+CONNECT tunnels could get mixed up when using multiple proxies to same remote host (:issue:`1912`). +.. _release-1.0.7: Scrapy 1.0.7 (2017-03-03) ------------------------- - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.0.6: Scrapy 1.0.6 (2016-05-04) ------------------------- @@ -1096,6 +1129,7 @@ Scrapy 1.0.6 (2016-05-04) - DOC: Support for Sphinx 1.4+ (:issue:`1893`) - DOC: Consistency in selectors examples (:issue:`1869`) +.. _release-1.0.5: Scrapy 1.0.5 (2016-02-04) ------------------------- @@ -1105,6 +1139,7 @@ Scrapy 1.0.5 (2016-02-04) - DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`) - DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`) +.. _release-1.0.4: Scrapy 1.0.4 (2015-12-30) ------------------------- @@ -1158,12 +1193,16 @@ Scrapy 1.0.4 (2015-12-30) - Small grammatical change (:commit:`8752294`) - Add openssl version to version command (:commit:`13c45ac`) +.. _release-1.0.3: + Scrapy 1.0.3 (2015-08-11) ------------------------- - add service_identity to scrapy install_requires (:commit:`cbc2501`) - Workaround for travis#296 (:commit:`66af9cd`) +.. _release-1.0.2: + Scrapy 1.0.2 (2015-08-06) ------------------------- @@ -1174,6 +1213,8 @@ Scrapy 1.0.2 (2015-08-06) - Fixed typos (:commit:`a9ae7b0`) - Fix doc reference. (:commit:`7c8a4fe`) +.. _release-1.0.1: + Scrapy 1.0.1 (2015-07-01) ------------------------- @@ -1184,6 +1225,8 @@ Scrapy 1.0.1 (2015-07-01) - DOC remove version suffix from ubuntu package (:commit:`5303c66`) - DOC Update release date for 1.0 (:commit:`c89fa29`) +.. _release-1.0.0: + Scrapy 1.0.0 (2015-06-19) ------------------------- From 2c8c8b2dd8683787826713ed1d0fbfb2ec1af04a Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 17:30:13 +0500 Subject: [PATCH 25/67] DOC fix after bad merge - remove duplicate entries in changelog --- docs/news.rst | 72 +++++++++++++-------------------------------------- 1 file changed, 18 insertions(+), 54 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index a4f07efad..4711d2f35 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,7 +26,6 @@ Highlights: * various bug fixes, small new features and usability improvements across the codebase. - Selector API changes ~~~~~~~~~~~~~~~~~~~~ @@ -82,6 +81,8 @@ New extensibility features * new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows to select sitemap entries based on their attributes in SitemapSpider subclasses (:issue:`3512`). +* Lazy loading of Downloader Handlers is now optional; this enables better + initialization error handling in custom Downloader Handlers (:issue:`3394`). New FilePipeline and MediaPipeline features ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -104,34 +105,20 @@ New FilePipeline and MediaPipeline features * Fixed errback handling in contracts, e.g. for cases where a contract is executed for URL which returns non-200 response (:issue:`3371`). -Usability and other improvements, cleanups -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Usability improvements +~~~~~~~~~~~~~~~~~~~~~~ -* All Scrapy tests now pass on Windows; Scrapy testing suite is executed - in a Windows environment on CI (:issue:`3315`). -* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). -* Lazy loading of Downloader Handlers is now optional; this enables better - initialization error handling in custom Downloader Handlers (:issue:`3394`). -* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, - :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) -* better error message when an exporter is disabled (:issue:`3358`); -* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" - optional arguments (:issue:`3231`). * more stats for RobotsTxtMiddleware (:issue:`3100`) * INFO log level is used to show telnet host/port (:issue:`3115`) * a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) * better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) * non-zero exit code is returned from Scrapy commands when error happens - on spider inititalization (:issue:`3226`); -* link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + on spider inititalization (:issue:`3226`) +* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); "flv" is added to common video extensions (:issue:`3165`) +* better error message when an exporter is disabled (:issue:`3358`); * `scrapy shell --help` mentions syntax required for local files (``./file.html``) - :issue:`3496`. -* additional files are included to sdist (:issue:`3495`); -* code style fixes (:issue:`3405`, :issue:`3304`); -* unneeded .strip() call is removed (:issue:`3519`); -* collections.deque is used to store MiddlewareManager methods instead - of a list (:issue:`3476`) * Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) Bug fixes @@ -195,44 +182,21 @@ Other deprecation removals: * Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes are removed (:issue:`3327`, :issue:`3359`). -Internal improvements -~~~~~~~~~~~~~~~~~~~~~ +Other improvements, cleanups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows - to access e.g. settings or a spider from a dupefilter. -* :signal:`item_error` is fired when an error happens in a pipeline - (:issue:`3256`); -* :signal:`request_reached_downloader` is fired when Downloader gets - a new Request; this signal can be useful e.g. for custom Schedulers - (:issue:`3393`). +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) * ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" optional arguments (:issue:`3231`). - -Usability improvements -~~~~~~~~~~~~~~~~~~~~~~ - -* more stats for RobotsTxtMiddleware (:issue:`3100`) -* INFO log level is used to show telnet host/port (:issue:`3115`) -* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) -* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) -* non-zero exit code is returned from Scrapy commands when error happens - on spider inititalization (:issue:`3226`) -* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); - "flv" is added to common video extensions (:issue:`3165`) - -Bug fixes -~~~~~~~~~ -* proper handling of pickling errors in Python 3 when serializing objects - for disk queues (:issue:`3082`) -* flags are now preserved when copying Requests (:issue:`3342`); -* FormRequest.from_response clickdata shouldn't ignore elements with - ``input[type=image]`` (:issue:`3153`). -* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) - -Cleanups -~~~~~~~~ * additional files are included to sdist (:issue:`3495`); -* code style fixes (:issue:`3405`, :issue:`3304`) +* code style fixes (:issue:`3405`, :issue:`3304`); +* unneeded .strip() call is removed (:issue:`3519`); +* collections.deque is used to store MiddlewareManager methods instead + of a list (:issue:`3476`) .. _release-1.5.2: From 91791cd329936ee6ac53523460f9b72c20c66afb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 17:53:58 +0500 Subject: [PATCH 26/67] DOC final changelog cleanups --- docs/news.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 4711d2f35..543901809 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -63,7 +63,8 @@ Telnet console ~~~~~~~~~~~~~~ **Backwards incompatible**: Scrapy's telnet console now requires username -and password. See :ref:`topics-telnetconsole` for more details. +and password. See :ref:`topics-telnetconsole` for more details. This change +fixes a **security issue**; see :ref:`release-1.5.2` release notes for details. New extensibility features ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -117,7 +118,7 @@ Usability improvements * Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); "flv" is added to common video extensions (:issue:`3165`) * better error message when an exporter is disabled (:issue:`3358`); -* `scrapy shell --help` mentions syntax required for local files +* ``scrapy shell --help`` mentions syntax required for local files (``./file.html``) - :issue:`3496`. * Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) From b8594353d03be5574f51766c35566b713584302b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 30 Jan 2019 18:00:40 -0300 Subject: [PATCH 27/67] =?UTF-8?q?Bump=20version:=201.5.0=20=E2=86=92=201.6?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6e7be142e..8cecb7ad4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.5.0 +current_version = 1.6.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index bc80560fa..dc1e644a1 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.5.0 +1.6.0 From 88326cd8be7f9c9f09924144a4d3a9666fdcf0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 31 Jan 2019 01:16:28 -0300 Subject: [PATCH 28/67] Set release date to 1.6.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 543901809..668473887 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-1.6.0: -Scrapy 1.6.0 (unreleased) +Scrapy 1.6.0 (2019-01-30) ------------------------- Highlights: From 65d631329a1434ec013f24341e4b8520241aec70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 31 Jan 2019 01:28:53 -0300 Subject: [PATCH 29/67] Be consistent with domain used for links to documentation website --- CONTRIBUTING.md | 2 +- INSTALL | 2 +- README.rst | 8 ++++---- docs/contributing.rst | 2 +- docs/topics/selectors.rst | 4 ++-- scrapy/extensions/telnet.py | 2 +- scrapy/templates/project/module/items.py.tmpl | 2 +- .../project/module/middlewares.py.tmpl | 2 +- .../project/module/pipelines.py.tmpl | 2 +- .../templates/project/module/settings.py.tmpl | 20 +++++++++---------- sep/sep-001.rst | 2 +- sep/sep-006.rst | 4 ++-- tests/__init__.py | 2 +- 13 files changed, 27 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a11b05d2..a05d07aee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ The guidelines for contributing are available here: -https://doc.scrapy.org/en/master/contributing.html +https://docs.scrapy.org/en/master/contributing.html Please do not abuse the issue tracker for support questions. If your issue topic can be rephrased to "How to ...?", please use the diff --git a/INSTALL b/INSTALL index a3c7899c6..06e812936 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ For information about installing Scrapy see: * docs/intro/install.rst (local file) -* https://doc.scrapy.org/en/latest/intro/install.html (online version) +* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/README.rst b/README.rst index 1361eac26..c28d217ff 100644 --- a/README.rst +++ b/README.rst @@ -51,18 +51,18 @@ The quick way:: pip install scrapy For more details see the install section in the documentation: -https://doc.scrapy.org/en/latest/intro/install.html +https://docs.scrapy.org/en/latest/intro/install.html Documentation ============= -Documentation is available online at https://doc.scrapy.org/ and in the ``docs`` +Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` directory. Releases ======== -You can find release notes at https://doc.scrapy.org/en/latest/news.html +You can find release notes at https://docs.scrapy.org/en/latest/news.html Community (blog, twitter, mail list, IRC) ========================================= @@ -72,7 +72,7 @@ See https://scrapy.org/community/ Contributing ============ -See https://doc.scrapy.org/en/master/contributing.html +See https://docs.scrapy.org/en/master/contributing.html Code of Conduct --------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index 2369c3436..cf27337c8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,7 +7,7 @@ Contributing to Scrapy .. important:: Double check that you are reading the most recent version of this document at - https://doc.scrapy.org/en/master/contributing.html + https://docs.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9dced7473..df1d67ae8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -100,7 +100,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which provides interactive testing) and an example page located in the Scrapy documentation server: - https://doc.scrapy.org/en/latest/_static/selectors-sample1.html + https://docs.scrapy.org/en/latest/_static/selectors-sample1.html .. _topics-selectors-htmlcode: @@ -113,7 +113,7 @@ For the sake of completeness, here's its full HTML code: First, let's open the shell:: - scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html + scrapy shell https://docs.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have the response available as ``response`` shell variable, and its attached selector in ``response.selector`` attribute. diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index dcf73eb88..26b214ee2 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -112,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 7d766f4fc..a12d08414 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your scraped items # # See documentation in: -# https://doc.scrapy.org/en/latest/topics/items.html +# https://docs.scrapy.org/en/latest/topics/items.html import scrapy diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index c5b542bd6..5debe1cd2 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your spider middleware # # See documentation in: -# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index e58dab089..fb641d447 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -3,7 +3,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class ${ProjectName}Pipeline(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a0557473e..cb220eafc 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,9 +5,9 @@ # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # -# https://doc.scrapy.org/en/latest/topics/settings.html -# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html -# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' @@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) -# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: @@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True #} # Enable or disable spider middlewares -# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares -# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions -# See https://doc.scrapy.org/en/latest/topics/extensions.html +# See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines -# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) -# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 @@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) -# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 3766f38fc..2a66f9802 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -61,7 +61,7 @@ ItemForm -------- Pros: -- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html) +- same API used for Items (see https://docs.scrapy.org/en/latest/topics/items.html) - some people consider setitem API more elegant than methods API Cons: diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 7425c0930..366fcf033 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -16,7 +16,7 @@ Motivation ========== When you use Selectors in Scrapy, your final goal is to "extract" the data that -you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html +you've selected, as the [https://docs.scrapy.org/en/latest/topics/selectors.html XPath Selectors documentation] says (bolding by me): When you’re scraping web pages, the most common task you need to perform is @@ -71,5 +71,5 @@ webpage or set of pages. References ========== - 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) + 1. XPath Selectors (https://docs.scrapy.org/topics/selectors.html) 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html) diff --git a/tests/__init__.py b/tests/__init__.py index 55b1ecde8..a54367f8c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ """ tests: this package contains all Scrapy unittests -see https://doc.scrapy.org/en/latest/contributing.html#running-tests +see https://docs.scrapy.org/en/latest/contributing.html#running-tests """ import os From 38af090f4d6799a0499b116a62c12509f59f561b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 4 Feb 2019 11:17:58 +0100 Subject: [PATCH 30/67] Indicate that users must implement their own authentication result check The example of form-based login could lead some users to think its authentication result check was final. See https://stackoverflow.com/a/54410966/939364 This change should make it more obvious that users are expected to implement their own logic to check whether authentication worked or not. --- docs/topics/request-response.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e29914dbf..76360b15f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -489,6 +489,11 @@ method for this job. Here's an example spider which uses it:: import scrapy + def authentication_failed(response): + # TODO: Check the contents of the response and return True if it failed + # or False if it succeeded. + pass + class LoginSpider(scrapy.Spider): name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] @@ -501,8 +506,7 @@ method for this job. Here's an example spider which uses it:: ) def after_login(self, response): - # check login succeed before going on - if "authentication failed" in response.body: + if authentication_failed(response): self.logger.error("Login failed") return From 013568097db04396d780d1c91d37027115af7fe2 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 29 Jan 2019 11:10:06 -0300 Subject: [PATCH 31/67] add FEED_STORAGE_S3_ACL setting --- scrapy/extensions/feedexport.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..eb0802261 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -118,6 +118,7 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" + self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private') if self.is_botocore: import botocore.session session = botocore.session.get_session() @@ -137,12 +138,13 @@ class S3FeedStorage(BlockingFeedStorage): file.seek(0) if self.is_botocore: self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file) + Bucket=self.bucketname, Key=self.keyname, Body=file, + ACL=self.policy) else: conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - key.set_contents_from_file(file) + key.set_contents_from_file(file, policy=self.policy) key.close() From ad83ffdf1f4d69ffb62b243429e7b59d0930524c Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:32:46 -0200 Subject: [PATCH 32/67] refactoring --- scrapy/extensions/feedexport.py | 19 ++++++-- tests/test_feedexport.py | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index eb0802261..ca30322be 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -93,7 +93,7 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None): # BEGIN Backwards compatibility for initialising without keys (and # without using from_crawler) no_defaults = access_key is None and secret_key is None @@ -118,7 +118,7 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" - self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private') + self.acl = acl if self.is_botocore: import botocore.session session = botocore.session.get_session() @@ -132,19 +132,28 @@ class S3FeedStorage(BlockingFeedStorage): @classmethod def from_crawler(cls, crawler, uri): return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], - crawler.settings['AWS_SECRET_ACCESS_KEY']) + crawler.settings['AWS_SECRET_ACCESS_KEY'], + crawler.settings.get('FEED_STORAGE_S3_ACL')) def _store_in_thread(self, file): file.seek(0) if self.is_botocore: + kwargs = dict() + if self.acl: + kwargs.update(dict(ACL=self.acl)) + self.s3_client.put_object( Bucket=self.bucketname, Key=self.keyname, Body=file, - ACL=self.policy) + **kwargs) else: conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - key.set_contents_from_file(file, policy=self.policy) + kwargs = dict() + if self.acl: + kwargs.update(dict(policy=self.acl)) + + key.set_contents_from_file(file, **kwargs) key.close() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..b07635cb0 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,6 +18,7 @@ from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri +import botocore.client import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( @@ -186,6 +187,89 @@ class S3FeedStorageTest(unittest.TestCase): content = get_s3_content_and_delete(u.hostname, u.path[1:]) self.assertEqual(content, expected_content) + def test_init_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_init_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + def test_from_crawler_without_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_from_crawler_with_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_STORAGE_S3_ACL': 'custom-acl', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + def test_store_in_thread_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + storage._store_in_thread(BytesIO(b'test file')) + operation_name, api_params = _make_api_call_mock.call_args[0] + self.assertEqual(operation_name, 'PutObject') + self.assertNotIn('ACL', api_params) + + def test_store_in_thread_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + storage._store_in_thread(BytesIO(b'test file')) + operation_name, api_params = _make_api_call_mock.call_args[0] + self.assertEqual(operation_name, 'PutObject') + self.assertEqual(api_params.get('ACL'), 'custom-acl') + class StdoutFeedStorageTest(unittest.TestCase): From 126207fb7bca21d3d95ed9c66028e82771180370 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:38:17 -0200 Subject: [PATCH 33/67] PEP8: use short name for mock method --- tests/test_feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b07635cb0..bfac06efc 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -247,9 +247,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) - with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + with mock.patch('botocore.client.BaseClient._make_api_call') as m: storage._store_in_thread(BytesIO(b'test file')) - operation_name, api_params = _make_api_call_mock.call_args[0] + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) @@ -264,9 +264,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + with mock.patch('botocore.client.BaseClient._make_api_call') as m: storage._store_in_thread(BytesIO(b'test file')) - operation_name, api_params = _make_api_call_mock.call_args[0] + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') From e0f34be383e361c75b22da59c97dee1db189937e Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:50:19 -0200 Subject: [PATCH 34/67] update docs --- docs/topics/feed-exports.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b64dbfbfd..661751ed9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,6 +185,10 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` +You can also define a custom ACL for exported objects using this setting: + + * :setting:`FEED_STORAGE_S3_ACL` + .. _topics-feed-storage-stdout: Standard output @@ -205,6 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_URI` (mandatory) * :setting:`FEED_FORMAT` * :setting:`FEED_STORAGES` + * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` @@ -302,11 +307,22 @@ Default: ``{}`` A dict containing additional feed storage backends supported by your project. The keys are URI schemes and the values are paths to storage classes. +.. setting:: FEED_STORAGE_S3_ACL + +FEED_STORAGE_S3_ACL +------------------- + +Default: ``None`` + +A string containing a custom ACL for feeds exported to Amazon S3 by your project. + +For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs. + .. setting:: FEED_STORAGES_BASE FEED_STORAGES_BASE ------------------ - +` Default:: { @@ -366,3 +382,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore +.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl From 7b83ed7c5e1fcd81baf50db3a76f10ade7aa226e Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:52:24 -0200 Subject: [PATCH 35/67] remove typo --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 661751ed9..25979dfef 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -322,7 +322,7 @@ For a complete list of available values, access the `Canned ACL`_ section on Ama FEED_STORAGES_BASE ------------------ -` + Default:: { From e25b9a2323c169a4032ff07f912299b32de4b2e0 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:52:39 -0200 Subject: [PATCH 36/67] calling it feeds instead of objects --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 25979dfef..dee0c3ffa 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,7 +185,7 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` -You can also define a custom ACL for exported objects using this setting: +You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` From dbeb088eea1713ac43f3d23579c36ece5f67563f Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 7 Feb 2019 09:29:16 -0200 Subject: [PATCH 37/67] trying to fix jessie testenv by adding botocore to requirements and fixing its version --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 0c0f8f7b7..f2f3e1293 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,7 @@ deps = lxml==3.4.0 Twisted==14.0.2 boto==2.34.0 + botocore==1.12.89 Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 From 079af889e7d010a79640e6874c3e6dc394b936ae Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 7 Feb 2019 10:42:59 -0200 Subject: [PATCH 38/67] also testing without botocore --- tests/test_feedexport.py | 53 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index bfac06efc..520ca4a8f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -237,7 +237,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - def test_store_in_thread_without_acl(self): + def test_store_in_thread_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -253,7 +253,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) - def test_store_in_thread_with_acl(self): + def test_store_in_thread_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -270,6 +270,55 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') + def test_store_in_thread_not_botocore_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + storage._store_in_thread(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertNotIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + + def test_store_in_thread_not_botocore_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + storage._store_in_thread(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + class StdoutFeedStorageTest(unittest.TestCase): From ceae356e62dc2e56465a58b3d9fe00813e289dd6 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 11:47:35 -0200 Subject: [PATCH 39/67] add FEED_STORAGE_S3_ACL to default_settings.py file --- scrapy/settings/default_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3734a0a58..776c5af23 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -158,6 +158,8 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_S3_ACL = None + FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' From cfd183a9d19563f09487af942c9a635d665a1905 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 14:49:26 -0200 Subject: [PATCH 40/67] no need to use get here since we're defining a default value in default_settings.py --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ca30322be..2b4594ad8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -133,7 +133,7 @@ class S3FeedStorage(BlockingFeedStorage): def from_crawler(cls, crawler, uri): return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], crawler.settings['AWS_SECRET_ACCESS_KEY'], - crawler.settings.get('FEED_STORAGE_S3_ACL')) + crawler.settings['FEED_STORAGE_S3_ACL']) def _store_in_thread(self, file): file.seek(0) From f824f5b2d17b082dac04505ba27afdfa869a11c7 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 15:19:57 -0200 Subject: [PATCH 41/67] testing public method store instead of private method _store_in_thread need to mock deferToThread function --- tests/test_feedexport.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 520ca4a8f..e8c32ea43 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -237,7 +237,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - def test_store_in_thread_botocore_without_acl(self): + def test_store_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -247,13 +247,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + with mock.patch('botocore.client.BaseClient._make_api_call') as m: - storage._store_in_thread(BytesIO(b'test file')) + with mock.patch('twisted.internet.threads.deferToThread', + new=_defer): + storage.store(BytesIO(b'test file')) + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) - def test_store_in_thread_botocore_with_acl(self): + def test_store_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -264,13 +270,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + with mock.patch('botocore.client.BaseClient._make_api_call') as m: - storage._store_in_thread(BytesIO(b'test file')) + with mock.patch('twisted.internet.threads.deferToThread', + new=_defer): + storage.store(BytesIO(b'test file')) + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') - def test_store_in_thread_not_botocore_without_acl(self): + def test_store_not_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -284,7 +296,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - storage._store_in_thread(BytesIO(b'test file')) + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + + with mock.patch('twisted.internet.threads.deferToThread', new=_defer): + storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) @@ -294,7 +310,7 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) - def test_store_in_thread_not_botocore_with_acl(self): + def test_store_not_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -309,7 +325,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - storage._store_in_thread(BytesIO(b'test file')) + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + + with mock.patch('twisted.internet.threads.deferToThread', new=_defer): + storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) From 1eac2a163c2c734594d4f1e7e026eab309b2b0b5 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 16:50:39 -0200 Subject: [PATCH 42/67] simplifying how we deal with threads.deferToThread calls --- tests/test_feedexport.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e8c32ea43..0f31ef00e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -237,6 +237,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -247,18 +248,14 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - with mock.patch('twisted.internet.threads.deferToThread', - new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) + @defer.inlineCallbacks def test_store_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -270,18 +267,14 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - with mock.patch('twisted.internet.threads.deferToThread', - new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') + @defer.inlineCallbacks def test_store_not_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -296,11 +289,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - - with mock.patch('twisted.internet.threads.deferToThread', new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) @@ -310,6 +299,7 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) + @defer.inlineCallbacks def test_store_not_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -325,11 +315,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - - with mock.patch('twisted.internet.threads.deferToThread', new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) From 03e61b9908733f085d87da6bd29152389961b81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Feb 2019 13:50:01 +0100 Subject: [PATCH 43/67] Check that spidercls arguments in scrapy.crawler classes are not spider objects --- scrapy/crawler.py | 13 +++++++++++++ tests/test_crawler.py | 13 +++++++++++++ tests/test_downloadermiddleware_httpproxy.py | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04aee18ed..ee00d27b4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -7,6 +7,7 @@ import sys from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement +from scrapy import Spider from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader @@ -27,6 +28,10 @@ logger = logging.getLogger(__name__) class Crawler(object): def __init__(self, spidercls, settings=None): + if isinstance(spidercls, Spider): + raise ValueError( + 'The spidercls argument must be a class, not an object') + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -168,6 +173,10 @@ class CrawlerRunner(object): :param dict kwargs: keyword arguments to initialize the spider """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + 'The crawler_or_spidercls argument cannot be a spider object, ' + 'it must be a spider class (or a Crawler object)') crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) @@ -195,6 +204,10 @@ class CrawlerRunner(object): a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + 'The crawler_or_spidercls argument cannot be a spider object, ' + 'it must be a spider class (or a Crawler object)') if isinstance(crawler_or_spidercls, Crawler): return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 268948a70..37cea3ad3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import warnings from twisted.internet import defer from twisted.trial import unittest +from pytest import raises import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess @@ -66,6 +67,10 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') + def test_crawler_rejects_spider_objects(self): + with raises(ValueError): + Crawler(DefaultSpider()) + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -177,6 +182,14 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertEqual(len(w), 1) self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message)) + def test_crawl_rejects_spider_objects(self): + with raises(ValueError): + CrawlerRunner().crawl(DefaultSpider()) + + def test_create_crawler_rejects_spider_objects(self): + with raises(ValueError): + CrawlerRunner().create_crawler(DefaultSpider()) + class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 537126613..30920b2da 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -25,7 +25,7 @@ class TestHttpProxyMiddleware(TestCase): def test_not_enabled(self): settings = Settings({'HTTPPROXY_ENABLED': False}) - crawler = Crawler(spider, settings) + crawler = Crawler(Spider, settings) self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) def test_no_environment_proxies(self): From 7c9f0bd86c5f02ea803fa6bf1242d34d9c9f47d5 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 12 Feb 2019 12:19:30 -0200 Subject: [PATCH 44/67] using named params with optional amazon s3 params --- scrapy/extensions/feedexport.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2b4594ad8..f6bc460ea 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -131,9 +131,12 @@ class S3FeedStorage(BlockingFeedStorage): @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], - crawler.settings['AWS_SECRET_ACCESS_KEY'], - crawler.settings['FEED_STORAGE_S3_ACL']) + return cls( + uri=uri, + access_key=crawler.settings['AWS_ACCESS_KEY_ID'], + secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + acl=crawler.settings['FEED_STORAGE_S3_ACL'] + ) def _store_in_thread(self, file): file.seek(0) From c2dede27bd56bd783c45fb7302ca06b7c2c025c0 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 12 Feb 2019 12:22:05 -0200 Subject: [PATCH 45/67] reduce code with simple ternary operator --- scrapy/extensions/feedexport.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index f6bc460ea..40f985f19 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -141,10 +141,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) if self.is_botocore: - kwargs = dict() - if self.acl: - kwargs.update(dict(ACL=self.acl)) - + kwargs = {'ACL': self.acl} if self.acl else {} self.s3_client.put_object( Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs) @@ -152,10 +149,7 @@ class S3FeedStorage(BlockingFeedStorage): conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - kwargs = dict() - if self.acl: - kwargs.update(dict(policy=self.acl)) - + kwargs = {'policy': self.acl} if self.acl else {} key.set_contents_from_file(file, **kwargs) key.close() From 984e706fd2e06457bcdd1226366d08950ac101b0 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 12 Feb 2019 12:26:57 -0200 Subject: [PATCH 46/67] using blank string instead of None as default value as proposed by @kmike --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dee0c3ffa..cf70b8aca 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -312,7 +312,7 @@ The keys are URI schemes and the values are paths to storage classes. FEED_STORAGE_S3_ACL ------------------- -Default: ``None`` +Default: ``''`` (empty string) A string containing a custom ACL for feeds exported to Amazon S3 by your project. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 40f985f19..975fa1229 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -135,7 +135,7 @@ class S3FeedStorage(BlockingFeedStorage): uri=uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None ) def _store_in_thread(self, file): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 776c5af23..a800d39ab 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -158,7 +158,7 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 -FEED_STORAGE_S3_ACL = None +FEED_STORAGE_S3_ACL = '' FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' From 04ccf79e38561a2175c18440b3b4a53ba2f4992f Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Wed, 13 Feb 2019 15:39:45 +0000 Subject: [PATCH 47/67] A different S3 Endpoint URL is now possible when uploading images --- scrapy/pipelines/images.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 95323c613..8338a6281 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -89,6 +89,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs'] From 50bf4c60c480a276651ff703cf9fed8e7f981d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Feb 2019 17:39:20 +0100 Subject: [PATCH 48/67] Document that the main entry point of downloader and spider middlewares is from_crawler() --- docs/topics/downloader-middleware.rst | 8 ++++++-- docs/topics/spider-middleware.rst | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..18a0639ce 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -55,8 +55,12 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each middleware component is a Python class that defines one or -more of the following methods: +Each downloader middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.downloadermiddlewares diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..62b5ca0e8 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -57,8 +57,12 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each middleware component is a Python class that defines one or more of the -following methods: +Each spider middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.spidermiddlewares From 430e9392483b3992c16bd0314f1bcaed91a9d392 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Wed, 13 Feb 2019 19:59:40 +0000 Subject: [PATCH 49/67] Added missing AWS Settings for ImagesPipeline --- scrapy/pipelines/images.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8338a6281..a1457c7e9 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -90,6 +90,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs'] From b4d132b9f0824263d83331d0b36870f6f64918e4 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 13 Feb 2019 19:21:14 -0200 Subject: [PATCH 50/67] setting botocore version as described in debian jessie website https://packages.debian.org/en/jessie/python-botocore --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f2f3e1293..584da2dcd 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ deps = lxml==3.4.0 Twisted==14.0.2 boto==2.34.0 - botocore==1.12.89 + botocore==0.62 Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 From dc0b643832e9f3400e432c2ef7a34e6c75ac8366 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 13 Feb 2019 19:44:50 -0200 Subject: [PATCH 51/67] refactoring tests to avoid mocking private method --- tests/test_feedexport.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 0f31ef00e..c103593f9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -248,12 +248,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - yield storage.store(BytesIO(b'test file')) - - operation_name, api_params = m.call_args[0] - self.assertEqual(operation_name, 'PutObject') - self.assertNotIn('ACL', api_params) + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1]) @defer.inlineCallbacks def test_store_botocore_with_acl(self): @@ -267,12 +264,12 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - yield storage.store(BytesIO(b'test file')) - - operation_name, api_params = m.call_args[0] - self.assertEqual(operation_name, 'PutObject') - self.assertEqual(api_params.get('ACL'), 'custom-acl') + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertEqual( + storage.s3_client.put_object.call_args[1].get('ACL'), + 'custom-acl' + ) @defer.inlineCallbacks def test_store_not_botocore_without_acl(self): From ea8be627d15aa6fe1beaf50fff666cbeb161d94d Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 13 Feb 2019 19:53:10 -0200 Subject: [PATCH 52/67] botocore is not supported on debian jessie --- tests/test_feedexport.py | 7 ++++++- tox.ini | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c103593f9..2bf57e278 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,7 +18,6 @@ from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri -import botocore.client import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( @@ -239,6 +238,9 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): + if os.getenv('TOX_ENV_NAME') == 'jessie': + raise unittest.SkipTest('botocore is not supported on jessie') + storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -254,6 +256,9 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): + if os.getenv('TOX_ENV_NAME') == 'jessie': + raise unittest.SkipTest('botocore is not supported on jessie') + storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', diff --git a/tox.ini b/tox.ini index 584da2dcd..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,6 @@ deps = lxml==3.4.0 Twisted==14.0.2 boto==2.34.0 - botocore==0.62 Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 From 9b8ba4c383df0f3029d1b07ab9647a7d902600f4 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 14 Feb 2019 16:20:56 -0200 Subject: [PATCH 53/67] try to import botocore before runing some tests --- tests/test_feedexport.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2bf57e278..3ff79c912 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -238,8 +238,10 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): - if os.getenv('TOX_ENV_NAME') == 'jessie': - raise unittest.SkipTest('botocore is not supported on jessie') + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -256,8 +258,10 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): - if os.getenv('TOX_ENV_NAME') == 'jessie': - raise unittest.SkipTest('botocore is not supported on jessie') + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') storage = S3FeedStorage( 's3://mybucket/export.csv', From b02d26fae8892775ad6ef306d80b02e6bc69d12e Mon Sep 17 00:00:00 2001 From: John de la Garza Date: Fri, 15 Feb 2019 16:54:19 -0800 Subject: [PATCH 55/67] rel_has_nofollow: remove redundant if statement --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ccfdcd72..6de36d45c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -116,7 +116,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return True if rel is not None and 'nofollow' in rel.split() else False + return rel is not None and 'nofollow' in rel.split() def create_instance(objcls, settings, crawler, *args, **kwargs): From e3b15252c80ca3d0872f3068c382a0a3e7cc9db6 Mon Sep 17 00:00:00 2001 From: Matthieu Grandrie Date: Thu, 21 Feb 2019 17:19:58 +0100 Subject: [PATCH 56/67] New constructor arg *restrict_text* for FilteringLinkExtractor. Same as allow and deny args, it holds a string, a regex or an iterable of. Links whose text don't match one of the regex are filtered out. DOC restrict_text in LxmlLinkExtractor --- docs/topics/link-extractors.rst | 6 ++++++ scrapy/linkextractors/__init__.py | 6 +++++- scrapy/linkextractors/lxmlhtml.py | 9 +++++---- scrapy/linkextractors/sgml.py | 13 +++++++------ tests/test_linkextractors.py | 24 ++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f40a36d31..713a94e10 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -93,6 +93,12 @@ LxmlLinkExtractor Has the same behaviour as ``restrict_xpaths``. :type restrict_css: str or list + :param restrict_text: a single regular expression (or list of regular expressions) + that the link's text must match in order to be extracted. If not + given (or empty), it will match all links. If a list of regular expressions is + given, the link will be extracted if it matches at least one. + :type restrict_text: a regular expression (or list of) + :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. :type tags: str or list diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 97e8c0af1..ebf3cd7d8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -50,7 +50,7 @@ class FilteringLinkExtractor(object): _csstranslator = HTMLTranslator() def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css): + restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): self.link_extractor = link_extractor @@ -70,6 +70,8 @@ class FilteringLinkExtractor(object): if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] def _link_allowed(self, link): if not _is_valid_url(link.url): @@ -85,6 +87,8 @@ class FilteringLinkExtractor(object): return False if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False return True def matches(self, url): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index a7092f9b8..8f6f93a44 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -97,7 +97,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs @@ -111,9 +111,10 @@ class LxmlLinkExtractor(FilteringLinkExtractor): ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = get_base_url(response) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 5fa6b771c..8940a4d77 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -113,7 +113,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=()): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -127,13 +127,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip, - canonicalized=canonicalize) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = None diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 903032b52..c9cd629f4 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -479,6 +479,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_link_restrict_text(self): + html = b""" + Pic of a cat + Pic of a dog + Pic of a cow + """ + response = HtmlResponse("http://example.org/index.html", body=html) + # Simple text inclusion test + lx = self.extractor_cls(restrict_text='dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Unique regex test + lx = self.extractor_cls(restrict_text=r'of.*dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Multiple regex test + lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() From 858f5be74728209d8ef71794296814abca4c1c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2019 16:10:23 +0100 Subject: [PATCH 57/67] =?UTF-8?q?backwards=20=E2=86=92=20backward=20(adj.)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 64 ++++++++++++------------- docs/topics/request-response.rst | 2 +- docs/topics/spiders.rst | 2 +- docs/versioning.rst | 2 +- scrapy/cmdline.py | 4 +- scrapy/conf.py | 2 +- scrapy/core/downloader/handlers/http.py | 2 +- scrapy/extensions/feedexport.py | 4 +- scrapy/log.py | 2 +- scrapy/signals.py | 2 +- scrapy/utils/conf.py | 4 +- sep/sep-018.rst | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_feedexport.py | 2 +- tests/test_utils_conf.py | 2 +- 15 files changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 668473887..7ac1664fe 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -53,7 +53,7 @@ case of Scrapy spiders: callbacks are usually called several times, on different pages. If you're using custom ``Selector`` or ``SelectorList`` subclasses, -a **backwards incompatible** change in parsel may affect your code. +a **backward incompatible** change in parsel may affect your code. See `parsel changelog`_ for a detailed description, as well as for the full list of improvements. @@ -62,7 +62,7 @@ full list of improvements. Telnet console ~~~~~~~~~~~~~~ -**Backwards incompatible**: Scrapy's telnet console now requires username +**Backward incompatible**: Scrapy's telnet console now requires username and password. See :ref:`topics-telnetconsole` for more details. This change fixes a **security issue**; see :ref:`release-1.5.2` release notes for details. @@ -209,7 +209,7 @@ Scrapy 1.5.2 (2019-01-22) exploit it from Scrapy, but it is very easy to trick a browser to do so and elevates the risk for local development environment. - *The fix is backwards incompatible*, it enables telnet user-password + *The fix is backward incompatible*, it enables telnet user-password authentication by default with a random generated password. If you can't upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT` out of its default value. @@ -256,15 +256,15 @@ Some highlights: * Better default handling of HTTP 308, 522 and 524 status codes. * Documentation is improved, as usual. -Backwards Incompatible Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backward Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Scrapy 1.5 drops support for Python 3.3. * Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). - **This is technically backwards-incompatible**; override + **This is technically backward-incompatible**; override :setting:`USER_AGENT` if you relied on old value. * Logging of settings overridden by ``custom_settings`` is fixed; - **this is technically backwards-incompatible** because the logger + **this is technically backward-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're parsing Scrapy logs, please update your log parsers (:issue:`1343`). * LinkExtractor now ignores ``m4v`` extension by default, this is change @@ -301,11 +301,11 @@ Bug fixes ~~~~~~~~~ - Fix logging of settings overridden by ``custom_settings``; - **this is technically backwards-incompatible** because the logger + **this is technically backward-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please update your log parsers if needed (:issue:`1343`) - Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). - **This is technically backwards-incompatible**; override + **This is technically backward-incompatible**; override :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, @@ -415,18 +415,18 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting. Enjoy! (Or read on for the rest of changes in this release.) -Deprecations and Backwards Incompatible Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Deprecations and Backward Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): - **warning, this is technically backwards-incompatible** + **warning, this is technically backward-incompatible** - Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`); - **this is technically backwards-incompatible** so please check if you have + **this is technically backward-incompatible** so please check if you have any non-default ``MEMUSAGE_***`` options set. - ``EDITOR`` environment variable now takes precedence over ``EDITOR`` option defined in settings.py (:issue:`1829`); Scrapy default settings - no longer depend on environment variables. **This is technically a backwards + no longer depend on environment variables. **This is technically a backward incompatible change**. - ``Spider.make_requests_from_url`` is deprecated (:issue:`1728`, fixes :issue:`1495`). @@ -636,10 +636,10 @@ New Features scrapy shell now follow HTTP redirections by default (:issue:`2290`); See :command:`fetch` and :command:`shell` for details. - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; - this is technically **backwards incompatible** so please check your log parsers. + this is technically **backward incompatible** so please check your log parsers. - By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); - this is **backwards incompatible** if you have log parsers expecting the short + this is **backward incompatible** if you have log parsers expecting the short logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` set to ``True``. @@ -750,11 +750,11 @@ Bug fixes ~~~~~~~~~ - DefaultRequestHeaders middleware now runs before UserAgent middleware - (:issue:`2088`). **Warning: this is technically backwards incompatible**, + (:issue:`2088`). **Warning: this is technically backward incompatible**, though we consider this a bug fix. - HTTP cache extension and plugins that use the ``.scrapy`` data directory now work outside projects (:issue:`1581`). **Warning: this is technically - backwards incompatible**, though we consider this a bug fix. + backward incompatible**, though we consider this a bug fix. - ``Selector`` does not allow passing both ``response`` and ``text`` anymore (:issue:`2153`). - Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`). @@ -934,13 +934,13 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Accept XML node names containing dots as valid (:issue:`1533`). - When uploading files or images to S3 (with ``FilesPipeline`` or ``ImagesPipeline``), the default ACL policy is now "private" instead - of "public" **Warning: backwards incompatible!**. + of "public" **Warning: backward incompatible!**. You can use :setting:`FILES_STORE_S3_ACL` to change it. - We've reimplemented ``canonicalize_url()`` for more correct output, especially for URLs with non-ASCII characters (:issue:`1947`). This could change link extractors output compared to previous scrapy versions. This may also invalidate some cache entries you could still have from pre-1.1 runs. - **Warning: backwards incompatible!**. + **Warning: backward incompatible!**. Keep reading for more details on other improvements and bug fixes. @@ -973,7 +973,7 @@ Additional New Features and Enhancements - Support for bpython and configure preferred Python shell via ``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`). - Support URLs without scheme (:issue:`1498`) - **Warning: backwards incompatible!** + **Warning: backward incompatible!** - Bring back support for relative file path (:issue:`1710`, :issue:`1550`). - Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check @@ -1056,7 +1056,7 @@ Bugfixes ~~~~~~~~ - Scrapy does not retry requests that got a ``HTTP 400 Bad Request`` - response anymore (:issue:`1289`). **Warning: backwards incompatible!** + response anymore (:issue:`1289`). **Warning: backward incompatible!** - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). @@ -1646,7 +1646,7 @@ Scrapy 0.24.2 (2014-07-08) Scrapy 0.24.1 (2014-06-27) -------------------------- -- Fix deprecated CrawlerSettings and increase backwards compatibility with +- Fix deprecated CrawlerSettings and increase backward compatibility with .defaults attribute (:commit:`8e3f20a`) @@ -1772,7 +1772,7 @@ Scrapy 0.22.0 (released 2014-01-17) Enhancements ~~~~~~~~~~~~ -- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) +- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) @@ -2092,7 +2092,7 @@ Scrapy 0.16.1 (released 2012-10-26) ----------------------------------- - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`) -- better backwards compatibility for scrapy.conf.settings (:commit:`3403089`) +- better backward compatibility for scrapy.conf.settings (:commit:`3403089`) - extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`) - removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`) - fix dashes under rst headers (:commit:`fa4f7f9`) @@ -2107,7 +2107,7 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` -- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backwards compatibility is kept on the Stats Collector API and signals. +- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. @@ -2259,7 +2259,7 @@ Code rearranged and removed - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) - removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`) - Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`) -- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept. +- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backward compatibility kept. Scrapy 0.12 ----------- @@ -2356,11 +2356,11 @@ API changes - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` - default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201) - changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - moved ``scrapy.core.signals`` module to ``scrapy.signals`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - added ``handles_request()`` class method to ``BaseSpider`` - dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead) - dropped ``component`` argument of ``scrapy.log.msg()`` function @@ -2431,8 +2431,8 @@ New features - Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`) - Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`) -Backwards-incompatible changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`) - Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 76360b15f..4511f3469 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -728,7 +728,7 @@ TextResponse objects .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is - kept for backwards compatibility; please prefer ``response.text``. + kept for backward compatibility; please prefer ``response.text``. HtmlResponse objects diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..e1d36aa24 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -190,7 +190,7 @@ scrapy.Spider .. method:: log(message, [level, component]) Wrapper that sends a log message through the Spider's :attr:`logger`, - kept for backwards compatibility. For more information see + kept for backward compatibility. For more information see :ref:`topics-logging-from-spiders`. .. method:: closed(reason) diff --git a/docs/versioning.rst b/docs/versioning.rst index 0421ba544..227085f02 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very large changes. * *B* is the release number. This will include many changes including features - and things that possibly break backwards compatibility, although we strive to + and things that possibly break backward compatibility, although we strive to keep theses cases at a minimum. * *C* is the bugfix release number. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index dc6b59fe0..fa2506eb0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -99,7 +99,7 @@ def execute(argv=None, settings=None): if argv is None: argv = sys.argv - # --- backwards compatibility for scrapy.conf.settings singleton --- + # --- backward compatibility for scrapy.conf.settings singleton --- if settings is None and 'scrapy.conf' in sys.modules: from scrapy import conf if hasattr(conf, 'settings'): @@ -116,7 +116,7 @@ def execute(argv=None, settings=None): settings['EDITOR'] = editor check_deprecated_settings(settings) - # --- backwards compatibility for scrapy.conf.settings singleton --- + # --- backward compatibility for scrapy.conf.settings singleton --- import warnings from scrapy.exceptions import ScrapyDeprecationWarning with warnings.catch_warnings(): diff --git a/scrapy/conf.py b/scrapy/conf.py index 23efc6ffd..6c40edcdd 100644 --- a/scrapy/conf.py +++ b/scrapy/conf.py @@ -1,4 +1,4 @@ -# This module is kept for backwards compatibility, so users can import +# This module is kept for backward compatibility, so users can import # scrapy.conf.settings and get the settings they expect import sys diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index e4a7d8564..e76823623 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -3,7 +3,7 @@ from .http10 import HTTP10DownloadHandler from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler -# backwards compatibility +# backward compatibility class HttpDownloadHandler(HTTP10DownloadHandler): def __init__(self, *args, **kwargs): diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..3b4d809e8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,7 +94,7 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None): - # BEGIN Backwards compatibility for initialising without keys (and + # BEGIN Backward compatibility for initialising without keys (and # without using from_crawler) no_defaults = access_key is None and secret_key is None if no_defaults: @@ -111,7 +111,7 @@ class S3FeedStorage(BlockingFeedStorage): ) access_key = settings['AWS_ACCESS_KEY_ID'] secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backwards compatibility + # END Backward compatibility u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/scrapy/log.py b/scrapy/log.py index 719fceaad..777bd6dc4 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -17,7 +17,7 @@ warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on " ScrapyDeprecationWarning, stacklevel=2) -# Imports and level_names variable kept for backwards-compatibility +# Imports and level_names variable kept for backward-compatibility DEBUG = logging.DEBUG INFO = logging.INFO diff --git a/scrapy/signals.py b/scrapy/signals.py index c0e4bb74e..6b9125302 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -20,7 +20,7 @@ item_scraped = object() item_dropped = object() item_error = object() -# for backwards compatibility +# for backward compatibility stats_spider_opened = spider_opened stats_spider_closing = spider_closed stats_spider_closed = spider_closed diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 435e9a6b3..fbd297340 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -42,14 +42,14 @@ def build_component_list(compdict, custom=None, convert=update_classpath): raise ValueError('Invalid value {} for component {}, please provide ' \ 'a real number or None instead'.format(value, name)) - # BEGIN Backwards compatibility for old (base, custom) call signature + # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) return type(custom)(convert(c) for c in custom) if custom is not None: compdict.update(custom) - # END Backwards compatibility + # END Backward compatibility _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index aca7ac342..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -211,7 +211,7 @@ spider methods on each event such as: - call additional spider middlewares defined in the ``Spider.middlewares`` attribute - call ``Spider.next_request()`` and ``Spider.start_requests()`` on - ``next_request()`` middleware method (this would implicitly support backwards + ``next_request()`` middleware method (this would implicitly support backward compatibility) Differences with Spider middleware v1 diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0d0829793..81235a16f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -50,7 +50,7 @@ class DummyDH(object): class DummyLazyDH(object): - # Default is lazy for backwards compatibility + # Default is lazy for backward compatibility def __init__(self, crawler): pass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..b254b9f38 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -161,7 +161,7 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backwards compatibility for initialising without settings + # Backward compatibility for initialising without settings with warnings.catch_warnings(record=True) as w: storage = S3FeedStorage('s3://mybucket/export.csv') self.assertEqual(storage.access_key, 'conf_key') diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f203c32ef..29937c189 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -11,7 +11,7 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ['one', 'four', 'three']) - def test_backwards_compatible_build_dict(self): + def test_backward_compatible_build_dict(self): base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} custom = {'two': None, 'three': 8, 'four': 4} self.assertEqual(build_component_list(base, custom, From 75d6f56c8a731ea4e1c06814a59a0b51741d04a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2019 16:56:58 +0100 Subject: [PATCH 58/67] Switch from ` to `` where inline code formatting is desired --- docs/contributing.rst | 2 +- docs/news.rst | 130 +++++++++++++------------- docs/topics/api.rst | 4 +- docs/topics/downloader-middleware.rst | 40 ++++---- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 4 +- docs/topics/jobs.rst | 2 +- docs/topics/loaders.rst | 2 +- docs/topics/logging.rst | 2 +- docs/topics/media-pipeline.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/request-response.rst | 8 +- docs/topics/selectors.rst | 2 +- docs/topics/settings.rst | 4 +- docs/topics/spider-middleware.rst | 4 +- docs/topics/spiders.rst | 6 +- docs/topics/ubuntu.rst | 4 +- scrapy/crawler.py | 10 +- scrapy/logformatter.py | 18 ++-- scrapy/pipelines/files.py | 6 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/python.py | 18 ++-- scrapy/utils/url.py | 8 +- sep/sep-006.rst | 5 +- tests/mocks/dummydbm.py | 2 +- tests/test_command_shell.py | 4 +- 27 files changed, 148 insertions(+), 147 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index cf27337c8..9b508e418 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -55,7 +55,7 @@ guidelines when you're going to report a new bug. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has been discussed there, or if you're not sure if what you're seeing is a bug. - You can also ask in the `#scrapy` IRC channel. + You can also ask in the ``#scrapy`` IRC channel. * write **complete, reproducible, specific bug reports**. The smaller the test case, the better. Remember that other developers won't have your project to diff --git a/docs/news.rst b/docs/news.rst index 668473887..1849a3ca8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -149,7 +149,7 @@ Documentation improvements * improved links to beginner resources in the tutorial (:issue:`3367`, :issue:`3468`); * fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); -* remove unused `DEPTH_STATS` option from docs (:issue:`3245`); +* remove unused ``DEPTH_STATS`` option from docs (:issue:`3245`); * other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`, :issue:`3605`). @@ -1313,7 +1313,7 @@ Module Relocations There’s been a large rearrangement of modules trying to improve the general structure of Scrapy. Main changes were separating various subpackages into -new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp` +new projects and dissolving both ``scrapy.contrib`` and ``scrapy.contrib_exp`` into top level packages. Backward compatibility was kept among internal relocations, while importing deprecated modules expect warnings indicating their new place. @@ -1344,7 +1344,7 @@ Outsourced packages | | /scrapy-plugins/scrapy-jsonrpc>`_ | +-------------------------------------+-------------------------------------+ -`scrapy.contrib_exp` and `scrapy.contrib` dissolutions +``scrapy.contrib_exp`` and ``scrapy.contrib`` dissolutions +-------------------------------------+-------------------------------------+ | Old location | New location | @@ -1556,7 +1556,7 @@ Code refactoring (:issue:`1078`) - Pydispatch pep8 (:issue:`992`) - Removed unused 'load=False' parameter from walk_modules() (:issue:`871`) -- For consistency, use `job_dir` helper in `SpiderState` extension. +- For consistency, use ``job_dir`` helper in ``SpiderState`` extension. (:issue:`805`) - rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`) @@ -1669,10 +1669,10 @@ Enhancements cache middleware (:issue:`541`, :issue:`500`, :issue:`571`) - Expose current crawler in Scrapy shell (:issue:`557`) - Improve testsuite comparing CSV and XML exporters (:issue:`570`) -- New `offsite/filtered` and `offsite/domains` stats (:issue:`566`) +- New ``offsite/filtered`` and ``offsite/domains`` stats (:issue:`566`) - Support process_links as generator in CrawlSpider (:issue:`555`) - Verbose logging and new stats counters for DupeFilter (:issue:`553`) -- Add a mimetype parameter to `MailSender.send()` (:issue:`602`) +- Add a mimetype parameter to ``MailSender.send()`` (:issue:`602`) - Generalize file pipeline log messages (:issue:`622`) - Replace unencodeable codepoints with html entities in SGMLLinkExtractor (:issue:`565`) - Converted SEP documents to rst format (:issue:`629`, :issue:`630`, @@ -1691,20 +1691,20 @@ Enhancements - Make scrapy.version_info a tuple of integers (:issue:`681`, :issue:`692`) - Infer exporter's output format from filename extensions (:issue:`546`, :issue:`659`, :issue:`760`) -- Support case-insensitive domains in `url_is_from_any_domain()` (:issue:`693`) +- Support case-insensitive domains in ``url_is_from_any_domain()`` (:issue:`693`) - Remove pep8 warnings in project and spider templates (:issue:`698`) -- Tests and docs for `request_fingerprint` function (:issue:`597`) -- Update SEP-19 for GSoC project `per-spider settings` (:issue:`705`) +- Tests and docs for ``request_fingerprint`` function (:issue:`597`) +- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`) - Set exit code to non-zero when contracts fails (:issue:`727`) - Add a setting to control what class is instanciated as Downloader component (:issue:`738`) -- Pass response in `item_dropped` signal (:issue:`724`) -- Improve `scrapy check` contracts command (:issue:`733`, :issue:`752`) -- Document `spider.closed()` shortcut (:issue:`719`) -- Document `request_scheduled` signal (:issue:`746`) +- Pass response in ``item_dropped`` signal (:issue:`724`) +- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`) +- Document ``spider.closed()`` shortcut (:issue:`719`) +- Document ``request_scheduled`` signal (:issue:`746`) - Add a note about reporting security issues (:issue:`697`) - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`) -- Sort spider list output of `scrapy list` command (:issue:`742`) +- Sort spider list output of ``scrapy list`` command (:issue:`742`) - Multiple documentation enhancemens and fixes (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`, :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`, @@ -1772,23 +1772,23 @@ Scrapy 0.22.0 (released 2014-01-17) Enhancements ~~~~~~~~~~~~ -- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) - To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage` +- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) + To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage`` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`) - Selectors register EXSLT namespaces by default (:issue:`472`) - Unify item loaders similar to selectors renaming (:issue:`461`) -- Make `RFPDupeFilter` class easily subclassable (:issue:`533`) +- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in `get_func_args` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) - Allow running indiviual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) - Improve offsite middleware tests (:issue:`478`) - Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`) -- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`) +- Do not send ``x-gzip`` in default ``Accept-Encoding`` header (:issue:`469`) - Support defining http error handling using settings (:issue:`466`) - Use modern python idioms wherever you find legacies (:issue:`497`) - Improve and correct documentation @@ -1799,14 +1799,14 @@ Fixes ~~~~~ - Update Selector class imports in CrawlSpider template (:issue:`484`) -- Fix unexistent reference to `engine.slots` (:issue:`464`) -- Do not try to call `body_as_unicode()` on a non-TextResponse instance (:issue:`462`) +- Fix unexistent reference to ``engine.slots`` (:issue:`464`) +- Do not try to call ``body_as_unicode()`` on a non-TextResponse instance (:issue:`462`) - Warn when subclassing XPathItemLoader, previously it only warned on instantiation. (:issue:`523`) - Warn when subclassing XPathSelector, previously it only warned on instantiation. (:issue:`537`) - Multiple fixes to memory stats (:issue:`531`, :issue:`530`, :issue:`529`) -- Fix overriding url in `FormRequest.from_response()` (:issue:`507`) +- Fix overriding url in ``FormRequest.from_response()`` (:issue:`507`) - Fix tests runner under pip 1.5 (:issue:`513`) - Fix logging error when spider name is unicode (:issue:`479`) @@ -1833,7 +1833,7 @@ Enhancements (modifying them had been deprecated for a long time) - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) - Sitemap spider can fetch alternate URLs (:issue:`360`) -- `Selector.remove_namespaces()` now remove namespaces from element's attributes. (:issue:`416`) +- ``Selector.remove_namespaces()`` now remove namespaces from element's attributes. (:issue:`416`) - Paved the road for Python 3.3+ (:issue:`435`, :issue:`436`, :issue:`431`, :issue:`452`) - New item exporter using native python types with nesting support (:issue:`366`) - Tune HTTP1.1 pool size so it matches concurrency defined by settings (:commit:`b43b5f575`) @@ -1844,13 +1844,13 @@ Enhancements - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`) - Remove multi spider support from multiple core components (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`) -- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages. +- Travis-CI now tests Scrapy changes against development versions of ``w3lib`` and ``queuelib`` python packages. - Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`) - Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`) - Use importlib for parametric imports (:issue:`445`) - Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`) - Bugfix crawling shutdown on SIGINT (:issue:`450`) -- Do not submit `reset` type inputs in FormRequest.from_response (:commit:`b326b87`) +- Do not submit ``reset`` type inputs in FormRequest.from_response (:commit:`b326b87`) - Do not silence download errors when request errback raises an exception (:commit:`684cfc0`) Bugfixes @@ -1865,8 +1865,8 @@ Bugfixes - Improve request-response docs (:issue:`391`) - Improve best practices docs (:issue:`399`, :issue:`400`, :issue:`401`, :issue:`402`) - Improve django integration docs (:issue:`404`) -- Document `bindaddress` request meta (:commit:`37c24e01d7`) -- Improve `Request` class documentation (:issue:`226`) +- Document ``bindaddress`` request meta (:commit:`37c24e01d7`) +- Improve ``Request`` class documentation (:issue:`226`) Other ~~~~~ @@ -1875,7 +1875,7 @@ Other - Add `cssselect`_ python package as install dependency - Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on. - Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support. -- Running test suite now requires `mock` python library (:issue:`390`) +- Running test suite now requires ``mock`` python library (:issue:`390`) Thanks @@ -1929,7 +1929,7 @@ Scrapy 0.18.3 (released 2013-10-03) Scrapy 0.18.2 (released 2013-09-03) ----------------------------------- -- Backport `scrapy check` command fixes and backward compatible multi +- Backport ``scrapy check`` command fixes and backward compatible multi crawler process(:issue:`339`) Scrapy 0.18.1 (released 2013-08-27) @@ -1958,31 +1958,31 @@ Scrapy 0.18.0 (released 2013-08-09) - Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) - Use lxml recover option to parse sitemaps (:issue:`347`) - Bugfix cookie merging by hostname and not by netloc (:issue:`352`) -- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`) -- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`) -- Support `dont_cache` request meta flag (:issue:`19`) -- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`) -- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`) -- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`) +- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`) +- Support xml namespaces using ``iternodes`` parser in ``XMLFeedSpider`` (:issue:`12`) +- Support ``dont_cache`` request meta flag (:issue:`19`) +- Bugfix ``scrapy.utils.gz.gunzip`` broken by changes in python 2.7.4 (:commit:`4dc76e`) +- Bugfix url encoding on ``SgmlLinkExtractor`` (:issue:`24`) +- Bugfix ``TakeFirst`` processor shouldn't discard zero (0) value (:issue:`59`) - Support nested items in xml exporter (:issue:`66`) - Improve cookies handling performance (:issue:`77`) - Log dupe filtered requests once (:issue:`105`) - Split redirection middleware into status and meta based middlewares (:issue:`78`) - Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`) -- Support xpath form selection on `FormRequest.from_response` (:issue:`185`) -- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`) +- Support xpath form selection on ``FormRequest.from_response`` (:issue:`185`) +- Bugfix unicode decoding error on ``SgmlLinkExtractor`` (:issue:`199`) - Bugfix signal dispatching on pypi interpreter (:issue:`205`) - Improve request delay and concurrency handling (:issue:`206`) -- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`) +- Add RFC2616 cache policy to ``HttpCacheMiddleware`` (:issue:`212`) - Allow customization of messages logged by engine (:issue:`214`) -- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`) +- Multiples improvements to ``DjangoItem`` (:issue:`217`, :issue:`218`, :issue:`221`) - Extend Scrapy commands using setuptools entry points (:issue:`260`) -- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`) -- Support `settings.getdict` (:issue:`269`) -- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`) -- Added `Item.copy` (:issue:`290`) +- Allow spider ``allowed_domains`` value to be set/tuple (:issue:`261`) +- Support ``settings.getdict`` (:issue:`269`) +- Simplify internal ``scrapy.core.scraper`` slot handling (:issue:`271`) +- Added ``Item.copy`` (:issue:`290`) - Collect idle downloader slots (:issue:`297`) -- Add `ftp://` scheme downloader handler (:issue:`329`) +- Add ``ftp://`` scheme downloader handler (:issue:`329`) - Added downloader benchmark webserver and spider tools :ref:`benchmarking` - Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on - Add scrapy commands using external libraries (:issue:`260`) @@ -2113,7 +2113,7 @@ Scrapy changes: - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` -- `lxml` is now the default selectors backend instead of `libxml2` +- ``lxml`` is now the default selectors backend instead of ``libxml2`` - ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_ - removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm`` - SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`) @@ -2206,16 +2206,16 @@ New features and settings - New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`) - Add boto 2.0 support for S3 downloader handler (:rev:`2763`) - Added `marshal`_ to formats supported by feed exports (:rev:`2744`) -- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`) +- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) -- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`) -- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`) -- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) +- Moved spider queues to scrapyd: ``scrapy.spiderqueue`` -> ``scrapyd.spiderqueue`` (:rev:`2708`) +- Moved sqlite utils to scrapyd: ``scrapy.utils.sqlite`` -> ``scrapyd.sqlite`` (:rev:`2781`) +- Real support for returning iterators on ``start_requests()`` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) - Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`) - Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`) - Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`) @@ -2223,19 +2223,19 @@ New features and settings - Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`) - Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`) - Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`) -- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`. +- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an ``IOError``. - Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`) -- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`) +- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and ``-e`` flag to ``genspider`` command that uses it (:rev:`2653`) - Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines. - Added :signal:`spider_error` signal (:rev:`2628`) - Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`) -- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. +- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to ``True``). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. - Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`) - Added new DBM HTTP cache storage backend (:rev:`2576`) - Added ``listjobs.json`` API to Scrapyd (:rev:`2571`) - ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`) - Added namespace support to ``xmliter_lxml`` (:rev:`2552`) -- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`) +- Improved cookies middleware by making ``COOKIES_DEBUG`` nicer and documenting it (:rev:`2579`) - Several improvements to Scrapyd and Link extractors Code rearranged and removed @@ -2249,11 +2249,11 @@ Code rearranged and removed - Reduced Scrapy codebase by striping part of Scrapy code into two new libraries: - `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`) - `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`) -- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`) -- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot +- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`) +- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on github: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) @@ -2289,13 +2289,13 @@ Scrapyd changes - Scrapyd now uses one process per spider - It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default -- There is now a `scrapy server` command to start a Scrapyd server of the current project +- There is now a ``scrapy server`` command to start a Scrapyd server of the current project Changes to settings ~~~~~~~~~~~~~~~~~~~ -- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware -- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire". +- added ``HTTPCACHE_ENABLED`` setting (False by default) to enable HTTP cache middleware +- changed ``HTTPCACHE_EXPIRATION_SECS`` semantics: now zero means "never expire". Deprecated/obsoleted functionality ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2326,17 +2326,17 @@ New features and improvements - Splitted Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) -- Added `dont_redirect` request.meta key for avoiding redirects (#233) -- Added `dont_retry` request.meta key for avoiding retries (#234) +- Added ``dont_redirect`` request.meta key for avoiding redirects (#233) +- Added ``dont_retry`` request.meta key for avoiding retries (#234) Command-line tool changes ~~~~~~~~~~~~~~~~~~~~~~~~~ -- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199) - - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project - - Added `scrapy.bat` script for running more conveniently from Windows +- New ``scrapy`` command which replaces the old ``scrapy-ctl.py`` (#199) + - there is only one global ``scrapy`` command now, instead of one ``scrapy-ctl.py`` per project + - Added ``scrapy.bat`` script for running more conveniently from Windows - Added bash completion to command-line tool (#210) -- Renamed command `start` to `runserver` (#209) +- Renamed command ``start`` to ``runserver`` (#209) API changes ~~~~~~~~~~~ diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 985cc0433..ba832ab5d 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -94,7 +94,7 @@ how you :ref:`configure the downloader middlewares .. method:: crawl(\*args, \**kwargs) Starts the crawler by instantiating its spider class with the given - `args` and `kwargs` arguments, while setting the execution engine in + ``args`` and ``kwargs`` arguments, while setting the execution engine in motion. Returns a deferred that is fired when the crawl is finished. @@ -180,7 +180,7 @@ SpiderLoader API .. method:: load(spider_name) Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name `spider_name` and will raise + loaded spiders for a spider class with name ``spider_name`` and will raise a KeyError if not found. :param spider_name: spider class name diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..e6812eddd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it -in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { @@ -357,7 +357,7 @@ HttpCacheMiddleware .. reqmeta:: dont_cache - You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`. + You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``. .. _httpcache-policy-dummy: @@ -390,17 +390,17 @@ runs to avoid downloading unmodified data (to save bandwidth and speed up crawls what is implemented: -* Do not attempt to store responses/requests with `no-store` cache-control directive set -* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses -* Compute freshness lifetime from `max-age` cache-control directive -* Compute freshness lifetime from `Expires` response header -* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox) -* Compute current age from `Age` response header -* Compute current age from `Date` header -* Revalidate stale responses based on `Last-Modified` response header -* Revalidate stale responses based on `ETag` response header -* Set `Date` header for any received response missing it -* Support `max-stale` cache-control directive in requests +* Do not attempt to store responses/requests with ``no-store`` cache-control directive set +* Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses +* Compute freshness lifetime from ``max-age`` cache-control directive +* Compute freshness lifetime from ``Expires`` response header +* Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox) +* Compute current age from ``Age`` response header +* Compute current age from ``Date`` header +* Revalidate stale responses based on ``Last-Modified`` response header +* Revalidate stale responses based on ``ETag`` response header +* Set ``Date`` header for any received response missing it +* Support ``max-stale`` cache-control directive in requests This allows spiders to be configured with the full RFC2616 cache policy, but avoid revalidation on a request-by-request basis, while remaining @@ -408,15 +408,15 @@ what is implemented: Example: - Add `Cache-Control: max-stale=600` to Request headers to accept responses that + Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that have exceeded their expiration time by no more than 600 seconds. See also: RFC2616, 14.9.3 what is missing: -* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 -* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 +* ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 +* ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 * ... probably others .. @@ -626,12 +626,12 @@ Default: ``False`` If enabled, will cache pages unconditionally. A spider may wish to have all responses available in the cache, for -future use with `Cache-Control: max-stale`, for instance. The +future use with ``Cache-Control: max-stale``, for instance. The DummyPolicy caches all responses but never revalidates them, and sometimes a more nuanced policy is desirable. -This setting still respects `Cache-Control: no-store` directives in responses. -If you don't want that, filter `no-store` out of the Cache-Control headers in +This setting still respects ``Cache-Control: no-store`` directives in responses. +If you don't want that, filter ``no-store`` out of the Cache-Control headers in responses you feedto the cache middleware. .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS @@ -940,7 +940,7 @@ UserAgentMiddleware Middleware that allows spiders to override the default user agent. - In order for a spider to override the default user agent, its `user_agent` + In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. .. _ajaxcrawl-middleware: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 95f7920f8..f5048d2da 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -303,7 +303,7 @@ CsvItemExporter The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor, and the leftover arguments to the - `csv.writer`_ constructor, so you can use any `csv.writer` constructor + `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor argument to customize this exporter. A typical output of this exporter would be:: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index c421a5e05..d6e7452a1 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -19,7 +19,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like -`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. +``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps @@ -368,7 +368,7 @@ Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues running normally. -For more info see `Debugging in Python`. +For more info see `Debugging in Python`_. This extension only works on POSIX-compliant platforms (ie. not Windows). diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index ea684b4cf..1a5d52487 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -71,7 +71,7 @@ on cookies. Request serialization --------------------- -Requests must be serializable by the `pickle` module, in order for persistence +Requests must be serializable by the ``pickle`` module, in order for persistence to work, so you should make sure that your requests are serializable. The most common issue here is to use ``lambda`` functions on request callbacks that diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index f3b6aa4a1..1c2f1da4d 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -286,7 +286,7 @@ ItemLoader objects given, one is instantiated automatically using the class in :attr:`default_item_class`. - When instantiated with a `selector` or a `response` parameters + When instantiated with a ``selector`` or a ``response`` parameters the :class:`ItemLoader` class provides convenient mechanisms for extracting data from web pages using :ref:`selectors `. diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 0986929ad..8e280d929 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -243,7 +243,7 @@ scrapy.utils.log module case, its usage is not required but it's recommended. If you plan on configuring the handlers yourself is still recommended you - call this function, passing `install_root_handler=False`. Bear in mind + call this function, passing ``install_root_handler=False``. Bear in mind there won't be any log output set by default in that case. To get you started on manually configuring logging's output, you can use diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index c60b55391..381a2988a 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -132,7 +132,7 @@ For example, the following image URL:: http://www.example.com/image.jpg -Whose `SHA1 hash` is:: +Whose ``SHA1 hash`` is:: 3afec3b4765f8f0a07b78f98c07b83f013567a0a diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 02cfa9b05..298a078a7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -80,7 +80,7 @@ returned by the :meth:`CrawlerRunner.crawl ` method. Here's an example of its usage, along with a callback to manually stop the -reactor after `MySpider` has finished running. +reactor after ``MySpider`` has finished running. :: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 76360b15f..8b3ba4f2d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -50,7 +50,7 @@ Request objects :type meta: dict :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the `encoding` passed (which defaults to ``utf-8``). If + ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). @@ -610,7 +610,7 @@ Response objects .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for - tagging Responses. For example: `'cached'`, `'redirected`', etc. And + tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. @@ -682,7 +682,7 @@ TextResponse objects ``unicode(response.body)`` is not a correct way to convert response body to unicode: you would be using the system default encoding - (typically `ascii`) instead of the response encoding. + (typically ``ascii``) instead of the response encoding. .. attribute:: TextResponse.encoding @@ -690,7 +690,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the constructor `encoding` argument + 1. the encoding passed in the constructor ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (ie. unknown), it is ignored and the next diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index df1d67ae8..edc18f14d 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -96,7 +96,7 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of Using selectors --------------- -To explain how to use the selectors we'll use the `Scrapy shell` (which +To explain how to use the selectors we'll use the ``Scrapy shell`` (which provides interactive testing) and an example page located in the Scrapy documentation server: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..1afa513c8 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -599,7 +599,7 @@ The amount of time (in secs) that the downloader will wait before timing out. DOWNLOAD_MAXSIZE ---------------- -Default: `1073741824` (1024MB) +Default: ``1073741824`` (1024MB) The maximum response size (in bytes) that downloader will download. @@ -620,7 +620,7 @@ If you want to disable it set to 0. DOWNLOAD_WARNSIZE ----------------- -Default: `33554432` (32MB) +Default: ``33554432`` (32MB) The response size (in bytes) that downloader will start to warn. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..b551aa47d 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied. If you want to disable a builtin middleware (the ones defined in :setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it -in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its +in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the off-site middleware:: SPIDER_MIDDLEWARES = { @@ -200,7 +200,7 @@ DepthMiddleware .. class:: DepthMiddleware DepthMiddleware is used for tracking the depth of each Request inside the - site being scraped. It works by setting `request.meta['depth'] = 0` whenever + site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever there is no value previously set (usually just the first Request) and incrementing it by 1 otherwise. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..09feedefc 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -129,7 +129,7 @@ scrapy.Spider You probably won't need to override this directly because the default implementation acts as a proxy to the :meth:`__init__` method, calling - it with the given arguments `args` and named arguments `kwargs`. + it with the given arguments ``args`` and named arguments ``kwargs``. Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` attributes in the new instance so they can be accessed later inside the @@ -298,13 +298,13 @@ The above example can also be written as follows:: Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. -If you were to set the `start_urls` attribute from the command line, +If you were to set the ``start_urls`` attribute from the command line, you would have to parse it on your own into a list using something like `ast.literal_eval `_ or `json.loads `_ and then set it as an attribute. -Otherwise, you would cause iteration over a `start_urls` string +Otherwise, you would cause iteration over a ``start_urls`` string (a very common python pitfall) resulting in each character being seen as a separate url. diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 81ce800aa..6c993a970 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -22,7 +22,7 @@ To use the packages: sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7 -2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command:: +2. Create ``/etc/apt/sources.list.d/scrapy.list`` file using the following command:: echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list @@ -34,7 +34,7 @@ To use the packages: .. note:: Repeat step 3 if you are trying to upgrade Scrapy. -.. warning:: `python-scrapy` is a different package provided by official debian +.. warning:: ``python-scrapy`` is a different package provided by official debian repositories, it's very outdated and it isn't supported by Scrapy team. .. _Scrapinghub: https://scrapinghub.com/ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04aee18ed..2ecc4daad 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -153,7 +153,7 @@ class CrawlerRunner(object): It will call the given Crawler's :meth:`~Crawler.crawl` method, while keeping track of it so it can be stopped later. - If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler` + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as the spider class given to it. @@ -188,10 +188,10 @@ class CrawlerRunner(object): """ Return a :class:`~scrapy.crawler.Crawler` object. - * If `crawler_or_spidercls` is a Crawler, it is returned as-is. - * If `crawler_or_spidercls` is a Spider subclass, a new Crawler + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler is constructed for it. - * If `crawler_or_spidercls` is a string, this function finds + * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. """ @@ -273,7 +273,7 @@ class CrawlerProcess(CrawlerRunner): :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. - If `stop_after_crawl` is True, the reactor will be stopped after all + If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 075a6d862..65f347dcf 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -13,21 +13,21 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - All methods must return a dictionary listing the parameters `level`, `msg` - and `args` which are going to be used for constructing the log message when - calling logging.log. + All methods must return a dictionary listing the parameters ``level``, + ``msg`` and ``args`` which are going to be used for constructing the log + message when calling logging.log. Dictionary keys for the method outputs: - * `level` should be the log level for that action, you can use those + * ``level`` should be the log level for that action, you can use those from the python logging library: logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR and logging.CRITICAL. - * `msg` should be a string that can contain different formatting - placeholders. This string, formatted with the provided `args`, is going - to be the log message for that action. + * ``msg`` should be a string that can contain different formatting + placeholders. This string, formatted with the provided ``args``, is + going to be the log message for that action. - * `args` should be a tuple or dict with the formatting placeholders for - `msg`. The final log message is computed as output['msg'] % + * ``args`` should be a tuple or dict with the formatting placeholders + for ``msg``. The final log message is computed as output['msg'] % output['args']. """ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 510cc23c7..2d8091f5b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -255,13 +255,13 @@ class FilesPipeline(MediaPipeline): doing stat of the files and determining if file is new, uptodate or expired. - `new` files are those that pipeline never processed and needs to be + ``new`` files are those that pipeline never processed and needs to be downloaded from supplier site the first time. - `uptodate` files are the ones that the pipeline processed and are still + ``uptodate`` files are the ones that the pipeline processed and are still valid files. - `expired` files are those that pipeline already processed but the last + ``expired`` files are those that pipeline already processed but the last modification was made long time ago, so a reprocessing is recommended to refresh it in case of change. diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f255d436f..9eca6a4da 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -2,7 +2,7 @@ from ftplib import error_perm from posixpath import dirname def ftp_makedirs_cwd(ftp, path, first_call=True): - """Set the current directory of the FTP connection given in the `ftp` + """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. """ diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 828880709..e07fb8698 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -32,7 +32,7 @@ class TopLevelFormatter(logging.Filter): Since it can't be set for just one logger (it won't propagate for its children), it's going to be set in the root handler, with a parametrized - `loggers` list where it should act. + ``loggers`` list where it should act. """ def __init__(self, loggers=None): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 732ca13a0..aade3d9ac 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -97,8 +97,8 @@ def unicode_to_str(text, encoding=None, errors='strict'): def to_unicode(text, encoding=None, errors='strict'): - """Return the unicode representation of a bytes object `text`. If `text` - is already an unicode object, return it as-is.""" + """Return the unicode representation of a bytes object ``text``. If + ``text`` is already an unicode object, return it as-is.""" if isinstance(text, six.text_type): return text if not isinstance(text, (bytes, six.text_type)): @@ -110,7 +110,7 @@ def to_unicode(text, encoding=None, errors='strict'): def to_bytes(text, encoding=None, errors='strict'): - """Return the binary representation of `text`. If `text` + """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text @@ -123,7 +123,7 @@ def to_bytes(text, encoding=None, errors='strict'): def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of `text` + """ Return str representation of ``text`` (bytes in Python 2.x and unicode in Python 3.x). """ if six.PY2: return to_bytes(text, encoding, errors) @@ -189,7 +189,7 @@ def isbinarytext(text): def binary_is_text(data): - """ Returns `True` if the given ``data`` argument (a ``bytes`` object) + """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ if not isinstance(data, bytes): @@ -314,7 +314,7 @@ class WeakKeyCache(object): @deprecated def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """Return a (new) dict with unicode keys (and values when "keys_only" is - False) of the given dict converted to strings. `dct_or_tuples` can be a + False) of the given dict converted to strings. ``dct_or_tuples`` can be a dict or a list of tuples, like any dict constructor supports. """ d = {} @@ -357,10 +357,10 @@ def retry_on_eintr(function, *args, **kw): def without_none_values(iterable): - """Return a copy of `iterable` with all `None` entries removed. + """Return a copy of ``iterable`` with all ``None`` entries removed. - If `iterable` is a mapping, return a dictionary where all pairs that have - value `None` have been removed. + If ``iterable`` is a mapping, return a dictionary where all pairs that have + value ``None`` have been removed. """ try: return {k: v for k, v in six.iteritems(iterable) if v is not None} diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 657c53815..b3a4be007 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -109,12 +109,12 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= """Strip URL string from some of its components: - - `strip_credentials` removes "user:password@" - - `strip_default_port` removes ":80" (resp. ":443", ":21") + - ``strip_credentials`` removes "user:password@" + - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - - `origin_only` replaces path component with "/", also dropping + - ``origin_only`` replaces path component with "/", also dropping query and fragment components ; it also strips credentials - - `strip_fragment` drops any #fragment component + - ``strip_fragment`` drops any #fragment component """ parsed_url = urlparse(url) diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 366fcf033..eb362e945 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -10,7 +10,8 @@ Status Obsolete (discarded) SEP-006: Rename of Selectors to Extractors ========================================== -This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and their `x` method. +This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and +their ``x`` method. Motivation ========== @@ -57,7 +58,7 @@ Additional changes As the name of the method for performing selection (the ``x`` method) is not descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method (x sounds like a short for extract in english), we propose to rename it to -`select`, `sel` (is shortness if required), or `xpath` after `lxml's +``select``, ``sel`` (is shortness if required), or ``xpath`` after `lxml's `_ ``xpath`` method. Bonus (ItemBuilder) diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 40d9293b2..431428331 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -16,7 +16,7 @@ _DATABASES = collections.defaultdict(DummyDB) def open(file, flag='r', mode=0o666): """Open or create a dummy database compatible. - Arguments `flag` and `mode` are ignored. + Arguments ``flag`` and ``mode`` are ignored. """ # return same instance for same file argument return _DATABASES[file] diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 36baacfbd..d664b6ade 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -61,7 +61,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_fetch_redirect_follow_302(self): - """Test that calling `fetch(url)` follows HTTP redirects by default.""" + """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') code = "fetch('{0}')" errcode, out, errout = yield self.execute(['-c', code.format(url)]) @@ -71,7 +71,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_fetch_redirect_not_follow_302(self): - """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') code = "fetch('{0}', redirect=False)" errcode, out, errout = yield self.execute(['-c', code.format(url)]) From 184def1060f95767ef767948edeb3bbb4ed0e428 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 7 Mar 2019 00:09:10 +0530 Subject: [PATCH 59/67] fix a link inside docs --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4ac39ad2d..2effe94dc 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -172,5 +172,5 @@ links: .. _Twisted: https://twistedmatrix.com/trac/ .. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/ +.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ From 4ef38d925e0ded380a7cebabe3aab2340d4f3d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 14:21:00 +0100 Subject: [PATCH 60/67] Remove the unexisting retry_complete signal from the documentation --- docs/topics/downloader-middleware.rst | 2 -- scrapy/downloadermiddlewares/retry.py | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..9988ab18b 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -834,8 +834,6 @@ RetryMiddleware Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. -Once there are no more failed pages to retry, this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. The :class:`RetryMiddleware` can be configured through the following settings (see the settings documentation for more info): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 07e979628..dbc605a4c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,9 +7,7 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. Once -there is no more failed pages to retry this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. +once the spider has finished crawling all regular (non failed) pages. """ import logging From e108e3adbfa1a1f9bdd2a84180f18e5e43a39d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 15:13:11 +0100 Subject: [PATCH 61/67] Clarify the documentation of DEPTH_PRIORITY further --- docs/topics/settings.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..229a9e956 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -331,16 +331,16 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the request priority based on its depth: +An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of +a :class:`~scrapy.http.Request` based on its depth. -- if zero (default), no priority adjustment is made from depth -- **a positive value will decrease the priority, i.e. higher depth - requests will be processed later** ; this is commonly used when doing - breadth-first crawls (BFO) -- a negative value will increase priority, i.e., higher depth requests - will be processed sooner (DFO) +The priority of a request is adjusted as follows:: -See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. + request.priority = request.priority - ( depth * DEPTH_PRIORITY ) + +As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request +priority (BFO), while negative values increase request priority (DFO). See +also :ref:`faq-bfo-dfo`. .. note:: From b1063d9b2ca1a6bfb947fdd8b0158633184114ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 17:22:49 +0100 Subject: [PATCH 62/67] Use the description from README.rst on index.rst --- docs/index.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 0a96aa88e..cedde8f38 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,13 @@ Scrapy |version| documentation ============================== -This documentation contains everything you need to know about Scrapy. +Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used +to crawl websites and extract structured data from their pages. It can be used +for a wide range of purposes, from data mining to monitoring and automated +testing. + +.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler +.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping Getting help ============ From 91aec8b3bb805e9595f1b778fb14f703c6acf2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 18:19:30 +0100 Subject: [PATCH 63/67] Update developer-tools.rst Fixes #3674 --- docs/topics/developer-tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c1976258d..82857c9da 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -233,7 +233,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1] + start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) From 70aa5b1333a981b3b5eae70e29144c4c25fcfd4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Wed, 20 Mar 2019 15:32:20 +0100 Subject: [PATCH 64/67] Fix numeration --- docs/topics/selectors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index edc18f14d..282a585d4 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -436,7 +436,7 @@ The following examples show how these methods map to each other. >>> response.css('a::attr(href)').extract() ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] -2. ``Selector.get()`` is the same as ``Selector.extract()``:: +3. ``Selector.get()`` is the same as ``Selector.extract()``:: >>> response.css('a::attr(href)')[0].get() 'image1.html' From 2cb4dc32052c306568206f0997de4b4e53069efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 22 Mar 2019 09:50:11 -0300 Subject: [PATCH 65/67] Mentioning to use JSON API for ACLs --- docs/topics/settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90ae8fd93..fcdf31cac 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. .. setting:: FTP_PASSIVE_MODE @@ -1388,4 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search -.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file +.. _Google Cloud Storage: https://cloud.google.com/storage/ From 7c148fce5acc100f5f01719db374578bfca2512a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 15:40:16 +0100 Subject: [PATCH 66/67] Implement Item.deepcopy() --- docs/topics/items.rst | 44 +++++++++++++++++++++++++++++++++---------- scrapy/item.py | 8 ++++++++ tests/test_item.py | 8 ++++++++ 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index ae44aecd3..d744fd9ea 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -40,6 +40,7 @@ objects. Here is an example:: name = scrapy.Field() price = scrapy.Field() stock = scrapy.Field() + tags = scrapy.Field() last_updated = scrapy.Field(serializer=str) .. note:: Those familiar with `Django`_ will notice that Scrapy Items are @@ -155,19 +156,42 @@ To access all populated values, just use the typical `dict API`_:: >>> product.items() [('price', 1000), ('name', 'Desktop PC')] + +Copying items +------------- + +To copy an item, you must first decide whether you want a shallow copy or a +deep copy. + +If your item contains mutable_ values like lists or dictionaries, a shallow +copy will keep references to the same mutable values across all different +copies. + +.. _mutable: https://docs.python.org/glossary.html#term-mutable + +For example, if you have an item with a list of tags, and you create a shallow +copy of that item, both the original item and the copy have the same list of +tags. Adding a tag to the list of one of the items will add the tag to the +other item as well. + +If that is not the desired behavior, use a deep copy instead. + +See the `documentation of the copy module`_ for more information. + +.. _documentation of the copy module: https://docs.python.org/library/copy.html + +To create a shallow copy of an item, you can either call +:meth:`~scrapy.item.Item.copy` on an existing item +(``product2 = product.copy()``) or instantiate your item class from an existing +item (``product2 = Product(product)``). + +To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +(``product2 = product.deepcopy()``). + + Other common tasks ------------------ -Copying items:: - - >>> product2 = Product(product) - >>> print(product2) - Product(name='Desktop PC', price=1000) - - >>> product3 = product2.copy() - >>> print(product3) - Product(name='Desktop PC', price=1000) - Creating dicts from items:: >>> dict(product) # create a dict from all populated values diff --git a/scrapy/item.py b/scrapy/item.py index aa05e9c69..031b80a2d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst from pprint import pformat from collections import MutableMapping +from copy import deepcopy from abc import ABCMeta import six @@ -96,6 +97,13 @@ class DictItem(MutableMapping, BaseItem): def copy(self): return self.__class__(self) + def deepcopy(self): + """Return a `deep copy`_ of this item. + + .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy + """ + return deepcopy(self) + @six.add_metaclass(ItemMeta) class Item(DictItem): diff --git a/tests/test_item.py b/tests/test_item.py index 2c1eb0dd3..010d3b141 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -249,6 +249,14 @@ class ItemTest(unittest.TestCase): copied_item['name'] = copied_item['name'].upper() self.assertNotEqual(item['name'], copied_item['name']) + def test_deepcopy(self): + class TestItem(Item): + tags = Field() + item = TestItem({'tags': ['tag1']}) + copied_item = item.deepcopy() + item['tags'].append('tag2') + assert item['tags'] != copied_item['tags'] + class ItemMetaTest(unittest.TestCase): From 9a0fe8bf2dc108b1f2c50aaef211b39981e65c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez=20Cabot?= Date: Wed, 20 Mar 2019 16:13:31 +0100 Subject: [PATCH 67/67] remove duplicated entry in gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7392ed31e..ff6e2ea65 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ htmlcov/ .pytest_cache/ .coverage.* .cache/ -.pytest_cache/ # Windows Thumbs.db