From f4dd8bcdc29cb5411d0fe2ed8f53e75883402ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Tue, 16 Jun 2015 17:31:37 +0300 Subject: [PATCH 001/137] Disable dupefilter in shell --- scrapy/commands/shell.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 92ebbe605..77ae1358b 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -16,7 +16,11 @@ from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): requires_project = False - default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0} + default_settings = { + 'KEEP_ALIVE': True, + 'LOGSTATS_INTERVAL': 0, + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', + } def syntax(self): return "[url|file]" From 1b4fd3a8dffbbfc0207d707cd11ca84d11e76064 Mon Sep 17 00:00:00 2001 From: nyov Date: Sun, 12 Jul 2015 18:27:41 +0000 Subject: [PATCH 002/137] Support anonymous connections in S3DownloadHandler Also consider any unknown keyword args for S3DownloadHandler as arguments to pass on to S3Connection (e.g. proxy settings). --- scrapy/core/downloader/handlers/s3.py | 11 +++++++++-- tests/test_downloader_handlers.py | 21 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 311815b70..38cfd1e10 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -35,7 +35,7 @@ def get_s3_connection(): class S3DownloadHandler(object): def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ - httpdownloadhandler=HTTPDownloadHandler): + httpdownloadhandler=HTTPDownloadHandler, **kw): _S3Connection = get_s3_connection() if _S3Connection is None: @@ -46,8 +46,15 @@ class S3DownloadHandler(object): if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + # If no credentials could be found anywhere, + # consider this an anonymous connection request by default; + # unless 'anon' was set explicitly (True/False). + anon = kw.get('anon', None) + if anon is None and not aws_access_key_id and not aws_secret_access_key: + kw['anon'] = True + try: - self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key) + self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw) except Exception as ex: raise NotConfigured(str(ex)) self._download_http = httpdownloadhandler(settings).download_request diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e4d957d8e..8280b21aa 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -403,6 +403,23 @@ class HttpDownloadHandlerMock(object): def download_request(self, request, spider): return request +class S3AnonTestCase(unittest.TestCase): + skip = 'boto' not in optional_features and 'missing boto library' + + def setUp(self): + self.s3reqh = S3DownloadHandler(Settings(), + httpdownloadhandler=HttpDownloadHandlerMock, + #anon=True, # is implicit + ) + self.download_request = self.s3reqh.download_request + self.spider = Spider('foo') + + def test_anon_request(self): + req = Request('s3://aws-publicdatasets/') + httpreq = self.download_request(req, self.spider) + self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) + self.assertEqual(self.s3reqh.conn.anon, True) + class S3TestCase(unittest.TestCase): download_handler_cls = S3DownloadHandler try: @@ -420,8 +437,8 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' def setUp(self): - s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \ - self.AWS_SECRET_ACCESS_KEY, \ + s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, + self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request self.spider = Spider('foo') From ecbfe4bd6661acd165407e7f1e5abf1f0a2fa31c Mon Sep 17 00:00:00 2001 From: nyov Date: Sun, 12 Jul 2015 16:53:54 +0000 Subject: [PATCH 003/137] drop deprecated "optional_features" set --- scrapy/__init__.py | 9 +-------- scrapy/core/downloader/handlers/http.py | 4 ++-- scrapy/utils/log.py | 3 --- tests/test_downloader_handlers.py | 7 +++---- tests/test_downloadermiddleware_retry.py | 4 ++-- tests/test_toplevel.py | 4 ---- 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c0477f509..03ec6c667 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,7 +2,7 @@ Scrapy - a web crawling and web scraping framework written for Python """ -__all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version', +__all__ = ['__version__', 'version_info', 'twisted_version', 'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field'] # Scrapy version @@ -27,15 +27,8 @@ del warnings from . import _monkeypatches del _monkeypatches -# WARNING: optional_features set is deprecated and will be removed soon. Do not use. -optional_features = set() -# TODO: backwards compatibility, remove for Scrapy 0.20 -optional_features.add('ssl') - from twisted import version as _txv twisted_version = (_txv.major, _txv.minor, _txv.micro) -if twisted_version >= (11, 1, 0): - optional_features.add('http11') # Declare top-level shortcuts from scrapy.spiders import Spider diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 1efebb939..81da2615a 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,7 +1,7 @@ -from scrapy import optional_features +from scrapy import twisted_version from .http10 import HTTP10DownloadHandler -if 'http11' in optional_features: +if twisted_version >= (11, 1, 0): from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler else: HTTPDownloadHandler = HTTP10DownloadHandler diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index d40202953..cc2f0b164 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -126,9 +126,6 @@ def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Optional features available: %(features)s", - {'features': ", ".join(scrapy.optional_features)}) - d = dict(overridden_settings(settings)) logger.info("Overridden settings: %(settings)r", {'settings': d}) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e4d957d8e..d2a349b40 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -27,7 +27,6 @@ from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.settings import Settings -from scrapy import optional_features from scrapy.utils.test import get_crawler from scrapy.exceptions import NotConfigured @@ -220,7 +219,7 @@ class Http10TestCase(HttpTestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler - if 'http11' not in optional_features: + if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' def test_download_without_maxsize_limit(self): @@ -267,7 +266,7 @@ class Http11TestCase(HttpTestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - if 'http11' not in optional_features: + if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' def setUp(self): @@ -392,7 +391,7 @@ class Http10ProxyTestCase(HttpProxyTestCase): class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler - if 'http11' not in optional_features: + if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c0381e144..20561e771 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -4,7 +4,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError -from scrapy import optional_features +from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.xlib.tx import ResponseFailed from scrapy.spiders import Spider @@ -75,7 +75,7 @@ class RetryTest(unittest.TestCase): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost] - if 'http11' in optional_features: + if twisted_version >= (11, 1, 0): # http11 available exceptions.append(ResponseFailed) for exc in exceptions: diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index e9f220092..91bbe43bc 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -11,10 +11,6 @@ class ToplevelTestCase(TestCase): def test_version_info(self): self.assertIs(type(scrapy.version_info), tuple) - def test_optional_features(self): - self.assertIs(type(scrapy.optional_features), set) - self.assertIn('ssl', scrapy.optional_features) - def test_request_shortcut(self): from scrapy.http import Request, FormRequest self.assertIs(scrapy.Request, Request) From c34dbe955d1a72ddef97afa1f0fb92becd9f4ca3 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 29 Oct 2015 14:18:59 +0800 Subject: [PATCH 004/137] fixed: Issue #1562 (Incorrectly picked URL in `scrapy.http.FormRequest.from_response` when there is a `` tag) --- scrapy/http/request/form.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a12a2fd07..4a9bd732e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -11,6 +11,7 @@ from parsel.selector import create_root_node import six from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike +from scrapy.utils.response import get_base_url class FormRequest(Request): @@ -44,7 +45,7 @@ class FormRequest(Request): def _get_form_url(form, url): if url is None: - return form.action or form.base_url + return urljoin(form.base_url, form.action) return urljoin(form.base_url, url) @@ -58,7 +59,7 @@ def _urlencode(seq, enc): def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ text = response.body_as_unicode() - root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url) + root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError("No
element found in %s" % response) From e379f58cad0289d725a5241606542d0e20ecd73d Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 29 Oct 2015 14:52:31 +0800 Subject: [PATCH 005/137] fixed: Issue #1564 (Incorrectly picked URL in `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `` tag. ) --- scrapy/linkextractors/regex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index b6f8d5d30..0fc7b079f 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,7 +1,7 @@ import re from six.moves.urllib.parse import urljoin -from w3lib.html import remove_tags, replace_entities, replace_escape_chars +from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link from .sgml import SgmlLinkExtractor @@ -31,7 +31,7 @@ class RegexLinkExtractor(SgmlLinkExtractor): return clean_url if base_url is None: - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url + base_url = get_base_url(response_text, response_url, response_encoding) links_text = linkre.findall(response_text) return [Link(clean_url(url).encode(response_encoding), From e19bf4aecc9027fd4023282a371c13f77fadd510 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Mon, 2 Nov 2015 22:52:41 +0800 Subject: [PATCH 006/137] added: Test case for the fix --- tests/test_http_request.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ff0941961..60fd855dd 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -801,6 +801,25 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'test2'], [b'val2']) self.assertEqual(fs[b'button1'], [b'']) + def test_html_base_form_action(self): + response = _buildresponse( + """ + + + + + + + + + + """, + url='http://a.com/' + ) + req = self.request_class.from_response(response) + self.assertEqual(req.url, 'http://b.com/test_form') + + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) kwargs.setdefault('url', 'http://example.com') From 94486bb294a6e765efe14affddb1df355a6c298b Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Mon, 2 Nov 2015 23:00:42 +0800 Subject: [PATCH 007/137] added: Test case for the fix. --- tests/test_linkextractors_deprecated.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index e3664f8d8..89dcb75c2 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -190,3 +190,20 @@ class RegexLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + + def test_html_base_href(self): + html = """ + + + + + + + + + """ + response = HtmlResponse("http://a.com/", body=html) + lx = RegexLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://b.com/test.html', text=u'', nofollow=False), + ]) From e66f64989409ddf9117f854934d52b2772aace74 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 01:14:49 +0100 Subject: [PATCH 008/137] Bring back _BASE settings --- docs/topics/downloader-middleware.rst | 22 +++-- docs/topics/extensions.rst | 13 +-- docs/topics/feed-exports.rst | 47 +++++++--- docs/topics/settings.rst | 128 +++++++++++++++++--------- docs/topics/spider-middleware.rst | 22 +++-- scrapy/settings/default_settings.py | 25 +++-- 6 files changed, 169 insertions(+), 88 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 08d8f3edf..cc0254d29 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -23,20 +23,22 @@ Here's an example:: 'myproject.middlewares.CustomDownloaderMiddleware': 543, } -The specified :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the -default one (i.e. it does not overwrite it) and then sorted by order to get the -final sorted list of enabled middlewares: the first middleware is the one -closer to the engine and the last is the one closer to the downloader. +The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the +:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant +to be overridden) and then sorted by order to get the final sorted list of +enabled middlewares: the first middleware is the one closer to the engine and +the last is the one closer to the downloader. -To decide which order to assign to your middleware see the default -:setting:`DOWNLOADER_MIDDLEWARES` setting and pick a value according to +To decide which order to assign to your middleware see the +:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to where you want to insert the middleware. The order does matter because each middleware performs a different action and your middleware could depend on some previous (or subsequent) middleware being applied. -If you want to disable a built-in middleware you must define it in your -project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its -value. For example, if you want to disable the user-agent middleware:: +If you want to disable a built-in middleware (the ones defined in +:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it +in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { 'myproject.middlewares.CustomDownloaderMiddleware': 543, @@ -162,7 +164,7 @@ middleware, see the :ref:`downloader middleware usage guide `. For a list of the components enabled by default (and their orders) see the -:setting:`DOWNLOADER_MIDDLEWARES` setting. +:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting. .. _cookies-mw: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 11c0aadb6..847353868 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -42,13 +42,14 @@ by a string: the full Python path to the extension's class name. For example:: As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are the extension paths, and their values are the orders, which define the -extension *loading* order. The specified :setting:`EXTENSIONS` setting is merged -with the default one (i.e. it does not overwrite it) and then sorted by order -to get the final sorted list of enabled extensions. +extension *loading* order. The :setting:`EXTENSIONS` setting is merged with the +:setting:`EXTENSIONS_BASE` setting defined in Scrapy (and not meant to be +overridden) and then sorted by order to get the final sorted list of enabled +extensions. As extensions typically do not depend on each other, their loading order is -irrelevant in most cases. This is why the default :setting:`EXTENSIONS` setting -defines all extensions with the same order (``500``). However, this feature can +irrelevant in most cases. This is why the :setting:`EXTENSIONS_BASE` setting +defines all extensions with the same order (``0``). However, this feature can be exploited if you need to add an extension which depends on other extensions already loaded. @@ -63,7 +64,7 @@ Disabling an extension ====================== In order to disable an extension that comes enabled by default (ie. those -included in the default :setting:`EXTENSIONS` setting) you must set its order to +included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to ``None``. For example:: EXTENSIONS = { diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index d8b8da166..03c6fb3fb 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -265,6 +265,16 @@ Whether to export empty feeds (ie. feeds with no items). FEED_STORAGES ------------- +Default:: ``{}`` + +A dict containing additional feed storage backends supported by your project. +The keys are URI schemes and the values are paths to storage classes. + +.. setting:: FEED_STORAGES_BASE + +FEED_STORAGES_BASE +------------------ + Default:: { @@ -275,19 +285,30 @@ Default:: 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -A dict containing all feed storage backends supported by your project. The keys -are URI schemes and the values are paths to storage classes. +A dict containing the built-in feed storage backends supported by Scrapy. You +can disable any of these backends by assigning ``None`` to their URI scheme in +:setting:`FEED_STORAGES`. E.g., to disable the built-in FTP storage backend +(without replacement), place this in your ``settings.py``:: -When you set :setting:`FEED_STORAGES` manually, e.g. in your project's settings -module, it will be merged with the default, not overwrite it. If you want to -disable any of the default feed storage backends, you must assign ``None`` as -their value. + FEED_STORAGES = { + 'ftp': None, + } .. setting:: FEED_EXPORTERS FEED_EXPORTERS -------------- +Default:: ``{}`` + +A dict containing additional exporters supported by your project. The keys are +serialization formats and the values are paths to :ref:`Item exporter +` classes. + +.. setting:: FEED_EXPORTERS_BASE + +FEED_EXPORTERS_BASE +------------------- Default:: { @@ -300,14 +321,14 @@ Default:: 'pickle': 'scrapy.exporters.PickleItemExporter', } -A dict containing all feed exporters supported by your project. The keys are -URI schemes and the values are paths to :ref:`Item exporter ` -classes. +A dict containing the built-in feed exporters supported by Scrapy. You can +disable any of these exporters by assigning ``None`` to their serialization +format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter +(without replacement), place this in your ``settings.py``:: -When you set :setting:`FEED_EXPORTERS` manually, e.g. in your project's settings -module, it will be merged with the default, not overwrite it. If you want to -disable any of the default feed exporters, you must assign ``None`` as their -value. + FEED_EXPORTERS = { + 'csv': None, + } .. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: http://aws.amazon.com/s3/ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8908fae7e..aa0417e1a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -269,11 +269,6 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. -When you set :setting:`DEFAULT_REQUEST_HEADERS` manually, e.g. in your -project's settings module, it will be merged with the default, not overwrite it. -If you want to disable any of the default request headers (and not replace them) -you must assign ``None`` as their value. - .. setting:: DEPTH_LIMIT DEPTH_LIMIT @@ -355,6 +350,16 @@ The downloader to use for crawling. DOWNLOADER_MIDDLEWARES ---------------------- +Default:: ``{}`` + +A dict containing the downloader middlewares enabled in your project, and their +orders. For more info see :ref:`topics-downloader-middleware-setting`. + +.. setting:: DOWNLOADER_MIDDLEWARES_BASE + +DOWNLOADER_MIDDLEWARES_BASE +--------------------------- + Default:: { @@ -375,16 +380,11 @@ Default:: 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, } -A dict containing the downloader middlewares enabled in your project, and their -orders. Low orders are closer to the engine, high orders are closer to the -downloader. - -When you set :setting:`DOWNLOADER_MIDDLEWARES` manually, e.g. in your project's -settings module, it will be merged with the default, not overwrite it. If you -want to disable any of the default downloader middlewares you must assign -``None`` as their value. - -For more info see :ref:`topics-downloader-middleware-setting`. +A dict containing the downloader middlewares enabled by default in Scrapy. Low +orders are closer to the engine, high orders are closer to the downloader. You +should never modify this setting in your project, modify +:setting:`DOWNLOADER_MIDDLEWARES` instead. For more info see +:ref:`topics-downloader-middleware-setting`. .. setting:: DOWNLOADER_STATS @@ -425,6 +425,16 @@ spider attribute. DOWNLOAD_HANDLERS ----------------- +Default: ``{}`` + +A dict containing the request downloader handlers enabled in your project. +See :setting:`DOWNLOAD_HANDLERS_BASE` for example format. + +.. setting:: DOWNLOAD_HANDLERS_BASE + +DOWNLOAD_HANDLERS_BASE +---------------------- + Default:: { @@ -436,15 +446,16 @@ Default:: } -A dict containing the request downloader handlers enabled in your project. +A dict containing the request download handlers enabled by default in Scrapy. +You should never modify this setting in your project, modify +:setting:`DOWNLOAD_HANDLERS` instead. -When you set :setting:`DOWNLOAD_HANDLERS` manually, e.g. in your project's -settings module, it will be merged with the default, not overwrite it. If you -want to disable any of the default download handlers you must assign ``None`` -as their value. For example, if you want to disable the file download handler:: +You can disable any of these download handlers by assigning ``None`` to their +URI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP +handler (without replacement), place this in your ``settings.py``:: DOWNLOAD_HANDLERS = { - 'file': None, + 'ftp': None, } .. setting:: DOWNLOAD_TIMEOUT @@ -544,6 +555,15 @@ to ``vi`` (on Unix systems) or the IDLE editor (on Windows). EXTENSIONS ---------- +Default:: ``{}`` + +A dict containing the extensions enabled in your project, and their orders. + +.. setting:: EXTENSIONS_BASE + +EXTENSIONS_BASE +--------------- + Default:: { @@ -558,15 +578,10 @@ Default:: 'scrapy.extensions.throttle.AutoThrottle': 0, } -A dict containing the extensions enabled in your project, and their orders. By -default, this setting contains all stable built-in extensions. Keep in mind that +A dict containing the extensions available by default in Scrapy, and their +orders. This setting contains all stable built-in extensions. Keep in mind that some of them need to be enabled through a setting. -When you set :setting:`EXTENSIONS` manually, e.g. in your project's settings -module, it will be merged with the default, not overwrite it. If you want to -disable any of the default enabled extensions you must assign ``None`` as their -value. - For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. @@ -589,6 +604,16 @@ Example:: 'mybot.pipelines.validate.StoreMyItem': 800, } +.. setting:: ITEM_PIPELINES_BASE + +ITEM_PIPELINES_BASE +------------------- + +Default: ``{}`` + +A dict containing the pipelines enabled by default in Scrapy. You should never +modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. + .. setting:: LOG_ENABLED LOG_ENABLED @@ -878,6 +903,16 @@ The scheduler to use for crawling. SPIDER_CONTRACTS ---------------- +Default:: ``{}`` + +A dict containing the spider contracts enabled in your project, used for +testing spiders. For more info see :ref:`topics-contracts`. + +.. setting:: SPIDER_CONTRACTS_BASE + +SPIDER_CONTRACTS_BASE +--------------------- + Default:: { @@ -886,13 +921,17 @@ Default:: 'scrapy.contracts.default.ScrapesContract': 3, } -A dict containing the scrapy contracts enabled in your project, used for -testing spiders. For more info see :ref:`topics-contracts`. +A dict containing the scrapy contracts enabled by default in Scrapy. You should +never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` +instead. For more info see :ref:`topics-contracts`. -When you set :setting:`SPIDER_CONTRACTS` manually, e.g. in your project's -settings module, it will be merged with the default, not overwrite it. If you -want to disable any of the default contracts you must assign ``None`` as their -value. +You can disable any of these contracts by assigning ``None`` to their class +path in :setting:`SPIDER_CONTRACTS`. E.g., to disable the built-in +``ScrapesContract``, place this in your ``settings.py``:: + + SPIDER_CONTRACTS = { + 'scrapy.contracts.default.ScrapesContract': None, + } .. setting:: SPIDER_LOADER_CLASS @@ -909,6 +948,16 @@ The class that will be used for loading spiders, which must implement the SPIDER_MIDDLEWARES ------------------ +Default:: ``{}`` + +A dict containing the spider middlewares enabled in your project, and their +orders. For more info see :ref:`topics-spider-middleware-setting`. + +.. setting:: SPIDER_MIDDLEWARES_BASE + +SPIDER_MIDDLEWARES_BASE +----------------------- + Default:: { @@ -919,14 +968,9 @@ Default:: 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, } -A dict containing the spider middlewares enabled in your project, and their -orders. Low orders are closer to the engine, high orders are closer to the -spider. For more info see :ref:`topics-spider-middleware-setting`. - -When you set :setting:`SPIDER_MIDDLEWARES` manually, e.g. in your project's -settings module, it will be merged with the default, not overwrite it. If you -want to disable any of the default spider middlewares you must assign ``None`` -as their value. +A dict containing the spider middlewares enabled by default in Scrapy, and +their orders. Low orders are closer to the engine, high orders are closer to +the spider. For more info see :ref:`topics-spider-middleware-setting`. .. setting:: SPIDER_MODULES diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index d448801d3..84daaaa55 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -24,20 +24,22 @@ Here's an example:: 'myproject.middlewares.CustomSpiderMiddleware': 543, } -The specified :setting:`SPIDER_MIDDLEWARES` setting is merged with the default -one (i.e. it does not overwrite it) and then sorted by order to get the final -sorted list of enabled middlewares: the first middleware is the one closer to -the engine and the last is the one closer to the spider. +The :setting:`SPIDER_MIDDLEWARES` setting is merged with the +:setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to +be overridden) and then sorted by order to get the final sorted list of enabled +middlewares: the first middleware is the one closer to the engine and the last +is the one closer to the spider. -To decide which order to assign to your middleware see the default -:setting:`SPIDER_MIDDLEWARES` setting and pick a value according to where +To decide which order to assign to your middleware see the +:setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where you want to insert the middleware. The order does matter because each middleware performs a different action and your middleware could depend on some previous (or subsequent) middleware being applied. -If you want to disable a builtin middleware you must define it in your project's -:setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For -example, if you want to disable the off-site middleware:: +If you want to disable a builtin middleware (the ones defined in +:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it +in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its +value. For example, if you want to disable the off-site middleware:: SPIDER_MIDDLEWARES = { 'myproject.middlewares.CustomSpiderMiddleware': 543, @@ -171,7 +173,7 @@ information on how to use them and how to write your own spider middleware, see the :ref:`spider middleware usage guide `. For a list of the components enabled by default (and their orders) see the -:setting:`SPIDER_MIDDLEWARES` setting. +:setting:`SPIDER_MIDDLEWARES_BASE` setting. DepthMiddleware --------------- diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 375efcdbb..8435b0354 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -63,7 +63,8 @@ DNS_TIMEOUT = 60 DOWNLOAD_DELAY = 0 -DOWNLOAD_HANDLERS = { +DOWNLOAD_HANDLERS = {} +DOWNLOAD_HANDLERS_BASE = { 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', @@ -81,7 +82,9 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' -DOWNLOADER_MIDDLEWARES = { +DOWNLOADER_MIDDLEWARES = {} + +DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, @@ -113,7 +116,9 @@ except KeyError: else: EDITOR = 'vi' -EXTENSIONS = { +EXTENSIONS = {} + +EXTENSIONS_BASE = { 'scrapy.extensions.corestats.CoreStats': 0, 'scrapy.extensions.telnet.TelnetConsole': 0, 'scrapy.extensions.memusage.MemoryUsage': 0, @@ -130,14 +135,16 @@ FEED_URI_PARAMS = None # a function to extend uri arguments FEED_FORMAT = 'jsonlines' FEED_STORE_EMPTY = False FEED_EXPORT_FIELDS = None -FEED_STORAGES = { +FEED_STORAGES = {} +FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_EXPORTERS = { +FEED_EXPORTERS = {} +FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', 'jsonlines': 'scrapy.exporters.JsonLinesItemExporter', 'jl': 'scrapy.exporters.JsonLinesItemExporter', @@ -163,6 +170,7 @@ HTTPCACHE_GZIP = False ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} +ITEM_PIPELINES_BASE = {} LOG_ENABLED = True LOG_ENCODING = 'utf-8' @@ -221,7 +229,9 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' -SPIDER_MIDDLEWARES = { +SPIDER_MIDDLEWARES = {} + +SPIDER_MIDDLEWARES_BASE = { # Engine side 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, @@ -248,7 +258,8 @@ TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' -SPIDER_CONTRACTS = { +SPIDER_CONTRACTS = {} +SPIDER_CONTRACTS_BASE = { 'scrapy.contracts.default.UrlContract': 1, 'scrapy.contracts.default.ReturnsContract': 2, 'scrapy.contracts.default.ScrapesContract': 3, From b6a023ce987a064b222b1aa2de03a50991f387fe Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 11:35:59 +0100 Subject: [PATCH 009/137] Add backwards compatibility for build_component_list --- scrapy/utils/conf.py | 14 ++++++++++---- tests/test_utils_conf.py | 37 +++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 57f2b6322..e8af90f11 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -10,7 +10,7 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values -def build_component_list(compdict, convert=update_classpath): +def build_component_list(compdict, custom=None, convert=update_classpath): """Compose a component list from a { class: order } dictionary.""" def _check_components(complist): @@ -34,9 +34,15 @@ def build_component_list(compdict, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in six.iteritems(compdict)} - if isinstance(compdict, (list, tuple)): - _check_components(compdict) - return type(compdict)(convert(c) for c in compdict) + # BEGIN Backwards compatibility for old (base, custom) call signature + if isinstance(custom, (list, tuple)): + _check_components(custom) + return type(custom)(convert(c) for c in custom) + + if custom is not None: + compdict.update(custom) + # END Backwards compatibility + compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index af15d3184..dab41ac8d 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -8,46 +8,59 @@ class BuildComponentListTest(unittest.TestCase): def test_build_dict(self): d = {'one': 1, 'two': None, 'three': 8, 'four': 4} - self.assertEqual(build_component_list(d, lambda x: x), + self.assertEqual(build_component_list(d, convert=lambda x: x), ['one', 'four', 'three']) + def test_backwards_compatible_build_dict(self): + base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} + custom = {'two': None, 'three': 8, 'four': 4} + self.assertEqual(build_component_list(base, custom, + convert=lambda x: x), + ['one', 'four', 'five', 'three']) + def test_return_list(self): custom = ['a', 'b', 'c'] - self.assertEqual(build_component_list(custom, lambda x: x), custom) + self.assertEqual(build_component_list(None, custom, + convert=lambda x: x), + custom) def test_map_dict(self): custom = {'one': 1, 'two': 2, 'three': 3} - self.assertEqual(build_component_list(custom, lambda x: x.upper()), + self.assertEqual(build_component_list({}, custom, + convert=lambda x: x.upper()), ['ONE', 'TWO', 'THREE']) def test_map_list(self): custom = ['a', 'b', 'c'] - self.assertEqual(build_component_list(custom, lambda x: x.upper()), + self.assertEqual(build_component_list(None, custom, + lambda x: x.upper()), ['A', 'B', 'C']) def test_duplicate_components_in_dict(self): duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4} - self.assertRaises(ValueError, - build_component_list, duplicate_dict, lambda x: x.lower()) + self.assertRaises(ValueError, build_component_list, {}, duplicate_dict, + convert=lambda x: x.lower()) def test_duplicate_components_in_list(self): duplicate_list = ['a', 'b', 'a'] - self.assertRaises(ValueError, - build_component_list, duplicate_list, lambda x: x) + self.assertRaises(ValueError, build_component_list, None, + duplicate_list, convert=lambda x: x) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0) duplicate_bs.set('ONE', 4, priority=10) - self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()), + self.assertEqual(build_component_list(duplicate_bs, + convert=lambda x: x.lower()), ['two', 'one']) duplicate_bs.set('one', duplicate_bs['one'], priority=20) - self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()), + self.assertEqual(build_component_list(duplicate_bs, + convert=lambda x: x.lower()), ['one', 'two']) # Same priority raises ValueError duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20) - self.assertRaises(ValueError, - build_component_list, duplicate_bs, convert=lambda x: x.lower()) + self.assertRaises(ValueError, build_component_list, duplicate_bs, + convert=lambda x: x.lower()) class UtilsConfTestCase(unittest.TestCase): From 52ecee6a6271ef2d981f7f1993311abd8ed31bd9 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 13:15:32 +0100 Subject: [PATCH 010/137] Replace BaseSettings._getcomposite() with public .getwithbase() method --- scrapy/commands/check.py | 2 +- scrapy/commands/crawl.py | 3 ++- scrapy/commands/runspider.py | 2 +- scrapy/core/downloader/handlers/__init__.py | 3 ++- scrapy/core/downloader/middleware.py | 3 ++- scrapy/core/spidermw.py | 2 +- scrapy/extension.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/pipelines/__init__.py | 2 +- scrapy/settings/__init__.py | 30 ++++++++------------- tests/test_settings/__init__.py | 25 ++++++----------- 11 files changed, 31 insertions(+), 45 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index a423ba2c9..b8a9ef989 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -58,7 +58,7 @@ class Command(ScrapyCommand): def run(self, args, opts): # load contracts - contracts = build_component_list(self.settings._getcomposite('SPIDER_CONTRACTS')) + contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS')) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 7f5c64c20..4b986bf9d 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -35,7 +35,8 @@ class Command(ScrapyCommand): self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) + feed_exporters = without_none_values( + self.settings.getwithbase('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 72229bcf5..1da09e4da 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -58,7 +58,7 @@ class Command(ScrapyCommand): self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) + feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 0e78e04f4..bc5cd742e 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -20,7 +20,8 @@ class DownloadHandlers(object): self._schemes = {} # stores acceptable schemes on instancing self._handlers = {} # stores instanced handlers for schemes self._notconfigured = {} # remembers failed handlers - handlers = without_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS')) + handlers = without_none_values( + crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 958113fc3..c3b23e284 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -19,7 +19,8 @@ class DownloaderMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('DOWNLOADER_MIDDLEWARES')) + return build_component_list( + settings.getwithbase('DOWNLOADER_MIDDLEWARES')) def _add_middleware(self, mw): if hasattr(mw, 'process_request'): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b5c80c350..a206e4b0c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -18,7 +18,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('SPIDER_MIDDLEWARES')) + return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): super(SpiderMiddlewareManager, self)._add_middleware(mw) diff --git a/scrapy/extension.py b/scrapy/extension.py index 4ceb32c68..e39e456fa 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -12,4 +12,4 @@ class ExtensionManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('EXTENSIONS')) + return build_component_list(settings.getwithbase('EXTENSIONS')) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1e27a1e7e..daea551cb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -196,7 +196,7 @@ class FeedExporter(object): return item def _load_components(self, setting_prefix): - conf = without_none_values(self.settings._getcomposite(setting_prefix)) + conf = without_none_values(self.settings.getwithbase(setting_prefix)) d = {} for k, v in conf.items(): try: diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 8df0d3154..2ef8786d0 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -13,7 +13,7 @@ class ItemPipelineManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings._getcomposite('ITEM_PIPELINES')) + return build_component_list(settings.getwithbase('ITEM_PIPELINES')) def _add_middleware(self, pipe): super(ItemPipelineManager, self)._add_middleware(pipe) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 13656298b..b0f59ccc7 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -195,25 +195,17 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) - def _getcomposite(self, name): - # DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS - # It's for internal use in the transition away from the _BASE settings - # and will be removed along with _BASE support in a future release - basename = name + "_BASE" - if basename in self: - warnings.warn('_BASE settings are deprecated.', - category=ScrapyDeprecationWarning) - # When users defined a _BASE setting, they explicitly don't want to - # use any of Scrapy's defaults. Therefore, we only use these entries - # from self[name] (where the defaults now live) that have a priority - # higher than 'default' - compsett = BaseSettings(self[basename], priority='default') - for k in self[name]: - prio = self[name].getpriority(k) - if prio > get_settings_priority('default'): - compsett.set(k, self[name][k], prio) - return compsett - return self[name] + def getwithbase(self, name): + """Get a composition of a dictionary-like setting and its `_BASE` + counterpart. + + :param name: name of the dictionary-like setting + :type name: string + """ + compbs = BaseSettings() + compbs.update(self[name + '_BASE']) + compbs.update(self[name]) + return compbs def getpriority(self, name): """ diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4ef08bb0b..8d98d2cec 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -263,24 +263,15 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getpriority('key'), 99) self.assertEqual(settings.getpriority('nonexistentkey'), None) - def test_getcomposite(self): - s = BaseSettings({'TEST_BASE': {1: 1, 2: 2}, + def test_getwithbase(self): + s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), - 'HASNOBASE': BaseSettings({1: 1}, 'default')}) - s['TEST'].set(4, 4, priority='project') - # When users specify a _BASE setting they explicitly don't want to use - # Scrapy's defaults, so we don't want to see anything that has a - # 'default' priority from TEST - cs = s._getcomposite('TEST') - self.assertEqual(len(cs), 3) - self.assertEqual(cs[1], 1) - self.assertEqual(cs[2], 2) - self.assertEqual(cs[4], 4) - cs = s._getcomposite('HASNOBASE') - self.assertEqual(len(cs), 1) - self.assertEqual(cs[1], 1) - cs = s._getcomposite('NONEXISTENT') - self.assertIsNone(cs) + 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) + s['TEST'].set(2, 200, 'cmdline') + six.assertCountEqual(self, s.getwithbase('TEST'), + {1: 1, 2: 200, 3: 30}) + six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE']) + self.assertEqual(s.getwithbase('NONEXISTENT'), {}) def test_maxpriority(self): # Empty settings should return 'default' From 44f6ada034f9d75e0089f45d3ad9f3c6fa5777f1 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 15:54:50 +0100 Subject: [PATCH 011/137] Overwrite, not update, dictionary-like settings --- scrapy/settings/__init__.py | 13 +++++-------- tests/test_cmdline/__init__.py | 19 ++++++++++--------- tests/test_settings/__init__.py | 25 +++++++++++++------------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b0f59ccc7..e62bdd08e 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -49,14 +49,11 @@ class SettingsAttribute(object): def set(self, value, priority): """Sets value if priority is higher or equal than current priority.""" - if isinstance(self.value, BaseSettings): - # Ignore self.priority if self.value has per-key priorities - self.value.update(value, priority) - self.priority = max(self.value.maxpriority(), priority) - else: - if priority >= self.priority: - self.value = value - self.priority = priority + if priority >= self.priority: + if isinstance(self.value, BaseSettings): + value = BaseSettings(value, priority=priority) + self.value = value + self.priority = priority def __str__(self): return "", 'u"'): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - self.assertIn('tests.test_cmdline.extensions.DummyExtension', settingsdict) - self.assertIn('value=200', settingsdict['tests.test_cmdline.extensions.DummyExtension']) - self.assertIn('value=100', settingsdict['tests.test_cmdline.extensions.TestExtension']) + six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) + self.assertIn('value=200', settingsdict[EXT_PATH]) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 8d98d2cec..44b9b6df3 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -37,21 +37,22 @@ class SettingsAttributeTest(unittest.TestCase): self.assertEqual(self.attribute.value, 'value') self.assertEqual(self.attribute.priority, 10) - def test_set_per_key_priorities(self): - attribute = SettingsAttribute( - BaseSettings({'one': 10, 'two': 20}, 0), 0) + def test_overwrite_basesettings(self): + original_dict = {'one': 10, 'two': 20} + original_settings = BaseSettings(original_dict, 0) + attribute = SettingsAttribute(original_settings, 0) - new_dict = {'one': 11, 'two': 21} + new_dict = {'three': 11, 'four': 21} attribute.set(new_dict, 10) - self.assertEqual(attribute.value['one'], 11) - self.assertEqual(attribute.value['two'], 21) + self.assertIsInstance(attribute.value, BaseSettings) + six.assertCountEqual(self, attribute.value, new_dict) + six.assertCountEqual(self, original_settings, original_dict) - new_settings = BaseSettings() - new_settings.set('one', 12, 20) - new_settings.set('two', 12, 0) - attribute.set(new_settings, 0) - self.assertEqual(attribute.value['one'], 12) - self.assertEqual(attribute.value['two'], 21) + new_settings = BaseSettings({'five': 12}, 0) + attribute.set(new_settings, 0) # Insufficient priority + six.assertCountEqual(self, attribute.value, new_dict) + attribute.set(new_settings, 10) + six.assertCountEqual(self, attribute.value, new_settings) def test_repr(self): self.assertEqual(repr(self.attribute), From 4f364764aea624a3db9a6033b5dba8a70f93535d Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 10 Nov 2015 22:48:50 +0100 Subject: [PATCH 012/137] Simplify BaseSettings.__get__(), .getpriority() --- scrapy/settings/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index e62bdd08e..342d2585e 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -90,10 +90,9 @@ class BaseSettings(MutableMapping): self.update(values, priority) def __getitem__(self, opt_name): - value = None - if opt_name in self: - value = self.attributes[opt_name].value - return value + if opt_name not in self: + return None + return self.attributes[opt_name].value def __contains__(self, name): return name in self.attributes @@ -212,10 +211,9 @@ class BaseSettings(MutableMapping): :param name: the setting name :type name: string """ - prio = None - if name in self: - prio = self.attributes[name].priority - return prio + if name not in self: + return None + return self.attributes[name].priority def maxpriority(self): """ From 881bf19e6804c3b82543f4f23d6ed60e1a4bb651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 17:08:13 +0200 Subject: [PATCH 013/137] FormRequest: test case-insensitive type attribute --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 60fd855dd..593698dd2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -305,6 +305,19 @@ class FormRequestTest(RequestTest): request = FormRequest.from_response(response, url='/relative') self.assertEqual(request.url, 'http://example.com/relative') + def test_from_response_case_insensitive(self): + response = _buildresponse( + """
+ + + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs['clickable1'], ['clicked1']) + self.assertFalse('i1' in fs, fs) # xpath in _get_inputs() + self.assertFalse('clickable2' in fs, fs) # xpath in _get_clickable() + def test_from_response_submit_first_clickable(self): response = _buildresponse( """
From 650acad2b73c1abafe8ff95808b4d77b65ef67cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 17:57:46 +0200 Subject: [PATCH 014/137] FormRequest: fix case-insensitive type attributes --- scrapy/http/request/form.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4a9bd732e..ea4bfd564 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -107,8 +107,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type!="submit" and @type!="image" and @type!="reset"' - 'and ((@type!="checkbox" and @type!="radio") or @checked)]') + '|descendant::input[@type[' + ' translate(., "SUBMIT", "submit") != "submit"' + ' and translate(., "IMAGE", "image") !="image"' + ' and translate(., "RESET", "reset") != "reset"' + ' and (../@checked or (' + ' translate(., "CHECKBOX", "checkbox") != "checkbox"' + ' and translate(., "RADIO", "radio") != "radio"))]]') values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -151,9 +156,11 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [el for el in form.xpath('descendant::input[@type="submit"]' - '|descendant::button[@type="submit"]' - '|descendant::button[not(@type)]')] + clickables = [ + el for el in form.xpath( + 'descendant::*[(self::input or self::button)' + ' and translate(@type, "SUBMIT", "submit") = "submit"]' + '|descendant::button[not(@type)]')] if not clickables: return From 7a438a51b783f099894212abdefc95ad591fce3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:03:54 +0200 Subject: [PATCH 015/137] FormRequest: test default type (is text) --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 593698dd2..7964a6591 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -675,10 +675,11 @@ class FormRequestTest(RequestTest): + ''') req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']}) + self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']}) def test_from_response_input_hidden(self): res = _buildresponse( From 2d25eab0df4c5c8bb1894cbc45cc15059d726c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:05:07 +0200 Subject: [PATCH 016/137] FormRequest: 's default type must be text --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea4bfd564..8ff394602 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -107,7 +107,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type[' + '|descendant::input[not(@type) or @type[' ' translate(., "SUBMIT", "submit") != "submit"' ' and translate(., "IMAGE", "image") !="image"' ' and translate(., "RESET", "reset") != "reset"' From 395ef805eb74c1c575d32ab0c52d406f55c7d276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:54:02 +0200 Subject: [PATCH 017/137] FormRequest: test unicode xpath expr & exception --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7964a6591..f82b2de8d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,6 @@ import cgi import unittest +import re import six from six.moves import xmlrpc_client as xmlrpclib @@ -747,6 +748,18 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formxpath="//form/input[@name='abc']") + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + fs = _qs(r) + self.assertEqual(fs, {}) + + xpath = u"//form[@name='\u03b1']" + encoded = xpath if six.PY3 else xpath.encode('unicode_escape') + self.assertRaisesRegexp(ValueError, re.escape(encoded), + self.request_class.from_response, + response, formxpath=xpath) + def test_from_response_button_submit(self): response = _buildresponse( """
From 4f98be60be700e0c0c359f3cbaa1535ae9ece7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:56:05 +0200 Subject: [PATCH 018/137] FormRequest: fix unicode xpath exception --- scrapy/http/request/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8ff394602..948ad05c9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -85,7 +85,8 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape') + raise ValueError('No element found with %s' % encoded) # If we get here, it means that either formname was None # or invalid From ebfdb9bb03fd876bf519535dd57ec0816e0c3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Sat, 14 Nov 2015 23:24:07 +0200 Subject: [PATCH 019/137] readable xpath with exslt --- scrapy/http/request/form.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 948ad05c9..f623a5aa3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -109,12 +109,11 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' - ' translate(., "SUBMIT", "submit") != "submit"' - ' and translate(., "IMAGE", "image") !="image"' - ' and translate(., "RESET", "reset") != "reset"' - ' and (../@checked or (' - ' translate(., "CHECKBOX", "checkbox") != "checkbox"' - ' and translate(., "RADIO", "radio") != "radio"))]]') + ' not(re:test(., "^(?:submit|image|reset)$", "i"))' + ' and (../@checked or' + ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', + namespaces={ + "re": "http://exslt.org/regular-expressions"}) values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -160,8 +159,10 @@ def _get_clickable(clickdata, form): clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' - ' and translate(@type, "SUBMIT", "submit") = "submit"]' - '|descendant::button[not(@type)]')] + ' and re:test(@type, "^submit$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) + ] if not clickables: return From 0025d5a9439a1f737bccb8ac1dfc6ff85daabb85 Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 16 Nov 2015 07:30:17 +0800 Subject: [PATCH 020/137] Fixed minor grammar issues. --- docs/faq.rst | 2 +- docs/topics/broad-crawls.rst | 4 ++-- docs/topics/extensions.rst | 4 ++-- docs/topics/item-pipeline.rst | 2 +- docs/topics/items.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/selectors.rst | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2e61f44ee..3d2bd8d4d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -144,7 +144,7 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those message are thrown by the Offsite Spider Middleware, which is a spider +Those messages are thrown by the Offsite Spider Middleware, which is a spider middleware (enabled by default) whose purpose is to filter out requests to domains outside the ones covered by the spider. diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index aaf46bc92..79f0b3b53 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -34,7 +34,7 @@ These are some common properties often found in broad crawls: As said above, Scrapy default settings are optimized for focused crawls, not broad crawls. However, due to its asynchronous architecture, Scrapy is very -well suited for performing fast broad crawls. This page summarize some things +well suited for performing fast broad crawls. This page summarizes some things you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. @@ -46,7 +46,7 @@ Concurrency is the number of requests that are processed in parallel. There is a global limit and a per-domain limit. The default global concurrency limit in Scrapy is not suitable for crawling -many different domains in parallel, so you will want to increase it. How much +many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU you crawler will have available. A good starting point is ``100``, but the best way to find out is by doing some trials and identifying at what concurrency your Scrapy process gets CPU diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 11c0aadb6..f9e709514 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ Extensions use the :ref:`Scrapy settings ` to manage their settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, an +avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like `GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. @@ -143,7 +143,7 @@ Here is the code of such extension:: self.items_scraped += 1 if self.items_scraped % self.item_count == 0: logger.info("scraped %d items", self.items_scraped) - + .. _topics-extensions-ref: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index f74400b4d..28969be61 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -95,7 +95,7 @@ contain a price:: Write items to a JSON file -------------------------- -The following pipeline stores all scraped items (from all spiders) into a a +The following pipeline stores all scraped items (from all spiders) into a single ``items.jl`` file, containing one item per line serialized in JSON format:: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 21ec0ed8c..4a8f47e93 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -61,7 +61,7 @@ the example above. You can specify any kind of metadata for each field. There is no restriction on the values accepted by :class:`Field` objects. For this same reason, there is no reference list of all available metadata keys. Each key -defined in :class:`Field` objects could be used by a different components, and +defined in :class:`Field` objects could be used by a different component, and only those components know about it. You can also define and use any other :class:`Field` key in your project too, for your own needs. The main goal of :class:`Field` objects is to provide a way to define all field metadata in one diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 735137ea2..92590c180 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -97,7 +97,7 @@ subclasses): A real example -------------- -Let's see a concrete example of an hypothetical case of memory leaks. +Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 9ae34f423..60fe2267c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -228,7 +228,7 @@ with varying degrees of sophistication. Getting around those measures can be difficult and tricky, and may sometimes require special infrastructure. Please consider contacting `commercial support`_ if in doubt. -Here are some tips to keep in mind when dealing with these kind of sites: +Here are some tips to keep in mind when dealing with these kinds of sites: * rotate your user agent from a pool of well-known ones from browsers (google around to get a list of them) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 273cae0f8..8dc82dfe5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -579,7 +579,7 @@ Built-in Selectors reference is used together with ``text``. If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follow: + inferred from the response type as follows: * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type @@ -757,7 +757,7 @@ nodes can be accessed directly by their names:: Date: Thu, 26 Nov 2015 17:19:19 +0100 Subject: [PATCH 024/137] nextcall repetitive calls (heartbeats). --- scrapy/core/engine.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index eb2779b12..ef4403106 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,7 +7,7 @@ For more information see docs/topics/architecture.rst import logging from time import time -from twisted.internet import defer +from twisted.internet import defer, task from twisted.python.failure import Failure from scrapy import signals @@ -30,6 +30,7 @@ class Slot(object): self.close_if_idle = close_if_idle self.nextcall = nextcall self.scheduler = scheduler + self.heartbeat = task.LoopingCall(nextcall.schedule) def add_request(self, request): self.inprogress.add(request) @@ -47,6 +48,7 @@ class Slot(object): if self.closing and not self.inprogress: if self.nextcall: self.nextcall.cancel() + self.heartbeat.stop() self.closing.callback(None) @@ -113,7 +115,6 @@ class ExecutionEngine(object): return if self.paused: - slot.nextcall.schedule(5) return while not self._needs_backout(spider): @@ -254,6 +255,7 @@ class ExecutionEngine(object): self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) slot.nextcall.schedule() + slot.heartbeat.start(5) def _spider_idle(self, spider): """Called when a spider gets idle. This function is called when there @@ -267,7 +269,6 @@ class ExecutionEngine(object): spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \ for _, x in res): - self.slot.nextcall.schedule(5) return if self.spider_is_idle(spider): From c4d29ecaef534cdab31fb73ec83545010f7a5ad8 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 2 Dec 2015 17:05:44 +0100 Subject: [PATCH 025/137] Ignoring xlib/tx folder, depending on Twisted version. --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index f9ca3ab93..b0ac1badd 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ import glob import six import pytest +from twisted import version as twisted_version def _py_files(folder): @@ -26,6 +27,9 @@ collect_ignore = [ ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): + collect_ignore += _py_files("scrapy/xlib/tx") + if six.PY3: for line in open('tests/py3-ignores.txt'): From 016875fd513c66b7ee1bcc80a1f2ae0d6ffda2d2 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 15:30:06 +0300 Subject: [PATCH 026/137] added more verbosity for log and for exception when download is cancelled because of a size limit --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..78f3f74fa 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,11 +239,16 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - logger.error("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s).", - {'size': expected_size, 'maxsize': maxsize}) + error_message = ( + "Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize}).".format( + url=request.url, size=expected_size, maxsize=maxsize + ) + ) + logger.error(error_message) txresponse._transport._producer.loseConnection() - raise defer.CancelledError() + raise defer.CancelledError(error_message) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From 37289815ac98404458c43159abfd46dd54ed4b7b Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 16:15:50 +0300 Subject: [PATCH 027/137] code formatting fix --- scrapy/core/downloader/handlers/http11.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 78f3f74fa..9d075e877 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,13 +239,11 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ( - "Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize}).".format( - url=request.url, size=expected_size, maxsize=maxsize - ) - ) + error_message = ("Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize})." + ).format(url=request.url, size=expected_size, maxsize=maxsize) + logger.error(error_message) txresponse._transport._producer.loseConnection() raise defer.CancelledError(error_message) From f8ae99d18fb0365c340152e880245bb4340d7a1f Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Sat, 5 Dec 2015 09:48:17 -0400 Subject: [PATCH 028/137] DOC Removed pywin32 from install instructions as it's already declared as dependency. --- docs/intro/install.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index abb94c006..122de47f6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -44,13 +44,10 @@ Anaconda If you already have installed `Anaconda`_ or `Miniconda`_, the company `Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X. -To install Scrapy in Linux or OS X, use: +To install Scrapy using ``conda``, run:: conda install -c scrapinghub scrapy -To install Scrapy in Windows, use: - - conda install -c scrapinghub scrapy pywin32 Windows ------- From 4be4ef038ea8e206bdd3682dab3cb7b5184994eb Mon Sep 17 00:00:00 2001 From: orangain Date: Sat, 12 Dec 2015 16:21:00 +0900 Subject: [PATCH 029/137] DOC: Add captions to toctrees which appear in sidebar --- docs/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 0d21f5d40..4cb3eb741 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ First steps =========== .. toctree:: + :caption: First steps :hidden: intro/overview @@ -53,6 +54,7 @@ Basic concepts ============== .. toctree:: + :caption: Basic concepts :hidden: topics/commands @@ -110,6 +112,7 @@ Built-in services ================= .. toctree:: + :caption: Built-in services :hidden: topics/logging @@ -138,6 +141,7 @@ Solving specific problems ========================= .. toctree:: + :caption: Solving specific problems :hidden: faq @@ -203,6 +207,7 @@ Extending Scrapy ================ .. toctree:: + :caption: Extending Scrapy :hidden: topics/architecture @@ -240,6 +245,7 @@ All the rest ============ .. toctree:: + :caption: All the rest :hidden: news From 719b1353a7a49d13f1f54e37a85b8b692915c0c7 Mon Sep 17 00:00:00 2001 From: seales Date: Sun, 13 Dec 2015 19:39:48 -0800 Subject: [PATCH 030/137] Spelling fixes --- sep/sep-003.rst | 2 +- sep/sep-014.rst | 2 +- sep/sep-018.rst | 2 +- sep/sep-020.rst | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sep/sep-003.rst b/sep/sep-003.rst index 282574968..184839525 100644 --- a/sep/sep-003.rst +++ b/sep/sep-003.rst @@ -146,7 +146,7 @@ Default values p['numbers'] # returns [] -Accesing and changing nested item values +Accessing and changing nested item values ---------------------------------------- :: diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 98a31b1aa..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -54,7 +54,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. -This is an enhancemente to ``LinkExtractors`` which returns urls (links), +This is an enhancement to ``LinkExtractors`` which returns urls (links), Request Extractors return Request objects. Request Processors diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e30821917..aca7ac342 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -477,7 +477,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def should_follow(self, request, spider): info = self.spiders[spider] - # hostanme can be None for wrong urls (like javascript links) + # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or '' return bool(info.regex.search(host)) diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 7b2c043b7..49d068479 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -23,9 +23,9 @@ Rationale ========= There are certain markup patterns that lend themselves quite nicely to -automated parsing, for example the ```` tag outlilnes such a pattern +automated parsing, for example the ``
`` tag outlines such a pattern for populating a database table with the embedded ```` elements denoting -the rows and the furthur embedded ``
`` elements denoting the individual +the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader From bcce8d3d80462946483a403a1fb25b3e12229136 Mon Sep 17 00:00:00 2001 From: orangain Date: Thu, 17 Dec 2015 14:48:37 +0900 Subject: [PATCH 031/137] DOC: Update MetaRefreshMiddlware's setting variables * `REDIRECT_MAX_METAREFRESH_DELAY` has been deprecated and was renamed to `METAREFRESH_MAXDELAY`. * Merge duplicate documents about `METAREFRESH_MAXDELAY` appeared both in the settings page and the downloader-middlewares page. --- docs/topics/downloader-middleware.rst | 8 +++++--- docs/topics/settings.rst | 10 ---------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index cc0254d29..3641da231 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -787,14 +787,16 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. -.. setting:: REDIRECT_MAX_METAREFRESH_DELAY +.. setting:: METAREFRESH_MAXDELAY -REDIRECT_MAX_METAREFRESH_DELAY -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +METAREFRESH_MAXDELAY +^^^^^^^^^^^^^^^^^^^^ Default: ``100`` The maximum meta-refresh delay (in seconds) to follow the redirection. +Some sites use meta-refresh for redirecting to a session expired page, so we +restrict automatic redirection to the maximum delay. RetryMiddleware --------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 60b5f4585..cc070d8c0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -857,16 +857,6 @@ Defines the maximum times a request can be redirected. After this maximum the request's response is returned as is. We used Firefox default value for the same task. -.. setting:: REDIRECT_MAX_METAREFRESH_DELAY - -REDIRECT_MAX_METAREFRESH_DELAY ------------------------------- - -Default: ``100`` - -Some sites use meta-refresh for redirecting to a session expired page, so we -restrict automatic redirection to a maximum delay (in seconds) - .. setting:: REDIRECT_PRIORITY_ADJUST REDIRECT_PRIORITY_ADJUST From 4f49aab7c068c553c5dcb2935ee3e4df6c0b71aa Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 18 Dec 2015 16:16:05 -0500 Subject: [PATCH 032/137] BF: robustify _monkeypatches check for twisted - str() name first (Closes: #1634) In my case, while running datalad tests using nose, scrapy was failing since v was None --- scrapy/_monkeypatches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 782891326..60e0de1f2 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -20,6 +20,6 @@ if sys.version_info[0] == 2: import twisted.persisted.styles # NOQA # Remove only entries with twisted serializers for non-twisted types. for k, v in frozenset(copyreg.dispatch_table.items()): - if not getattr(k, '__module__', '').startswith('twisted') \ - and getattr(v, '__module__', '').startswith('twisted'): + if not str(getattr(k, '__module__', '')).startswith('twisted') \ + and str(getattr(v, '__module__', '')).startswith('twisted'): copyreg.dispatch_table.pop(k) From f57121c77be82ad7521c3f4ed5cda17ada118ef4 Mon Sep 17 00:00:00 2001 From: Aron Bordin Date: Wed, 30 Dec 2015 13:10:13 -0200 Subject: [PATCH 033/137] show download warnsize once --- scrapy/core/downloader/handlers/http11.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..403accef8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -292,6 +292,7 @@ class _ResponseReader(protocol.Protocol): self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize + self._reached_warnsize = False self._bytes_received = 0 def dataReceived(self, bodyBytes): @@ -305,11 +306,12 @@ class _ResponseReader(protocol.Protocol): 'maxsize': self._maxsize}) self._finished.cancel() - if self._warnsize and self._bytes_received > self._warnsize: - logger.warning("Received (%(bytes)s) bytes larger than download " - "warn size (%(warnsize)s).", - {'bytes': self._bytes_received, - 'warnsize': self._warnsize}) + if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: + self._reached_warnsize = True + logger.warning("Received more bytes than download " + "warn size (%(warnsize)s) in request %(request)s.", + {'warnsize': self._warnsize, + 'request': self._request}) def connectionLost(self, reason): if self._finished.called: From 1b435b2887ac4e19159cd4416f9396e022d0382a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 30 Dec 2015 15:43:04 -0300 Subject: [PATCH 034/137] Add 1.0.4 release notes --- docs/news.rst | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5df1b1a6a..4d7dc4d41 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,58 @@ Release notes ============= +1.0.4 (2015-12-30) +------------------ + +- Ignoring xlib/tx folder, depending on Twisted version. (:commit:`7dfa979`) +- Run on new travis-ci infra (:commit:`6e42f0b`) +- Spelling fixes (:commit:`823a1cc`) +- escape nodename in xmliter regex (:commit:`da3c155`) +- test xml nodename with dots (:commit:`4418fc3`) +- TST don't use broken Pillow version in tests (:commit:`a55078c`) +- disable log on version command. closes #1426 (:commit:`86fc330`) +- disable log on startproject command (:commit:`db4c9fe`) +- Add PyPI download stats badge (:commit:`df2b944`) +- don't run tests twice on Travis if a PR is made from a scrapy/scrapy branch (:commit:`a83ab41`) +- Add Python 3 porting status badge to the README (:commit:`73ac80d`) +- fixed RFPDupeFilter persistence (:commit:`97d080e`) +- TST a test to show that dupefilter persistence is not working (:commit:`97f2fb3`) +- explicit close file on file:// scheme handler (:commit:`d9b4850`) +- Disable dupefilter in shell (:commit:`c0d0734`) +- DOC: Add captions to toctrees which appear in sidebar (:commit:`aa239ad`) +- DOC Removed pywin32 from install instructions as it's already declared as dependency. (:commit:`10eb400`) +- Added installation notes about using Conda for Windows and other OSes. (:commit:`1c3600a`) +- Fixed minor grammar issues. (:commit:`7f4ddd5`) +- fixed a typo in the documentation. (:commit:`b71f677`) +- Version 1 now exists (:commit:`5456c0e`) +- fix another invalid xpath error (:commit:`0a1366e`) +- fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`) +- Typos corrections (:commit:`7067117`) +- fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`) +- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`) +- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`) +- Reference Homebrew's homepage for installation instructions (:commit:`1925db1`) +- Add oldest supported tox version to contributing docs (:commit:`5d10d6d`) +- Note in install docs about pip being already included in python>=2.7.9 (:commit:`85c980e`) +- Add non-python dependencies to Ubuntu install section in the docs (:commit:`fbd010d`) +- Add OS X installation section to docs (:commit:`d8f4cba`) +- DOC(ENH): specify path to rtd theme explicitly (:commit:`de73b1a`) +- minor: scrapy.Spider docs grammar (:commit:`1ddcc7b`) +- Make common practices sample code match the comments (:commit:`1b85bcf`) +- nextcall repetitive calls (heartbeats). (:commit:`55f7104`) +- Backport fix compatibility with Twisted 15.4.0 (:commit:`b262411`) +- pin pytest to 2.7.3 (:commit:`a6535c2`) +- Merge pull request #1512 from mgedmin/patch-1 (:commit:`8876111`) +- Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`) +- Typo (:commit:`f8d0682`) +- Fix list formatting (:commit:`5f83a93`) +- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`) +- Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`) +- Update tutorial.rst (:commit:`fbc1f25`) +- Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`) +- Small grammatical change (:commit:`8752294`) +- Add openssl version to version command (:commit:`13c45ac`) + 1.0.3 (2015-08-11) ------------------ From c702c5301523e57ce276d80c8106f9a52853948d Mon Sep 17 00:00:00 2001 From: palego Date: Sun, 3 Jan 2016 14:33:42 +0100 Subject: [PATCH 035/137] change os.mknod() for open() os.mknod() is a privileged command on OS X, making the test fail --- tests/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 5755b3881..8edccd4bd 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -83,7 +83,8 @@ class StartprojectTemplatesTest(ProjectTest): def test_startproject_template_override(self): copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) - os.mknod(join(self.tmpl_proj, 'root_template')) + with open(join(self.tmpl_proj, 'root_template'), 'w'): + pass assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] From d03d262f5290d7a439ac797947413894f158f425 Mon Sep 17 00:00:00 2001 From: palego Date: Mon, 4 Jan 2016 10:00:13 +0100 Subject: [PATCH 036/137] indentation --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 803ed5fc0..3324aa21a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -56,7 +56,7 @@ class RetryMiddleware(object): def process_exception(self, request, exception, spider): if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \ and not request.meta.get('dont_retry', False): - return self._retry(request, exception, spider) + return self._retry(request, exception, spider) def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 From 2abc9bc901491b24ea1a35058ae2b86e44492c88 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Jr Date: Wed, 6 Jan 2016 10:29:45 -0200 Subject: [PATCH 037/137] Update deprecated examples * update the scrapy.org example to deal with the new layout. * replaced slashdot.org by reddit.com, because it seems that slashdot is blocking requests. --- docs/topics/shell.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 2b118bfbd..3569cbf37 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -106,10 +106,10 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -http://scrapy.org page, and then proceed to scrape the http://slashdot.org -page. Finally, we modify the (Slashdot) request method to POST and re-fetch it -getting a HTTP 405 (method not allowed) error. We end the session by typing -Ctrl-D (in Unix systems) or Ctrl-Z in Windows. +http://scrapy.org page, and then proceed to scrape the http://reddit.com +page. Finally, we modify the (Reddit) request method to POST and re-fetch it +getting an error. We end the session by typing Ctrl-D (in Unix systems) or +Ctrl-Z in Windows. Keep in mind that the data extracted here may not be the same when you try it, as those pages are not static and could have changed by the time you test this. @@ -140,24 +140,24 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: - >>> response.xpath("//h1/text()").extract()[0] - u'Meet Scrapy' + >>> response.xpath('//title/text()').extract_first() + u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' - >>> fetch("http://slashdot.org") + >>> fetch("http://reddit.com") [s] Available Scrapy objects: - [s] crawler + [s] crawler [s] item {} - [s] request - [s] response <200 http://slashdot.org> - [s] settings - [s] spider + [s] request + [s] response <200 https://www.reddit.com/> + [s] settings + [s] spider [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() - [u'Slashdot: News for nerds, stuff that matters'] + [u'reddit: the front page of the internet'] >>> request = request.replace(method="POST") From d4872940dbb450fcd0fa688766af0aa6dfccc861 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 6 Jan 2016 21:21:21 +0100 Subject: [PATCH 038/137] PY3: port utils/iterators --- scrapy/utils/iterators.py | 14 ++++++++++---- tests/py3-ignores.txt | 1 - tests/test_utils_iterators.py | 21 +++++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c0d93f7a9..ed286f5c5 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -8,6 +8,8 @@ except ImportError: from io import BytesIO import six +if six.PY3: + from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector @@ -65,7 +67,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, unicode) + self._is_unicode = isinstance(self._text, six.text_type) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -94,7 +96,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. - + quotechar is the character used to enclosure fields on the given obj. """ @@ -102,7 +104,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _getrow(csv_r): return [to_unicode(field, encoding) for field in next(csv_r)] - lines = BytesIO(_body_or_str(obj, unicode=False)) + # Python 3 csv reader input object needs to return strings + if six.PY3: + lines = StringIO(_body_or_str(obj, unicode=True)) + else: + lines = BytesIO(_body_or_str(obj, unicode=False)) kwargs = {} if delimiter: kwargs["delimiter"] = delimiter @@ -125,7 +131,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types)), \ + assert isinstance(obj, (Response, six.string_types, six.binary_type)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 55ed75c92..015578e1e 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -16,7 +16,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_iterators.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 590c53302..d42ed2c91 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,5 @@ import os +import six from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml @@ -99,7 +100,7 @@ class XmliterTestCase(unittest.TestCase): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( - self.xmliter(response, 'item').next().extract(), + next(self.xmliter(response, 'item')).extract(), u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -189,11 +190,11 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assert_(all((isinstance(k, unicode) for k in result_row.keys()))) - self.assert_(all((isinstance(v, unicode) for v in result_row.values()))) + self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys()))) + self.assert_(all((isinstance(v, six.text_type) for v in result_row.values()))) def test_csviter_delimiter(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -205,8 +206,8 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') - body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|') - + body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|') + response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") @@ -237,7 +238,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') response = Response(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -249,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() - headers, body = sample[0].split(','), '\n'.join(sample[1:]) + headers, body = sample[0].split(b','), b'\n'.join(sample[1:]) response = TextResponse(url="http://example.com/", body=body) - csv = csviter(response, headers=headers) + csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, @@ -262,7 +263,7 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') - body = '\n'.join((body, 'a,b', 'a,b,c,d')) + body = b'\n'.join((body, b'a,b', b'a,b,c,d')) response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) From 6ddd8147382348f4796ac905f0357925dfa81da1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jan 2016 10:48:45 +0100 Subject: [PATCH 039/137] Support unicode tags in xml iterators (fixes #1665) --- scrapy/utils/iterators.py | 10 +++---- tests/test_utils_iterators.py | 51 ++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c0d93f7a9..bec985062 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def xmliter(obj, nodename): """Return a iterator of Selector's over all nodes of a XML document, - given tha name of the node to iterate. Useful for parsing XML feeds. + given the name of the node to iterate. Useful for parsing XML feeds. obj can be: - a Response object @@ -36,7 +36,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL) + r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -49,7 +49,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node) + nodetext = etree.tostring(node, encoding='unicode') node.clear() xs = Selector(text=nodetext, type='xml') if namespace: @@ -94,7 +94,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. - + quotechar is the character used to enclosure fields on the given obj. """ @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types)), \ + assert isinstance(obj, (Response, six.string_types, bytes)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 590c53302..8dceed7cc 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os from twisted.trial import unittest @@ -45,6 +46,54 @@ class XmliterTestCase(unittest.TestCase): for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) + def test_xmliter_unicode(self): + # example taken from https://github.com/scrapy/scrapy/issues/1665 + body = """ + <þingflokkar> + <þingflokkur id="26"> + + + - + + + + 80 + + + <þingflokkur id="21"> + Alþýðubandalag + + Ab + Alþb. + + + 76 + 123 + + + <þingflokkur id="27"> + Alþýðuflokkur + + A + Alþfl. + + + 27 + 120 + + + """ + response = XmlResponse(url="http://example.com", body=body) + attrs = [] + for x in self.xmliter(response, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" @@ -206,7 +255,7 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|') - + response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") From d7d4ef67a697243143df969e32b8ed956394f4fb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jan 2016 11:08:49 +0100 Subject: [PATCH 040/137] Changes following comments --- scrapy/utils/iterators.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index ed286f5c5..ce59c9719 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,15 +1,12 @@ import re import csv import logging - try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import StringIO import six -if six.PY3: - from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector @@ -131,7 +128,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types, six.binary_type)), \ + assert isinstance(obj, (Response, six.string_types, bytes)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: From 9fad25f3d14091d250cc4b1d668befca00c30ef0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 11:42:41 +0100 Subject: [PATCH 041/137] Use explicit Unicode and bytes for XML body in tests --- scrapy/utils/iterators.py | 9 ++++++--- tests/test_utils_iterators.py | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c215a0bdd..69c7f2c23 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding='unicode') + nodetext = etree.tostring(node, encoding=six.text_type) node.clear() xs = Selector(text=nodetext, type='xml') if namespace: @@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types, bytes)), \ - "obj must be Response or basestring, not %s" % type(obj).__name__ + expected_types = (Response, six.text_type, six.binary_type) + assert isinstance(obj, expected_types), \ + "obj must be %s, not %s" % ( + " or ".join(t.__name__ for t in expected_types), + type(obj).__name__) if isinstance(obj, Response): if not unicode: return obj.body diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index de103fea5..74c22d420 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -49,7 +49,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = """ + body = u""" <þingflokkar> <þingflokkur id="26"> @@ -84,7 +84,22 @@ class XmliterTestCase(unittest.TestCase): """ - response = XmlResponse(url="http://example.com", body=body) + + # with bytes + response = XmlResponse(url="http://example.com", body=body.encode('utf-8')) + attrs = [] + for x in self.xmliter(response, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) + + # Unicode body needs encoding information + response = XmlResponse(url="http://example.com", body=body, encoding='utf-8') attrs = [] for x in self.xmliter(response, u'þingflokkur'): attrs.append((x.xpath('@id').extract(), From d4c7d72b2b6fc20c1df0f697dd60801b93848628 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:13:47 +0100 Subject: [PATCH 042/137] Add tests for input type in xmliter calls --- tests/test_utils_iterators.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 74c22d420..8c4d6cf59 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -160,6 +160,10 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + def test_xmliter_objtype_exception(self): + i = self.xmliter(42, 'product') + self.assertRaises(AssertionError, next, i) + def test_xmliter_encoding(self): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) @@ -233,6 +237,9 @@ class LxmlXmliterTestCase(XmliterTestCase): node = next(my_iter) self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table']) + def test_xmliter_objtype_exception(self): + i = self.xmliter(42, 'product') + self.assertRaises(TypeError, next, i) class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') From 1347015a80b9d8dafe0e1ac067d65b7e0e7c3f84 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:32:28 +0100 Subject: [PATCH 043/137] Refactored test code --- tests/test_utils_iterators.py | 37 +++++++++++++---------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 8c4d6cf59..b2e3889a4 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -85,31 +85,22 @@ class XmliterTestCase(unittest.TestCase): """ - # with bytes - response = XmlResponse(url="http://example.com", body=body.encode('utf-8')) - attrs = [] - for x in self.xmliter(response, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + for r in ( + # with bytes + XmlResponse(url="http://example.com", body=body.encode('utf-8')), + # Unicode body needs encoding information + XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + attrs = [] + for x in self.xmliter(r, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) - # Unicode body needs encoding information - response = XmlResponse(url="http://example.com", body=body, encoding='utf-8') - attrs = [] - for x in self.xmliter(response, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) - - self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" From a93d49a64ca170d98de98ee44a181ced04a23bea Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:47:42 +0100 Subject: [PATCH 044/137] Add Python 3.5 tox env and Python 3.4 tests in Travis CI --- .travis.yml | 1 + tox.ini | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index e857abbd8..65cfaad03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 + - TOXENV=py34 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index eae7e8e47..b8d45d5b9 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,10 @@ deps = basepython = python3.4 deps = {[testenv:py33]deps} +[testenv:py35] +basepython = python3.5 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps = From f3889b0bce84cbd46852799a7bc95128c4c3b2d5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 11:41:49 +0300 Subject: [PATCH 045/137] py3 compat: encode delimiter, method and path in ScrapyHTTPPageGetter --- scrapy/core/downloader/webclient.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index add5576ef..c335939d0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,6 +7,7 @@ from twisted.internet import defer from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes from scrapy.responsetypes import responsetypes @@ -29,13 +30,17 @@ def _parse(url): class ScrapyHTTPPageGetter(HTTPClient): - delimiter = '\n' + delimiter = b'\n' def connectionMade(self): self.headers = Headers() # bucket for response headers # Method command - self.sendCommand(self.factory.method, self.factory.path) + self.sendCommand( + to_bytes(self.factory.method, encoding='ascii'), + # XXX - do we need to percent-encode path somewhere? + # https://en.wikipedia.org/wiki/Percent-encoding#Character_data + to_bytes(self.factory.path)) # Headers for key, values in self.factory.headers.items(): for value in values: From 9f2be23a3957ff17b252c47958affb9f5fcd6e72 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 11:42:23 +0300 Subject: [PATCH 046/137] webclient tests, py3: fix setUp, pass test_getPage --- tests/test_webclient.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index e0b46286a..d56c9f683 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -14,18 +14,21 @@ from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client from scrapy.http import Request, Headers +from scrapy.utils.python import to_bytes, to_unicode def getPage(url, contextFactory=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" - def _clientfactory(*args, **kwargs): + def _clientfactory(url, *args, **kwargs): + url = to_unicode(url) timeout = kwargs.pop('timeout', 0) - f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout) + f = client.ScrapyHTTPClientFactory( + Request(url, *args, **kwargs), timeout=timeout) f.deferred.addCallback(lambda r: r.body) return f from twisted.web.client import _makeGetterFactory - return _makeGetterFactory(url, _clientfactory, + return _makeGetterFactory(to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs).deferred @@ -212,7 +215,7 @@ class WebClientTestCase(unittest.TestCase): def setUp(self): name = self.mktemp() os.mkdir(name) - FilePath(name).child("file").setContent("0123456789") + FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) r.putChild("redirect", util.Redirect("/file")) r.putChild("wait", ForeverTakingResource()) @@ -250,7 +253,7 @@ class WebClientTestCase(unittest.TestCase): the body of the response if the default method B{GET} is used. """ d = getPage(self.getURL("file")) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d From 73ff87c1dc05e98104f854d16792e302625c5e98 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:03:08 +0300 Subject: [PATCH 047/137] decode body from utf-8, as scrapy stores body as bytes, and twisted has already converted to unicode --- scrapy/core/downloader/webclient.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c335939d0..ba7bd798c 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -132,6 +132,9 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self.url) + # XXX - scrapy response stores body as bytes, + # but maybe it makes sense to be able to store unicode? + body = to_bytes(body) return respcls(url=self.url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): From 1d5ab671833b1e054bf15a1b562a33b5ac5f1af7 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:04:26 +0300 Subject: [PATCH 048/137] pass test_getPageHead on py3 --- tests/test_webclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d56c9f683..3784aa401 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -266,8 +266,8 @@ class WebClientTestCase(unittest.TestCase): def _getPage(method): return getPage(self.getURL("file"), method=method) return defer.gatherResults([ - _getPage("head").addCallback(self.assertEqual, ""), - _getPage("HEAD").addCallback(self.assertEqual, "")]) + _getPage("head").addCallback(self.assertEqual, b""), + _getPage("HEAD").addCallback(self.assertEqual, b"")]) def test_timeoutNotTriggering(self): From 945674eb8f351e71aea090f1e94c3fa337c52cf8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:25:54 +0300 Subject: [PATCH 049/137] pass test_externalUnicodeInterference - the logic for py3 is clearly inverse of what was expected in this test, as scrapy Request url must be unicode --- tests/test_webclient.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3784aa401..84717e5eb 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -3,6 +3,7 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ import os +import six from six.moves.urllib.parse import urlparse from twisted.trial import unittest @@ -75,8 +76,10 @@ class ParseUrlTestCase(unittest.TestCase): elements of its return tuple, even when passed an URL which has previously been passed to L{urlparse} as a C{unicode} string. """ - badInput = u'http://example.com/path' - goodInput = badInput.encode('ascii') + goodInput = u'http://example.com/path' + badInput = goodInput.encode('ascii') + if six.PY2: + goodInput, badInput = badInput, goodInput urlparse(badInput) scheme, netloc, host, port, path = self._parse(goodInput) self.assertTrue(isinstance(scheme, str)) From 325b6af6c2c55bcf47fe339cfe139b357eb1673e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:29:59 +0300 Subject: [PATCH 050/137] fix ScrapyHTTPPageGetterTests for py3 - we expect bytes here --- tests/test_webclient.py | 78 ++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 84717e5eb..84b1011b8 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -105,22 +105,22 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): 'Useful': 'value'})) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Content-Length: 9\r\n" - "Useful: value\r\n" - "Connection: close\r\n" - "User-Agent: fooble\r\n" - "Host: example.net\r\n" - "Cookie: blah blah\r\n" - "\r\n" - "some data") + b"GET /bar HTTP/1.0\r\n" + b"Content-Length: 9\r\n" + b"Useful: value\r\n" + b"Connection: close\r\n" + b"User-Agent: fooble\r\n" + b"Host: example.net\r\n" + b"Cookie: blah blah\r\n" + b"\r\n" + b"some data") # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "\r\n") + b"GET /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"\r\n") # test a simple POST with body and content-type factory = client.ScrapyHTTPClientFactory(Request( @@ -130,13 +130,13 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers={'Content-Type': 'application/x-www-form-urlencoded'})) self._test(factory, - "POST /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "Connection: close\r\n" - "Content-Type: application/x-www-form-urlencoded\r\n" - "Content-Length: 10\r\n" - "\r\n" - "name=value") + b"POST /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"Connection: close\r\n" + b"Content-Type: application/x-www-form-urlencoded\r\n" + b"Content-Length: 10\r\n" + b"\r\n" + b"name=value") # test a POST method with no body provided factory = client.ScrapyHTTPClientFactory(Request( @@ -145,10 +145,10 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): )) self._test(factory, - "POST /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "Content-Length: 0\r\n" - "\r\n") + b"POST /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"Content-Length: 0\r\n" + b"\r\n") # test with single and multivalued headers factory = client.ScrapyHTTPClientFactory(Request( @@ -159,12 +159,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): })) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "X-Meta-Multivalued: value1\r\n" - "X-Meta-Multivalued: value2\r\n" - "X-Meta-Single: single\r\n" - "\r\n") + b"GET /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"X-Meta-Multivalued: value1\r\n" + b"X-Meta-Multivalued: value2\r\n" + b"X-Meta-Single: single\r\n" + b"\r\n") # same test with single and multivalued headers but using Headers class factory = client.ScrapyHTTPClientFactory(Request( @@ -175,12 +175,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }))) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "X-Meta-Multivalued: value1\r\n" - "X-Meta-Multivalued: value2\r\n" - "X-Meta-Single: single\r\n" - "\r\n") + b"GET /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"X-Meta-Multivalued: value1\r\n" + b"X-Meta-Multivalued: value2\r\n" + b"X-Meta-Single: single\r\n" + b"\r\n") def _test(self, factory, testvalue): transport = StringTransport() @@ -199,10 +199,10 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.headers = Headers() - protocol.dataReceived("HTTP/1.0 200 OK\n") - protocol.dataReceived("Hello: World\n") - protocol.dataReceived("Foo: Bar\n") - protocol.dataReceived("\n") + protocol.dataReceived(b"HTTP/1.0 200 OK\n") + protocol.dataReceived(b"Hello: World\n") + protocol.dataReceived(b"Foo: Bar\n") + protocol.dataReceived(b"\n") self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']})) From 85b0e6c9c766218bd48268f0c115c77b3886e539 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 14 Jan 2016 10:50:51 +0100 Subject: [PATCH 051/137] Travis: run tox with Python 3.5 + add Python 3.5 tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65cfaad03..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 2.7 +python: 3.5 sudo: false branches: only: @@ -9,7 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 - - TOXENV=py34 + - TOXENV=py35 - TOXENV=docs install: - pip install -U tox twine wheel codecov From ae4aa2c3b24a967556f602a031520c535ebeb77d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:04:10 +0300 Subject: [PATCH 052/137] py3 test fix: putChild expects bytes as path --- tests/test_webclient.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 84b1011b8..02e24de05 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -220,13 +220,13 @@ class WebClientTestCase(unittest.TestCase): os.mkdir(name) FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) - r.putChild("redirect", util.Redirect("/file")) - r.putChild("wait", ForeverTakingResource()) - r.putChild("error", ErrorResource()) - r.putChild("nolength", NoLengthResource()) - r.putChild("host", HostHeaderResource()) - r.putChild("payload", PayloadResource()) - r.putChild("broken", BrokenDownloadResource()) + r.putChild(b"redirect", util.Redirect("/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"error", ErrorResource()) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = self._listen(self.wrapper) From b5f9bc8499293aced54f54250a18e148af356267 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:04:45 +0300 Subject: [PATCH 053/137] py3 test fixes in test_webclient - expect bytes as page body --- tests/test_webclient.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 02e24de05..05038c407 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -240,14 +240,17 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s) + return getPage(self.getURL("payload"), body=s).addCallback( + self.assertEquals, to_bytes(s)) def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ - getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno), - getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")]) + getPage(self.getURL("host")).addCallback( + self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)), + getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( + self.assertEquals, to_bytes("www.example.com"))]) def test_getPage(self): @@ -280,7 +283,8 @@ class WebClientTestCase(unittest.TestCase): called back with the contents of the page. """ d = getPage(self.getURL("host"), timeout=100) - d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno) + d.addCallback( + self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)) return d @@ -309,7 +313,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assert_('404 - No Such Resource' in pageData) + self.assert_(b'404 - No Such Resource' in pageData) def testFactoryInfo(self): url = self.getURL('file') @@ -329,6 +333,6 @@ class WebClientTestCase(unittest.TestCase): def _cbRedirect(self, pageData): self.assertEquals(pageData, - '\n\n \n \n' - ' \n \n ' - 'click here\n \n\n') + b'\n\n \n \n' + b' \n \n ' + b'click here\n \n\n') From 88f55312af0d9551f00686d79eaaf1c122740aa1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:43:37 +0300 Subject: [PATCH 054/137] py3 fix in testFactoryInfo - factory attirbutes are bytes in twisted --- tests/test_webclient.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 05038c407..0a31d8b5d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -323,10 +323,10 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEquals(factory.status, '200') - self.assert_(factory.version.startswith('HTTP/')) - self.assertEquals(factory.message, 'OK') - self.assertEquals(factory.response_headers['content-length'], '10') + self.assertEquals(factory.status, b'200') + self.assert_(factory.version.startswith(b'HTTP/')) + self.assertEquals(factory.message, b'OK') + self.assertEquals(factory.response_headers[b'content-length'], b'10') def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) From 30c7b4e4cc5411af1015aa292e20e4c453607428 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:44:14 +0300 Subject: [PATCH 055/137] py3 compat in test_timeoutTriggering cleanup --- tests/test_webclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0a31d8b5d..fa0083d44 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -300,7 +300,7 @@ class WebClientTestCase(unittest.TestCase): def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything. - connected = self.wrapper.protocols.keys() + connected = list(six.iterkeys(self.wrapper.protocols)) # There might be nothing here if the server managed to already see # that the connection was lost. if connected: From 01783561781e1620580e0f1b66f32a514ac2b960 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:46:46 +0300 Subject: [PATCH 056/137] py3 fix testRedirect: url is bytes here --- tests/test_webclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index fa0083d44..66f8ed4cf 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -220,7 +220,7 @@ class WebClientTestCase(unittest.TestCase): os.mkdir(name) FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) - r.putChild(b"redirect", util.Redirect("/file")) + r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"error", ErrorResource()) r.putChild(b"nolength", NoLengthResource()) From 6a412d25037d40dfbcc6392b6a862488c153e46f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:48:48 +0300 Subject: [PATCH 057/137] all tests pass in test_webclient.py on py3 - removing from py3-ignores --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 015578e1e..9e75ecf92 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -17,7 +17,6 @@ tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py tests/test_utils_template.py -tests/test_webclient.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py @@ -29,7 +28,6 @@ scrapy/core/downloader/handlers/s3.py scrapy/core/downloader/handlers/http11.py scrapy/core/downloader/handlers/http.py scrapy/core/downloader/handlers/ftp.py -scrapy/core/downloader/webclient.py scrapy/pipelines/images.py scrapy/pipelines/files.py scrapy/linkextractors/sgml.py From e5fb6094384f8d96e8a677b5fd9f2e6bfe2b97d3 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 14:45:02 +0300 Subject: [PATCH 058/137] make ScrapyHTTPClientFactory comply to twisted HTTPClientFactory protocol - use bytes (encoding are likely wrong at this stage) --- scrapy/core/downloader/webclient.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ba7bd798c..841322bdd 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,13 +12,15 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): + b = lambda x: to_bytes(x, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - host = parsed.hostname + path = to_bytes(path) # FIXME + host = b(parsed.hostname) # FIXME port = parsed.port - scheme = parsed.scheme - netloc = parsed.netloc + scheme = b(parsed.scheme) + netloc = b(parsed.netloc) # FIXME - host + port if port is None: - port = 443 if scheme == 'https' else 80 + port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path @@ -36,11 +38,7 @@ class ScrapyHTTPPageGetter(HTTPClient): self.headers = Headers() # bucket for response headers # Method command - self.sendCommand( - to_bytes(self.factory.method, encoding='ascii'), - # XXX - do we need to percent-encode path somewhere? - # https://en.wikipedia.org/wiki/Percent-encoding#Character_data - to_bytes(self.factory.path)) + self.sendCommand(self.factory.method, self.factory.path) # Headers for key, values in self.factory.headers.items(): for value in values: @@ -96,8 +94,10 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): afterFoundGet = False def __init__(self, request, timeout=180): - self.url = urldefrag(request.url)[0] - self.method = request.method + self._url = urldefrag(request.url)[0] + # converting to bytes to comply to Twisted interface + self.url = to_bytes(self._url) # FIXME + self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers) self.response_headers = None @@ -131,11 +131,11 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): request.meta['download_latency'] = self.headers_time-self.start_time status = int(self.status) headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self.url) + respcls = responsetypes.from_args(headers=headers, url=self._url) # XXX - scrapy response stores body as bytes, # but maybe it makes sense to be able to store unicode? body = to_bytes(body) - return respcls(url=self.url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) From ac2cf191d1868f19897a108a861e3170485a676a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 14:50:58 +0300 Subject: [PATCH 059/137] py3: remove comments, utf-8 is fine here: as twisted ultimately uses urllib.parse.quote that assepts bytes and assumes utf-8 --- scrapy/core/downloader/webclient.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 841322bdd..2d848aee0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,13 +12,12 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): - b = lambda x: to_bytes(x, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - path = to_bytes(path) # FIXME - host = b(parsed.hostname) # FIXME + path = to_bytes(path) + host = to_bytes(parsed.hostname) port = parsed.port - scheme = b(parsed.scheme) - netloc = b(parsed.netloc) # FIXME - host + port + scheme = to_bytes(parsed.scheme, encoding='ascii') + netloc = to_bytes(parsed.netloc) if port is None: port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path @@ -96,7 +95,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url) # FIXME + self.url = to_bytes(self._url) self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers) From 8df35bcac6190ab92635e46009c2e099c8e6010d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 14:52:44 +0300 Subject: [PATCH 060/137] rm note to self: to be discussed in PR --- scrapy/core/downloader/webclient.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 2d848aee0..15d14ae49 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -131,8 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - # XXX - scrapy response stores body as bytes, - # but maybe it makes sense to be able to store unicode? body = to_bytes(body) return respcls(url=self._url, status=status, headers=headers, body=body) From 5c2241ccc7a886ea48ce12c2fb67cee5db24ce4e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 15:30:28 +0300 Subject: [PATCH 061/137] py3: fix webclient tests after making ScrapyHTTPClientFactory use bytes as in twisted --- scrapy/core/downloader/webclient.py | 4 ++-- tests/test_webclient.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 15d14ae49..d2cdd6f98 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -65,7 +65,7 @@ class ScrapyHTTPPageGetter(HTTPClient): self.factory.noPage(reason) def handleResponse(self, response): - if self.factory.method.upper() == 'HEAD': + if self.factory.method.upper() == b'HEAD': self.factory.page('') elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) @@ -123,7 +123,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") # Content-Length must be specified in POST method even with no body - elif self.method == 'POST': + elif self.method == b'POST': self.headers['Content-Length'] = 0 def _build_response(self, body, request): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 66f8ed4cf..412e10c89 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -68,6 +68,8 @@ class ParseUrlTestCase(unittest.TestCase): ) for url, test in tests: + test = tuple( + to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEquals(client._parse(url), test, url) def test_externalUnicodeInterference(self): @@ -82,10 +84,10 @@ class ParseUrlTestCase(unittest.TestCase): goodInput, badInput = badInput, goodInput urlparse(badInput) scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, str)) - self.assertTrue(isinstance(netloc, str)) - self.assertTrue(isinstance(host, str)) - self.assertTrue(isinstance(path, str)) + self.assertTrue(isinstance(scheme, bytes)) + self.assertTrue(isinstance(netloc, bytes)) + self.assertTrue(isinstance(host, bytes)) + self.assertTrue(isinstance(path, bytes)) self.assertTrue(isinstance(port, int)) @@ -317,9 +319,9 @@ class WebClientTestCase(unittest.TestCase): def testFactoryInfo(self): url = self.getURL('file') - scheme, netloc, host, port, path = client._parse(url) + _, _, host, port, _ = client._parse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(host, port, factory) + reactor.connectTCP(to_unicode(host), port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): From 32bb5b682a7cf4f2baca78914f18297c120eef2d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:11:16 +0300 Subject: [PATCH 062/137] fix import of test_downloader_handlers.py: use @implementer, move failing on py3 imports into corresponding tests --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_downloader_handlers.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..7c937a036 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -6,7 +6,7 @@ from io import BytesIO from time import time from six.moves.urllib.parse import urldefrag -from zope.interface import implements +from zope.interface import implementer from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH @@ -265,8 +265,8 @@ class ScrapyAgent(object): return respcls(url=url, status=status, headers=headers, body=body, flags=flags) +@implementer(IBodyProducer) class _RequestBodyProducer(object): - implements(IBodyProducer) def __init__(self, body): self.body = body diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d2a349b40..5f1703c5c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -10,9 +10,7 @@ from twisted.web import server, static, util, resource from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource -from twisted.protocols.ftp import FTPRealm, FTPFactory from twisted.cred import portal, checkers, credentials -from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri from scrapy import twisted_version @@ -22,7 +20,6 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownlo from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.spiders import Spider from scrapy.http import Request @@ -520,6 +517,9 @@ class FTPTestCase(unittest.TestCase): skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" def setUp(self): + from twisted.protocols.ftp import FTPRealm, FTPFactory + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler + # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) @@ -601,6 +601,8 @@ class FTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_invalid_credentials(self): + from twisted.protocols.ftp import ConnectionLost + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": 'invalid'}) d = self.download_handler.download_request(request, None) From 3509378b8be0df7cb38d3823068288b5daa37612 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:29:19 +0300 Subject: [PATCH 063/137] py3: pass first http downloader test, simple crawler works now, yay! --- scrapy/core/downloader/handlers/http11.py | 9 ++++++--- scrapy/http/response/__init__.py | 3 ++- tests/test_downloader_handlers.py | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7c937a036..34070ebc6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -19,6 +19,7 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object +from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version logger = logging.getLogger(__name__) @@ -200,8 +201,8 @@ class ScrapyAgent(object): agent = self._get_agent(request, timeout) # request details - url = urldefrag(request.url)[0] - method = request.method + url = to_bytes(urldefrag(request.url)[0]) + method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader('Proxy-Authorization') @@ -261,8 +262,10 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) + url = to_unicode(url) respcls = responsetypes.from_args(headers=headers, url=url) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags) + return respcls( + url=url, status=status, headers=headers, body=body, flags=flags) @implementer(IBodyProducer) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 983154001..59ef15682 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ +import six from six.moves.urllib.parse import urljoin from scrapy.http.headers import Headers @@ -34,7 +35,7 @@ class Response(object_ref): return self._url def _set_url(self, url): - if isinstance(url, str): + if isinstance(url, six.string_types): self._url = url else: raise TypeError('%s url must be str, got %s:' % (type(self).__name__, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 5f1703c5c..cdb1ad02d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -88,7 +88,7 @@ class FileTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.url, request.url) self.assertEquals(response.status, 200) - self.assertEquals(response.body, '0123456789') + self.assertEquals(response.body, b'0123456789') request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -107,15 +107,15 @@ class HttpTestCase(unittest.TestCase): def setUp(self): name = self.mktemp() os.mkdir(name) - FilePath(name).child("file").setContent("0123456789") + FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) - r.putChild("redirect", util.Redirect("/file")) - r.putChild("wait", ForeverTakingResource()) - r.putChild("hang-after-headers", ForeverTakingResource(write=True)) - r.putChild("nolength", NoLengthResource()) - r.putChild("host", HostHeaderResource()) - r.putChild("payload", PayloadResource()) - r.putChild("broken", BrokenDownloadResource()) + r.putChild(b"redirect", util.Redirect(b"/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') @@ -136,7 +136,7 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d def test_download_head(self): From 6b79fffa9a4c53cb6a6af2e9d9251b95c28496b4 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:37:18 +0300 Subject: [PATCH 064/137] py3: pass all of HttpTestCase --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_downloader_handlers.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 34070ebc6..dbb002710 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -182,7 +182,7 @@ class ScrapyAgent(object): _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] omitConnectTunnel = proxyParams.find('noconnect') >= 0 - if scheme == 'https' and not omitConnectTunnel: + if scheme == b'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, request.headers.get('Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, @@ -233,7 +233,7 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, '', None + return txresponse, b'', None maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index cdb1ad02d..c017a9eb2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,4 @@ import os -import twisted import six from twisted.trial import unittest @@ -25,6 +24,7 @@ from scrapy.spiders import Spider from scrapy.http import Request from scrapy.settings import Settings from scrapy.utils.test import get_crawler +from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured from tests.mockserver import MockServer @@ -143,7 +143,7 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, '') + d.addCallback(self.assertEquals, b'') return d def test_redirect_status(self): @@ -175,7 +175,7 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals(response.body, '127.0.0.1:%d' % self.portno) + self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno)) self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) @@ -183,19 +183,19 @@ class HttpTestCase(unittest.TestCase): def test_host_header_seted_in_request_headers(self): def _test(response): - self.assertEquals(response.body, 'example.com') - self.assertEquals(request.headers.get('Host'), 'example.com') + self.assertEquals(response.body, b'example.com') + self.assertEquals(request.headers.get('Host'), b'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, 'example.com') + d.addCallback(self.assertEquals, b'example.com') return d def test_payload(self): - body = '1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) From c6f14a39de14b15b78d3fd4a098ec246536e12c4 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:50:16 +0300 Subject: [PATCH 065/137] py3: fix http10 downloader - unicode host expected here --- scrapy/core/downloader/handlers/http10.py | 5 +++-- tests/test_downloader_handlers.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 11b2acdae..0322bbe49 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -2,6 +2,7 @@ """ from twisted.internet import reactor from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode class HTTP10DownloadHandler(object): @@ -17,8 +18,8 @@ class HTTP10DownloadHandler(object): return factory.deferred def _connect(self, factory): - host, port = factory.host, factory.port - if factory.scheme == 'https': + host, port = to_unicode(factory.host), factory.port + if factory.scheme == b'https': return reactor.connectSSL(host, port, factory, self.ClientContextFactory()) else: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c017a9eb2..780f08806 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -223,7 +223,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d @defer.inlineCallbacks @@ -234,7 +234,7 @@ class Http11TestCase(HttpTestCase): # response body. (regardless of headers) d = self.download_request(request, Spider('foo', download_maxsize=10)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") yield d d = self.download_request(request, Spider('foo', download_maxsize=9)) @@ -257,7 +257,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo', download_maxsize=100)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d From 4950f5988ef1df5bc6b6ec2c4a70a7956f64b539 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 17:24:08 +0300 Subject: [PATCH 066/137] py3: pass http proxy tests --- scrapy/core/downloader/handlers/http11.py | 7 ++++--- tests/test_downloader_handlers.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index dbb002710..d81093a9f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -181,10 +181,11 @@ class ScrapyAgent(object): if proxy: _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - omitConnectTunnel = proxyParams.find('noconnect') >= 0 + proxyHost = to_unicode(proxyHost) + omitConnectTunnel = proxyParams.find(b'noconnect') >= 0 if scheme == b'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, - request.headers.get('Proxy-Authorization', None)) + request.headers.get(b'Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) @@ -205,7 +206,7 @@ class ScrapyAgent(object): method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): - headers.removeHeader('Proxy-Authorization') + headers.removeHeader(b'Proxy-Authorization') bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 780f08806..ebf1d2f9c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -351,7 +351,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, 'http://example.com') + self.assertEquals(response.body, b'http://example.com') http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -361,7 +361,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, 'https://example.com') + self.assertEquals(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) @@ -371,7 +371,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, '/path/to/resource') + self.assertEquals(response.body, b'/path/to/resource') request = Request(self.getURL('path/to/resource')) return self.download_request(request, Spider('foo')).addCallback(_test) From f46a9d595dee801d0ea13d7cdaab8b8de952929f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 17:31:58 +0300 Subject: [PATCH 067/137] skip ftp tests on py3 for now --- tests/test_downloader_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ebf1d2f9c..b3c1c565f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -515,6 +515,8 @@ class FTPTestCase(unittest.TestCase): if twisted_version < (10, 2, 0): skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" + if six.PY3: + skip = "Twisted missing ftp support for PY3" def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory From 2aa6c92ffca50c8f6e5d057ac2808a99785eb88f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 17:52:50 +0300 Subject: [PATCH 068/137] py3 fixes in tests.mockserver --- tests/mockserver.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 3e4f8c0e5..1ab8e4b8d 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -134,12 +134,12 @@ class Echo(LeafResource): class Partial(LeafResource): def render_GET(self, request): - request.setHeader("Content-Length", "1024") + request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET def _delayedRender(self, request): - request.write("partial content\n") + request.write(b"partial content\n") request.finish() @@ -147,7 +147,7 @@ class Drop(Partial): def _delayedRender(self, request): abort = getarg(request, "abort", 0, type=int) - request.write("this connection will be dropped\n") + request.write(b"this connection will be dropped\n") tr = request.channel.transport try: if abort and hasattr(tr, 'abortConnection'): @@ -162,13 +162,13 @@ class Root(Resource): def __init__(self): Resource.__init__(self) - self.putChild("status", Status()) - self.putChild("follow", Follow()) - self.putChild("delay", Delay()) - self.putChild("partial", Partial()) - self.putChild("drop", Drop()) - self.putChild("raw", Raw()) - self.putChild("echo", Echo()) + self.putChild(b"status", Status()) + self.putChild(b"follow", Follow()) + self.putChild(b"delay", Delay()) + self.putChild(b"partial", Partial()) + self.putChild(b"drop", Drop()) + self.putChild(b"raw", Raw()) + self.putChild(b"echo", Echo()) if six.PY2 and twisted_version > (12, 3, 0): from twisted.web.test.test_webclient import PayloadResource @@ -181,7 +181,7 @@ class Root(Resource): return self def render(self, request): - return 'Scrapy mock HTTP server\n' + return b'Scrapy mock HTTP server\n' class MockServer(): From 81a90c3af65ce863c073e2d83a6b149a03e7d4cb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 18:47:06 +0300 Subject: [PATCH 069/137] unskip part of test_download_gzip_response on py3, file a twisted issue for the remaining part --- tests/mockserver.py | 6 +++--- tests/test_downloader_handlers.py | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 1ab8e4b8d..02bab0efd 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -170,12 +170,12 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) - if six.PY2 and twisted_version > (12, 3, 0): + if twisted_version > (12, 3, 0): from twisted.web.test.test_webclient import PayloadResource from twisted.web.server import GzipEncoderFactory from twisted.web.resource import EncodingResourceWrapper - self.putChild('payload', PayloadResource()) - self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"payload", PayloadResource()) + self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b3c1c565f..a8de28d4b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -294,27 +294,30 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): - if six.PY2 and twisted_version > (12, 3, 0): + if twisted_version > (12, 3, 0): crawler = get_crawler(SingleRequestSpider) - body = '1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') + if six.PY2: + # download_maxsize = 50 is enough for the gzipped response + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload only enabled for PY2") + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') else: - raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x") + raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") class UriResource(resource.Resource): From 99f1f2ad1dbff9ae2e97755b2d61e03ab2339a6d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 19:00:48 +0300 Subject: [PATCH 070/137] unskip tests and modules ported to py3 --- tests/py3-ignores.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 9e75ecf92..57e80f590 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,6 @@ tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloader_handlers.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py @@ -25,8 +24,6 @@ scrapy/xlib/tx/client.py scrapy/xlib/tx/_newclient.py scrapy/xlib/tx/__init__.py scrapy/core/downloader/handlers/s3.py -scrapy/core/downloader/handlers/http11.py -scrapy/core/downloader/handlers/http.py scrapy/core/downloader/handlers/ftp.py scrapy/pipelines/images.py scrapy/pipelines/files.py From 94ab7bee6c5630bcb36e7b758daf31e45d493613 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:26:01 +0300 Subject: [PATCH 071/137] py3: body to bytes in tests, unskip test file --- tests/py3-ignores.txt | 1 - tests/test_downloadermiddleware.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 57e80f590..0da1b6089 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -7,7 +7,6 @@ tests/test_crawl.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py tests/test_engine.py tests/test_mail.py diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 13f35b92a..fb51392b2 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -5,6 +5,7 @@ from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler +from scrapy.utils.python import to_bytes from tests import mock @@ -68,7 +69,7 @@ class DefaultsTest(ManagerTestCase): """ req = Request('http://example.com') - body = '

You are being redirected

' + body = b'

You are being redirected

' resp = Response(req.url, status=302, body=body, headers={ 'Content-Length': str(len(body)), 'Content-Type': 'text/html', @@ -78,12 +79,12 @@ class DefaultsTest(ManagerTestCase): ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), "Not redirected: {0!r}".format(ret)) - self.assertEqual(ret.url, resp.headers['Location'], + self.assertEqual(to_bytes(ret.url), resp.headers['Location'], "Not redirected to location header") def test_200_and_invalid_gzipped_body_must_fail(self): req = Request('http://example.com') - body = '

You are being redirected

' + body = b'

You are being redirected

' resp = Response(req.url, status=200, body=body, headers={ 'Content-Length': str(len(body)), 'Content-Type': 'text/html', From dbf6cc73d96f5015e14d4075e5a0d50db2a96453 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:46:56 +0300 Subject: [PATCH 072/137] py3: add leveldb to py33 test env, fix anydbm module name on py3 --- scrapy/settings/default_settings.py | 4 +++- tox.ini | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8435b0354..b151933b6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,6 +18,8 @@ import sys from importlib import import_module from os.path import join, abspath, dirname +import six + AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -163,7 +165,7 @@ HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] -HTTPCACHE_DBM_MODULE = 'anydbm' +HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False diff --git a/tox.ini b/tox.ini index eae7e8e47..874a22ee2 100644 --- a/tox.ini +++ b/tox.ini @@ -42,6 +42,7 @@ deps = -rrequirements-py3.txt # Extras Pillow + leveldb -rtests/requirements-py3.txt [testenv:py34] From e7ed1fd70df28d529b5f1df04bb850bb260dfc01 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:48:07 +0300 Subject: [PATCH 073/137] py3 compat in httpcache - headers are bytes --- scrapy/extensions/httpcache.py | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 2911dd6bc..80f615818 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -45,7 +45,7 @@ class RFC2616Policy(object): def _parse_cachecontrol(self, r): if r not in self._cc_parsed: - cch = r.headers.get('Cache-Control', '') + cch = r.headers.get(b'Cache-Control', b'') parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -58,7 +58,7 @@ class RFC2616Policy(object): return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if 'no-store' in cc: + if b'no-store' in cc: return False # Any other is eligible for caching return True @@ -69,7 +69,7 @@ class RFC2616Policy(object): # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) # obey directive "Cache-Control: no-store" - if 'no-store' in cc: + if b'no-store' in cc: return False # Never cache 304 (Not Modified) responses elif response.status == 304: @@ -78,14 +78,14 @@ class RFC2616Policy(object): elif self.always_store: return True # Any hint on response expiration is good - elif 'max-age' in cc or 'Expires' in response.headers: + elif b'max-age' in cc or b'Expires' in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set elif response.status in (300, 301, 308): return True # Other statuses without expiration requires at least one validator elif response.status in (200, 203, 401): - return 'Last-Modified' in response.headers or 'ETag' in response.headers + return b'Last-Modified' in response.headers or b'ETag' in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated @@ -95,7 +95,7 @@ class RFC2616Policy(object): def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) - if 'no-cache' in cc or 'no-cache' in ccreq: + if b'no-cache' in cc or b'no-cache' in ccreq: return False now = time() @@ -109,7 +109,7 @@ class RFC2616Policy(object): if currentage < freshnesslifetime: return True - if 'max-stale' in ccreq and 'must-revalidate' not in cc: + if b'max-stale' in ccreq and b'must-revalidate' not in cc: # From RFC2616: "Indicates that the client is willing to # accept a response that has exceeded its expiration time. # If max-stale is assigned a value, then the client is @@ -117,7 +117,7 @@ class RFC2616Policy(object): # expiration time by no more than the specified number of # seconds. If no value is assigned to max-stale, then the # client is willing to accept a stale response of any age." - staleage = ccreq['max-stale'] + staleage = ccreq[b'max-stale'] if staleage is None: return True @@ -136,22 +136,22 @@ class RFC2616Policy(object): # as long as the old response didn't specify must-revalidate. if response.status >= 500: cc = self._parse_cachecontrol(cachedresponse) - if 'must-revalidate' not in cc: + if b'must-revalidate' not in cc: return True # Use the cached response if the server says it hasn't changed. return response.status == 304 def _set_conditional_validators(self, request, cachedresponse): - if 'Last-Modified' in cachedresponse.headers: - request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] + if b'Last-Modified' in cachedresponse.headers: + request.headers[b'If-Modified-Since'] = cachedresponse.headers[b'Last-Modified'] - if 'ETag' in cachedresponse.headers: - request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + if b'ETag' in cachedresponse.headers: + request.headers[b'If-None-Match'] = cachedresponse.headers[b'ETag'] def _get_max_age(self, cc): try: - return max(0, int(cc['max-age'])) + return max(0, int(cc[b'max-age'])) except (KeyError, ValueError): return None @@ -164,18 +164,18 @@ class RFC2616Policy(object): return maxage # Parse date header or synthesize it if none exists - date = rfc1123_to_epoch(response.headers.get('Date')) or now + date = rfc1123_to_epoch(response.headers.get(b'Date')) or now # Try HTTP/1.0 Expires header - if 'Expires' in response.headers: - expires = rfc1123_to_epoch(response.headers['Expires']) + if b'Expires' in response.headers: + expires = rfc1123_to_epoch(response.headers[b'Expires']) # When parsing Expires header fails RFC 2616 section 14.21 says we # should treat this as an expiration time in the past. return max(0, expires - date) if expires else 0 # Fallback to heuristic using last-modified header # This is not in RFC but on Firefox caching implementation - lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) + lastmodified = rfc1123_to_epoch(response.headers.get(b'Last-Modified')) if lastmodified and lastmodified <= date: return (date - lastmodified) / 10 @@ -192,13 +192,13 @@ class RFC2616Policy(object): currentage = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server - date = rfc1123_to_epoch(response.headers.get('Date')) or now + date = rfc1123_to_epoch(response.headers.get(b'Date')) or now if now > date: currentage = now - date - if 'Age' in response.headers: + if b'Age' in response.headers: try: - age = int(response.headers['Age']) + age = int(response.headers[b'Age']) currentage = max(currentage, age) except ValueError: pass @@ -404,16 +404,16 @@ def parse_cachecontrol(header): http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 - >>> parse_cachecontrol('public, max-age=3600') == {'public': None, - ... 'max-age': '3600'} + >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None, + ... b'max-age': b'3600'} True - >>> parse_cachecontrol('') == {} + >>> parse_cachecontrol(b'') == {} True """ directives = {} - for directive in header.split(','): - key, sep, val = directive.strip().partition('=') + for directive in header.split(b','): + key, sep, val = directive.strip().partition(b'=') if key: directives[key.lower()] = val if sep else None return directives From ddc91dda270e44d80dbe08cc79d4249f82cde093 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:49:28 +0300 Subject: [PATCH 074/137] py3: fix _BaseTest in httpcache --- tests/test_downloadermiddleware_httpcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 53389ae3b..5a636cc53 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -31,7 +31,7 @@ class _BaseTest(unittest.TestCase): headers={'User-Agent': 'test'}) self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, - body='test body', + body=b'test body', status=202) self.crawler.stats.open_spider(self.spider) @@ -84,9 +84,9 @@ class _BaseTest(unittest.TestCase): def assertEqualRequestButWithCacheValidators(self, request1, request2): self.assertEqual(request1.url, request2.url) - assert not 'If-None-Match' in request1.headers - assert not 'If-Modified-Since' in request1.headers - assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since')) + assert not b'If-None-Match' in request1.headers + assert not b'If-Modified-Since' in request1.headers + assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) def test_dont_cache(self): From ea0471e33a769f98f853b5ba145b75a9227111a1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 12:22:41 +0300 Subject: [PATCH 075/137] py3: fix LeveldbCacheStorage - using bytes as keys and values in leveldb --- scrapy/extensions/httpcache.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 80f615818..02f9fcee6 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -12,6 +12,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes class DummyPolicy(object): @@ -305,7 +306,7 @@ class FilesystemCacheStorage(object): 'timestamp': time(), } with self._open(os.path.join(rpath, 'meta'), 'wb') as f: - f.write(repr(metadata)) + f.write(to_bytes(repr(metadata))) with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: pickle.dump(metadata, f, protocol=2) with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: @@ -373,14 +374,14 @@ class LeveldbCacheStorage(object): 'body': response.body, } batch = self._leveldb.WriteBatch() - batch.Put('%s_data' % key, pickle.dumps(data, protocol=2)) - batch.Put('%s_time' % key, str(time())) + batch.Put(key + b'_data', pickle.dumps(data, protocol=2)) + batch.Put(key + b'_time', to_bytes(str(time()))) self.db.Write(batch) def _read_data(self, spider, request): key = self._request_key(request) try: - ts = self.db.Get('%s_time' % key) + ts = self.db.Get(key + b'_time') except KeyError: return # not found or invalid entry @@ -388,14 +389,14 @@ class LeveldbCacheStorage(object): return # expired try: - data = self.db.Get('%s_data' % key) + data = self.db.Get(key + b'_data') except KeyError: return # invalid entry else: return pickle.loads(data) def _request_key(self, request): - return request_fingerprint(request) + return to_bytes(request_fingerprint(request)) From 87849780bcf77aa47ac42cc1c7b24e1185563c7d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 13:13:10 +0300 Subject: [PATCH 076/137] some py3 fixes for RFC2616Policy --- scrapy/extensions/httpcache.py | 3 ++- tests/test_downloadermiddleware_httpcache.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 02f9fcee6..91b3ef262 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -41,7 +41,8 @@ class RFC2616Policy(object): def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') + self.ignore_response_cache_controls = map( + to_bytes, settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')) self._cc_parsed = WeakKeyDictionary() def _parse_cachecontrol(self, r): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 5a636cc53..4e0c72304 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -291,7 +291,7 @@ class RFC2616PolicyTest(DefaultStorageTest): self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response # but it allows new response to be stored - res0b = res0.replace(body='foo') + res0b = res0.replace(body=b'foo') res4 = self._process_requestresponse(mw, req2, res0b) self.assertEqualResponse(res4, res0b) assert 'cached' not in res4.flags @@ -435,7 +435,7 @@ class RFC2616PolicyTest(DefaultStorageTest): assert 'cached' not in res1.flags # Same request but as cached response is stale a new response must # be returned - res0b = res0a.replace(body='bar') + res0b = res0a.replace(body=b'bar') res2 = self._process_requestresponse(mw, req0, res0b) self.assertEqualResponse(res2, res0b) assert 'cached' not in res2.flags From 131f4632472922234a7abde116eb566329a952a2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 11:17:52 +0100 Subject: [PATCH 077/137] Allow failures for Python 3.5 for now --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ae9c745ac..ac93e337d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs +matrix: + allow_failures: + - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox From 96fcf4cea41a067b4feb4f8adaa5b9ae1d5d38dd Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 13:27:28 +0300 Subject: [PATCH 078/137] add a check that byte url is not accepted in http.Response on py3 --- tests/test_http_response.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b49d46ea8..710a5b29d 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -17,6 +17,8 @@ class BaseResponseTest(unittest.TestCase): # Response requires url in the consturctor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) + if not six.PY2: + self.assertRaises(TypeError, self.response_class, b"http://example.com") # body can be str or None self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) From a4ca1668d894920e4a74c8f4204fa3ee53039a1d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 14:20:19 +0300 Subject: [PATCH 079/137] add https test for http10 handler (no luck with testing https with http11 so far) --- tests/mockserver.py | 12 ++++++++---- tests/test_downloader_handlers.py | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 02bab0efd..e7953c4d4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -199,14 +199,18 @@ class MockServer(): time.sleep(0.2) +def ssl_context_factory(): + return ssl.DefaultOpenSSLContextFactory( + os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + ) + + if __name__ == "__main__": root = Root() factory = Site(root) httpPort = reactor.listenTCP(8998, factory) - contextFactory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), - ) + contextFactory = ssl_context_factory() httpsPort = reactor.listenSSL(8999, factory, contextFactory) def print_listening(): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8de28d4b..84d1aa191 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -27,7 +27,7 @@ from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured -from tests.mockserver import MockServer +from tests.mockserver import MockServer, ssl_context_factory from tests.spiders import SingleRequestSpider class DummyDH(object): @@ -102,6 +102,7 @@ class FileTestCase(unittest.TestCase): class HttpTestCase(unittest.TestCase): + scheme = 'http' download_handler_cls = HTTPDownloadHandler def setUp(self): @@ -118,7 +119,12 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') + self.host = '127.0.0.1' + if self.scheme == 'https': + self.port = reactor.listenSSL( + 0, self.wrapper, ssl_context_factory(), interface=self.host) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port self.download_handler = self.download_handler_cls(Settings()) self.download_request = self.download_handler.download_request @@ -130,7 +136,7 @@ class HttpTestCase(unittest.TestCase): yield self.download_handler.close() def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) def test_download(self): request = Request(self.getURL('file')) @@ -213,6 +219,12 @@ class Http10TestCase(HttpTestCase): download_handler_cls = HTTP10DownloadHandler +class Https10TestCase(Http10TestCase): + scheme = 'https' + def test_timeout_download_from_spider(self): + raise unittest.SkipTest("test_timeout_download_from_spider skipped under https") + + class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler From 4398d95a02659c90df496a9156c7b3297cc42f50 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 14:54:12 +0300 Subject: [PATCH 080/137] skip this file on py3 again - it has one compression test, sould be done separately --- tests/py3-ignores.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 0da1b6089..57e80f590 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -7,6 +7,7 @@ tests/test_crawl.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py +tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py tests/test_engine.py tests/test_mail.py From 8330776c2193c3954cba2277525401de35672e00 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:16:12 +0300 Subject: [PATCH 081/137] fix error reporting in test: we can fail in process_request too, so result should always be defined --- tests/test_downloadermiddleware_httpcache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 4e0c72304..12b69860a 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -257,6 +257,7 @@ class RFC2616PolicyTest(DefaultStorageTest): policy_class = 'scrapy.extensions.httpcache.RFC2616Policy' def _process_requestresponse(self, mw, request, response): + result = None try: result = mw.process_request(request, self.spider) if result: From b0648271d69fc3e5b50c430ad50f1afe44acf0d4 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:17:30 +0300 Subject: [PATCH 082/137] py3 fix for rfc1123_to_epoch - "except Exception" was hiding bytes/str error --- scrapy/extensions/httpcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 91b3ef262..03ea88a10 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -12,7 +12,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode class DummyPolicy(object): @@ -423,6 +423,7 @@ def parse_cachecontrol(header): def rfc1123_to_epoch(date_str): try: + date_str = to_unicode(date_str, encoding='ascii') return mktime_tz(parsedate_tz(date_str)) except Exception: return None From 085fdd628314d6d3268c6ebbd6e9388fbbb2d37f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:40:45 +0300 Subject: [PATCH 083/137] py3 fix for ignoring cache controls - map is not a list --- scrapy/extensions/httpcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 03ea88a10..a871cc895 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -41,8 +41,8 @@ class RFC2616Policy(object): def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = map( - to_bytes, settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')) + self.ignore_response_cache_controls = [to_bytes(cc) for cc in + settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')] self._cc_parsed = WeakKeyDictionary() def _parse_cachecontrol(self, r): From 7d44c5dcea6c523b4cac19eefa5eb22966c5a90f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:41:31 +0300 Subject: [PATCH 084/137] py3: unskip tests/test_downloadermiddleware_httpcache.py and scrapy/downloadermiddlewares/httpcache.py --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 57e80f590..2a9f06c8c 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,6 @@ tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware.py @@ -31,7 +30,6 @@ scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py scrapy/downloadermiddlewares/retry.py -scrapy/downloadermiddlewares/httpcache.py scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py From ee4fadc00724f02f9098625ac4d72fb29eac4dcf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 14:57:15 +0100 Subject: [PATCH 085/137] Use .read1() if available when using GzipFile --- .travis.yml | 3 --- scrapy/utils/gz.py | 20 ++++++++++++++------ tests/test_squeues.py | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac93e337d..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,6 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs -matrix: - allow_failures: - - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fa4bba57..df1d29698 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,30 +4,38 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import UnsupportedOperation from gzip import GzipFile +class ReadOneGzipFile(GzipFile): + def readone(self, size=-1): + try: + return self.read1(size) + except UnsupportedOperation: + return self.read(size) def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = GzipFile(fileobj=BytesIO(data)) + f = ReadOneGzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.read(8196) + chunk = f.readone(8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf # contains the whole page content - if output or f.extrabuf: - output += f.extrabuf - break + if output or getattr(f, 'extrabuf', None): + try: + output += f.extrabuf + finally: + break else: raise return output diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..232f539e6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From bb38400db560a4d814ea6c12364404d6daa75da5 Mon Sep 17 00:00:00 2001 From: Ralph Gutkowski Date: Fri, 15 Jan 2016 19:00:58 +0100 Subject: [PATCH 086/137] Update Stats Collection documentation `pages_crawled` value doesn't exist. Replace it with `downloader/response_count` in order to avoid confusion. --- docs/topics/stats.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0837610d0..290fc065c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -47,7 +47,7 @@ Set stat value:: Increment stat value:: - stats.inc_value('pages_crawled') + stats.inc_value('downloader/response_count') Set stat value only if greater than previous:: @@ -59,13 +59,13 @@ Set stat value only if lower than previous:: Get stat value:: - >>> stats.get_value('pages_crawled') + >>> stats.get_value('downloader/response_count') 8 Get all stats:: >>> stats.get_stats() - {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 79147a61a797f81fee44b000eb5ac9591593ff88 Mon Sep 17 00:00:00 2001 From: Ralph Gutkowski Date: Fri, 15 Jan 2016 19:25:56 +0100 Subject: [PATCH 087/137] Update stats.rst --- docs/topics/stats.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 290fc065c..dd0c6216b 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -47,7 +47,7 @@ Set stat value:: Increment stat value:: - stats.inc_value('downloader/response_count') + stats.inc_value('custom_count') Set stat value only if greater than previous:: @@ -59,13 +59,13 @@ Set stat value only if lower than previous:: Get stat value:: - >>> stats.get_value('downloader/response_count') - 8 + >>> stats.get_value('custom_count') + 1 Get all stats:: >>> stats.get_stats() - {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 4e44766653c294c54a3bb960b3734ee70bc663a4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 19:51:21 +0100 Subject: [PATCH 088/137] Use "unicode" string for lxml.etree.tostring() serialization --- scrapy/utils/iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 69c7f2c23..b0688791e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding=six.text_type) + nodetext = etree.tostring(node, encoding='unicode') node.clear() xs = Selector(text=nodetext, type='xml') if namespace: From cd735e377c2a26bc8ebb925bf6031429549faadf Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 18 Jan 2016 07:45:36 +0100 Subject: [PATCH 089/137] Simplify if statement --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index b0ac1badd..8b4faf8fc 100644 --- a/conftest.py +++ b/conftest.py @@ -34,7 +34,7 @@ if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, if six.PY3: for line in open('tests/py3-ignores.txt'): file_path = line.strip() - if len(file_path) > 0 and file_path[0] != '#': + if file_path and file_path[0] != '#': collect_ignore.append(file_path) From d3c2b0cf7ec185d18464e85974ee7499acecad79 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 10:06:14 +0300 Subject: [PATCH 090/137] py3 webclient: I was mistaken about unicode body, revert conversion to bytes and fix HEAD response --- scrapy/core/downloader/webclient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index d2cdd6f98..bbbb98f60 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -66,7 +66,7 @@ class ScrapyHTTPPageGetter(HTTPClient): def handleResponse(self, response): if self.factory.method.upper() == b'HEAD': - self.factory.page('') + self.factory.page(b'') elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) else: @@ -131,7 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - body = to_bytes(body) return respcls(url=self._url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): From 673df5e4161337136154eeccfc4e327d053dbd12 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 11:44:49 +0300 Subject: [PATCH 091/137] add webclient test - check that non-standart body encoding matches Content-Encoding header --- tests/test_webclient.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 412e10c89..3ee6c24c2 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -7,7 +7,7 @@ import six from six.moves.urllib.parse import urlparse from twisted.trial import unittest -from twisted.web import server, static, error, util +from twisted.web import server, static, util, resource from twisted.internet import reactor, defer from twisted.test.proto_helpers import StringTransport from twisted.python.filepath import FilePath @@ -18,14 +18,14 @@ from scrapy.http import Request, Headers from scrapy.utils.python import to_bytes, to_unicode -def getPage(url, contextFactory=None, *args, **kwargs): +def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" def _clientfactory(url, *args, **kwargs): url = to_unicode(url) timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory( Request(url, *args, **kwargs), timeout=timeout) - f.deferred.addCallback(lambda r: r.body) + f.deferred.addCallback(r_transform or (lambda r: r.body)) return f from twisted.web.client import _makeGetterFactory @@ -213,6 +213,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ ErrorResource, NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource + +class EncodingResource(resource.Resource): + out_encoding = 'cp1251' + + def render(self, request): + body = to_unicode(request.content.read()) + request.setHeader(b'content-encoding', self.out_encoding) + return body.encode(self.out_encoding) + + class WebClientTestCase(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -229,6 +239,7 @@ class WebClientTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"encoding", EncodingResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = self._listen(self.wrapper) @@ -338,3 +349,17 @@ class WebClientTestCase(unittest.TestCase): b'\n\n \n \n' b' \n \n ' b'click here\n \n\n') + + def test_Encoding(self): + """ Test that non-standart body encoding matches + Content-Encoding header """ + body = b'\xd0\x81\xd1\x8e\xd0\xaf' + return getPage( + self.getURL('encoding'), body=body, r_transform=lambda r: r)\ + .addCallback(self._check_Encoding, body) + + def _check_Encoding(self, response, original_body): + content_encoding = to_unicode(response.headers[b'Content-Encoding']) + self.assertEquals(content_encoding, EncodingResource.out_encoding) + self.assertEquals( + response.body.decode(content_encoding), to_unicode(original_body)) From 6b905a9aecb0e0e22339353a983257f3707d98b8 Mon Sep 17 00:00:00 2001 From: palego Date: Sat, 16 Jan 2016 14:23:58 +0100 Subject: [PATCH 092/137] split-up the assertIn to deal with OS X intricacies (directories prefixed with /private) --- tests/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 8edccd4bd..aa1b7cc7a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -90,8 +90,8 @@ class StartprojectTemplatesTest(ProjectTest): args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] p = self.proc('startproject', self.project_name, *args) out = to_native_str(retry_on_eintr(p.stdout.read)) - self.assertIn("New Scrapy project %r, using template directory %r, created in:" % \ - (self.project_name, join(self.tmpl, 'project')), out) + self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) + self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) From f2e2ff5e1f2eb1d11e18be248e27bef7fc9b4ef7 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:20:01 +0300 Subject: [PATCH 093/137] py3: webclient assumes that urls come from Request.url and are ascii-only --- scrapy/core/downloader/webclient.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index bbbb98f60..9bcc51943 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): + # Assume parsed is urlparse-d from Request.url, + # which was passed via safe_url_string and is ascii-only. + b = lambda s: to_bytes(s, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - path = to_bytes(path) - host = to_bytes(parsed.hostname) + path = b(path) + host = b(parsed.hostname) port = parsed.port - scheme = to_bytes(parsed.scheme, encoding='ascii') - netloc = to_bytes(parsed.netloc) + scheme = b(parsed.scheme) + netloc = b(parsed.netloc) if port is None: port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path def _parse(url): + """ Return tuple of (scheme, netloc, host, port, path), + all in bytes except for port which is int. + Assume url is from Request.url, which was passed via safe_url_string + and is ascii-only. + """ url = url.strip() parsed = urlparse(url) return _parsed_url_args(parsed) @@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url) + self.url = to_bytes(self._url, encoding='ascii') self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers) From 04f69fd18406a9361f3c13df32b5f6ea295ecdbf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:37:46 +0300 Subject: [PATCH 094/137] add https 1.1 downloader test - localhost is a valid DNS-ID --- tests/test_downloader_handlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 84d1aa191..80eed86f2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -119,7 +119,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = '127.0.0.1' + self.host = 'localhost' if self.scheme == 'https': self.port = reactor.listenSSL( 0, self.wrapper, ssl_context_factory(), interface=self.host) @@ -273,6 +273,10 @@ class Http11TestCase(HttpTestCase): return d +class Https11TestCase(Http11TestCase): + scheme = 'https' + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" if twisted_version < (11, 1, 0): From 98c060d0b2cc76934e16abc03a033f21850fd565 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:42:21 +0300 Subject: [PATCH 095/137] py3: fix http 1.1 test with https, and use self.host everywhere --- tests/test_downloader_handlers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 80eed86f2..999fa4c0a 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -168,6 +168,9 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider(self): + if self.scheme == 'https': + raise unittest.SkipTest( + 'test_timeout_download_from_spider skipped under https') spider = Spider('foo') meta = {'download_timeout': 0.2} # client connects but no data is received @@ -181,7 +184,8 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno)) + self.assertEquals( + response.body, to_bytes('%s:%d' % (self.host, self.portno))) self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) @@ -221,8 +225,6 @@ class Http10TestCase(HttpTestCase): class Https10TestCase(Http10TestCase): scheme = 'https' - def test_timeout_download_from_spider(self): - raise unittest.SkipTest("test_timeout_download_from_spider skipped under https") class Http11TestCase(HttpTestCase): From 0f527849f2e8eddaf5d756b061699f2eca522a18 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 14:44:04 +0300 Subject: [PATCH 096/137] https proxy tunneling - add a test (not perfect, but covers all impl) and fix for py3 --- scrapy/core/downloader/handlers/http11.py | 14 +++++++++----- tests/test_downloader_handlers.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index d81093a9f..729b80b05 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -78,7 +78,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile('HTTP/1\.. 200') + _responseMatcher = re.compile(b'HTTP/1\.. 200') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): @@ -92,11 +92,15 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = 'CONNECT %s:%s HTTP/1.1\r\n' % (self._tunneledHost, - self._tunneledPort) + tunnelReq = ( + b'CONNECT ' + + to_bytes(self._tunneledHost, encoding='ascii') + b':' + + to_bytes(str(self._tunneledPort)) + + b' HTTP/1.1\r\n') if self._proxyAuthHeader: - tunnelReq += 'Proxy-Authorization: %s\r\n' % self._proxyAuthHeader - tunnelReq += '\r\n' + tunnelReq += \ + b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n' + tunnelReq += b'\r\n' protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived protocol.dataReceived = self.processProxyResponse diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 999fa4c0a..2d6c05741 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -388,6 +388,16 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('https://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + http_proxy = self.getURL('') + domain = 'https://no-such-domain.nosuch' + request = Request( + domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) + d = self.download_request(request, Spider('foo')) + timeout = yield self.assertFailure(d, error.TimeoutError) + self.assertIn(domain, timeout.osError) + def test_download_without_proxy(self): def _test(response): self.assertEquals(response.status, 200) From 7af64e8fd2a90102d85ff0f453cd6ba1dde71caa Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:00:43 +0300 Subject: [PATCH 097/137] py3: remove extra encoding/decoding of url: pass it as bytes only when required --- scrapy/core/downloader/handlers/http11.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 729b80b05..82cf507f7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -206,7 +206,7 @@ class ScrapyAgent(object): agent = self._get_agent(request, timeout) # request details - url = to_bytes(urldefrag(request.url)[0]) + url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): @@ -214,7 +214,8 @@ class ScrapyAgent(object): bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d = agent.request(method, url, headers, bodyproducer) + d = agent.request( + method, to_bytes(url, encoding='ascii'), headers, bodyproducer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed @@ -267,10 +268,8 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) - url = to_unicode(url) respcls = responsetypes.from_args(headers=headers, url=url) - return respcls( - url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, flags=flags) @implementer(IBodyProducer) From b940606b7e3dacbf5d639965c54ccbb87a214c95 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:06:15 +0300 Subject: [PATCH 098/137] this is a test for TunnelingTCP4ClientEndpoint - move into Http11ProxyTestCase --- tests/test_downloader_handlers.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2d6c05741..59320597e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -388,16 +388,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('https://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): - http_proxy = self.getURL('') - domain = 'https://no-such-domain.nosuch' - request = Request( - domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) - d = self.download_request(request, Spider('foo')) - timeout = yield self.assertFailure(d, error.TimeoutError) - self.assertIn(domain, timeout.osError) - def test_download_without_proxy(self): def _test(response): self.assertEquals(response.status, 200) @@ -422,6 +412,17 @@ class Http11ProxyTestCase(HttpProxyTestCase): if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + """ Test TunnelingTCP4ClientEndpoint """ + http_proxy = self.getURL('') + domain = 'https://no-such-domain.nosuch' + request = Request( + domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) + d = self.download_request(request, Spider('foo')) + timeout = yield self.assertFailure(d, error.TimeoutError) + self.assertIn(domain, timeout.osError) + class HttpDownloadHandlerMock(object): def __init__(self, settings): From a2efd389b09984de15ee0653d169c0bbbfd61a05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:09:54 +0300 Subject: [PATCH 099/137] clarify: rename r_transform to response_transform --- tests/test_webclient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3ee6c24c2..dbe659d5c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,14 +18,14 @@ from scrapy.http import Request, Headers from scrapy.utils.python import to_bytes, to_unicode -def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs): +def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" def _clientfactory(url, *args, **kwargs): url = to_unicode(url) timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory( Request(url, *args, **kwargs), timeout=timeout) - f.deferred.addCallback(r_transform or (lambda r: r.body)) + f.deferred.addCallback(response_transform or (lambda r: r.body)) return f from twisted.web.client import _makeGetterFactory @@ -355,7 +355,7 @@ class WebClientTestCase(unittest.TestCase): Content-Encoding header """ body = b'\xd0\x81\xd1\x8e\xd0\xaf' return getPage( - self.getURL('encoding'), body=body, r_transform=lambda r: r)\ + self.getURL('encoding'), body=body, response_transform=lambda r: r)\ .addCallback(self._check_Encoding, body) def _check_Encoding(self, response, original_body): From 494643458270311341c509f5476c61737aa27a70 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:23:01 +0300 Subject: [PATCH 100/137] revert most changes to this test, and clarify - it is valid only on py2, because urls are strictly unicode on py3 --- tests/test_webclient.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index dbe659d5c..9b5beda4c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -78,16 +78,17 @@ class ParseUrlTestCase(unittest.TestCase): elements of its return tuple, even when passed an URL which has previously been passed to L{urlparse} as a C{unicode} string. """ - goodInput = u'http://example.com/path' - badInput = goodInput.encode('ascii') - if six.PY2: - goodInput, badInput = badInput, goodInput - urlparse(badInput) + if not six.PY2: + raise unittest.SkipTest( + "Applies only to Py2, as urls can be ONLY unicode on Py3") + badInput = u'http://example.com/path' + goodInput = badInput.encode('ascii') + self._parse(badInput) # cache badInput in urlparse_cached scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, bytes)) - self.assertTrue(isinstance(netloc, bytes)) - self.assertTrue(isinstance(host, bytes)) - self.assertTrue(isinstance(path, bytes)) + self.assertTrue(isinstance(scheme, str)) + self.assertTrue(isinstance(netloc, str)) + self.assertTrue(isinstance(host, str)) + self.assertTrue(isinstance(path, str)) self.assertTrue(isinstance(port, int)) From 0b9336418ef40ca95052ebbaa02f12953e165115 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 16:43:58 +0300 Subject: [PATCH 101/137] py3: port compression downloader middleware and tests --- .../downloadermiddlewares/httpcompression.py | 8 +++---- tests/py3-ignores.txt | 1 - ...st_downloadermiddleware_httpcompression.py | 24 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 719507396..7ab304c17 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -9,13 +9,13 @@ from scrapy.exceptions import NotConfigured class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured return cls() - + def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', 'gzip,deflate') @@ -39,10 +39,10 @@ class HttpCompressionMiddleware(object): return response def _decode(self, body, encoding): - if encoding == 'gzip' or encoding == 'x-gzip': + if encoding == b'gzip' or encoding == b'x-gzip': body = gunzip(body) - if encoding == 'deflate': + if encoding == b'deflate': try: body = zlib.decompress(body) except zlib.error: diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 2a9f06c8c..dbf63f0f5 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,6 @@ tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a18994ef3..2e6e47fef 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -50,46 +50,46 @@ class HttpCompressionTest(TestCase): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual(request.headers.get('Accept-Encoding'), 'gzip,deflate') + self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate') def test_process_response_gzip(self): response = self._getresponse('gzip') request = response.request - self.assertEqual(response.headers['Content-Encoding'], 'gzip') + self.assertEqual(response.headers['Content-Encoding'], b'gzip') newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response - assert newresponse.body.startswith(' Date: Mon, 18 Jan 2016 16:45:22 +0300 Subject: [PATCH 102/137] common test_downloadermiddleware.py also passes now due to fixes in compression middleware --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index dbf63f0f5..1f7d85ef7 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -5,7 +5,6 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py tests/test_engine.py tests/test_mail.py From bcbad2905d3ff375e85ede723d13560d765ee729 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Jan 2016 15:22:29 +0100 Subject: [PATCH 103/137] Stick with ValueError for queue/serialization exception tests --- tests/test_squeues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 232f539e6..48871ceeb 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) + self.assertRaises(ValueError, q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) + self.assertRaises(ValueError, q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From 120fb4adeb957a348f01bf60e821b2dcab2f9de1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 19:00:40 +0300 Subject: [PATCH 104/137] revert bogus change --- scrapy/http/response/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 59ef15682..09c4e725a 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -35,7 +35,7 @@ class Response(object_ref): return self._url def _set_url(self, url): - if isinstance(url, six.string_types): + if isinstance(url, str): self._url = url else: raise TypeError('%s url must be str, got %s:' % (type(self).__name__, From 7fdd3225b293d4951ba079e1141683e7fc55f905 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 19:09:09 +0300 Subject: [PATCH 105/137] fix test skipping logic - this is (temporary) py2-only part --- tests/test_downloader_handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 59320597e..e6d219168 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -322,18 +322,18 @@ class Http11MockServerTestCase(unittest.TestCase): # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') - yield crawler.crawl(seed=request) - if six.PY2: + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') + yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') + else: + raise unittest.SkipTest("xpayload only enabled for PY2") else: raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") From de98d8d00658181ed46a96835e2ab95f2f6cd457 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 19:27:31 +0300 Subject: [PATCH 106/137] move comment about test skipped on py3 into a proper place --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e6d219168..c936b72ed 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -327,12 +327,12 @@ class Http11MockServerTestCase(unittest.TestCase): request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response - # See issue https://twistedmatrix.com/trac/ticket/8175 failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') else: + # See issue https://twistedmatrix.com/trac/ticket/8175 raise unittest.SkipTest("xpayload only enabled for PY2") else: raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") From fd99ef86dfca50dbd36b2c1a022cf30a0720dbea Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Jan 2016 17:57:55 +0100 Subject: [PATCH 107/137] Test for AttributeError when pickling objects (Python>=3.5) Same "fix" as in e.g. https://github.com/joblib/joblib/pull/246 --- scrapy/squeues.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6e2a60fd2..21520f454 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,7 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - except pickle.PicklingError as e: + # Python>=3.5 raises AttributeError here while + # Python<=3.4 raises pickle.PicklingError + except (pickle.PicklingError, AttributeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ From ff235fa19ad30f34be7ff435497e19d8e97a8970 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 19 Jan 2016 00:00:31 +0500 Subject: [PATCH 108/137] Remove --lsprof command-line option. Fixes GH-1531 --- scrapy/cmdline.py | 9 +-- scrapy/commands/__init__.py | 2 - scrapy/xlib/lsprofcalltree.py | 120 ---------------------------------- 3 files changed, 1 insertion(+), 130 deletions(-) delete mode 100644 scrapy/xlib/lsprofcalltree.py diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 35050c13d..cb7bbd64d 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -7,7 +7,6 @@ import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.xlib import lsprofcalltree from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules @@ -144,7 +143,7 @@ def execute(argv=None, settings=None): sys.exit(cmd.exitcode) def _run_command(cmd, args, opts): - if opts.profile or opts.lsprof: + if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) @@ -152,17 +151,11 @@ def _run_command(cmd, args, opts): def _run_command_profiled(cmd, args, opts): if opts.profile: sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) - if opts.lsprof: - sys.stderr.write("scrapy: writing lsprof stats to %r\n" % opts.lsprof) loc = locals() p = cProfile.Profile() p.runctx('cmd.run(args, opts)', globals(), loc) if opts.profile: p.dump_stats(opts.profile) - k = lsprofcalltree.KCacheGrind(p) - if opts.lsprof: - with open(opts.lsprof, 'w') as f: - k.output(f) if __name__ == '__main__': execute() diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9ac013098..43b420821 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -65,8 +65,6 @@ class ScrapyCommand(object): help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, help="write python cProfile stats to FILE") - group.add_option("--lsprof", metavar="FILE", default=None, - help="write lsprof profiling stats to FILE") group.add_option("--pidfile", metavar="FILE", help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", diff --git a/scrapy/xlib/lsprofcalltree.py b/scrapy/xlib/lsprofcalltree.py deleted file mode 100644 index a604016cc..000000000 --- a/scrapy/xlib/lsprofcalltree.py +++ /dev/null @@ -1,120 +0,0 @@ -# lsprofcalltree.py: lsprof output which is readable by kcachegrind -# David Allouche -# Jp Calderone & Itamar Shtull-Trauring -# Johan Dahlin - -from __future__ import print_function -import optparse -import os -import sys - -try: - import cProfile -except ImportError: - raise SystemExit("This script requires cProfile from Python 2.5") - -def label(code): - if isinstance(code, str): - return ('~', 0, code) # built-in functions ('~' sorts at the end) - else: - return '%s %s:%d' % (code.co_name, - code.co_filename, - code.co_firstlineno) - -class KCacheGrind(object): - def __init__(self, profiler): - self.data = profiler.getstats() - self.out_file = None - - def output(self, out_file): - self.out_file = out_file - print('events: Ticks', file=out_file) - self._print_summary() - for entry in self.data: - self._entry(entry) - - def _print_summary(self): - max_cost = 0 - for entry in self.data: - totaltime = int(entry.totaltime * 1000) - max_cost = max(max_cost, totaltime) - print('summary: %d' % (max_cost,), file=self.out_file) - - def _entry(self, entry): - out_file = self.out_file - - code = entry.code - #print >> out_file, 'ob=%s' % (code.co_filename,) - if isinstance(code, str): - print('fi=~', file=out_file) - else: - print('fi=%s' % (code.co_filename,), file=out_file) - print('fn=%s' % (label(code),), file=out_file) - - inlinetime = int(entry.inlinetime * 1000) - if isinstance(code, str): - print('0 ', inlinetime, file=out_file) - else: - print('%d %d' % (code.co_firstlineno, inlinetime), file=out_file) - - # recursive calls are counted in entry.calls - if entry.calls: - calls = entry.calls - else: - calls = [] - - if isinstance(code, str): - lineno = 0 - else: - lineno = code.co_firstlineno - - for subentry in calls: - self._subentry(lineno, subentry) - print(file=out_file) - - def _subentry(self, lineno, subentry): - out_file = self.out_file - code = subentry.code - #print >> out_file, 'cob=%s' % (code.co_filename,) - print('cfn=%s' % (label(code),), file=out_file) - if isinstance(code, str): - print('cfi=~', file=out_file) - print('calls=%d 0' % (subentry.callcount,), file=out_file) - else: - print('cfi=%s' % (code.co_filename,), file=out_file) - print('calls=%d %d' % ( - subentry.callcount, code.co_firstlineno), file=out_file) - - totaltime = int(subentry.totaltime * 1000) - print('%d %d' % (lineno, totaltime), file=out_file) - -def main(args): - usage = "%s [-o output_file_path] scriptfile [arg] ..." - parser = optparse.OptionParser(usage=usage % sys.argv[0]) - parser.allow_interspersed_args = False - parser.add_option('-o', '--outfile', dest="outfile", - help="Save stats to ", default=None) - - if not sys.argv[1:]: - parser.print_usage() - sys.exit(2) - - options, args = parser.parse_args() - - if not options.outfile: - options.outfile = '%s.log' % os.path.basename(args[0]) - - sys.argv[:] = args - - prof = cProfile.Profile() - try: - try: - prof = prof.run('execfile(%r)' % (sys.argv[0],)) - except SystemExit: - pass - finally: - kg = KCacheGrind(prof) - kg.output(file(options.outfile, 'w')) - -if __name__ == '__main__': - sys.exit(main(sys.argv)) From 5ac54ed3391b027cccb140731f82f1f026ea2f7c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 17:25:50 +0300 Subject: [PATCH 109/137] raise minimal twisted version for py3 --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index a9a2e3be0..065095101 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 15.1.0 +Twisted >= 15.5.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From bb50c0be2fd7e737f2e2bd772f333e68cd02db06 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 17:30:59 +0300 Subject: [PATCH 110/137] remove unused import --- scrapy/http/response/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 09c4e725a..983154001 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,6 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -import six from six.moves.urllib.parse import urljoin from scrapy.http.headers import Headers From 9c3117a914c27b3acbb2b0300d8938d6e3b49b9e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 17:32:53 +0300 Subject: [PATCH 111/137] more pythonic check of noconnect in proxy params --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 82cf507f7..bda72f5e6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -186,7 +186,7 @@ class ScrapyAgent(object): _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) - omitConnectTunnel = proxyParams.find(b'noconnect') >= 0 + omitConnectTunnel = b'noconnect' in proxyParams if scheme == b'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, request.headers.get(b'Proxy-Authorization', None)) From 4c8417284062af7fe1c0107a0fcfdd377424dc50 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:24:58 +0300 Subject: [PATCH 112/137] move leveldb to tests/requirements-py3.txt --- tests/requirements-py3.txt | 1 + tox.ini | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index a709a734e..5cf786a89 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -3,6 +3,7 @@ pytest-twisted pytest-cov testfixtures jmespath +leveldb # optional for shell wrapper tests bpython ipython diff --git a/tox.ini b/tox.ini index 874a22ee2..eae7e8e47 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,6 @@ deps = -rrequirements-py3.txt # Extras Pillow - leveldb -rtests/requirements-py3.txt [testenv:py34] From 1f2233837a4219d02be73dc0836dfc885d47fffb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 16:58:24 +0100 Subject: [PATCH 113/137] Use if Py2/Py3 function instead of custom GzipFile class method --- scrapy/utils/gz.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index df1d29698..3e6596b0b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,24 +7,35 @@ except ImportError: from io import UnsupportedOperation from gzip import GzipFile -class ReadOneGzipFile(GzipFile): - def readone(self, size=-1): - try: - return self.read1(size) - except UnsupportedOperation: - return self.read(size) +import six + + +# - Python>=3.5 GzipFile's read() has issues returning leftover +# uncompressed data when input is corrupted +# (regression or bug-fix compared to Python 3.4) +# - read1(), which fetches data before raising EOFError on next call +# works here but is only available from Python>=3.3 +# - scrapy does not support Python 3.2 +# - Python 2.7 GzipFile works fine with standard read() + extrabuf +if six.PY3: + def read1(gzf, size=-1): + return gzf.read1(size) +else: + def read1(gzf, size=-1): + return gzf.read(size) + def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = ReadOneGzipFile(fileobj=BytesIO(data)) + f = GzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.readone(8196) + chunk = read1(f, 8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise From 2b5245839ceaaa256c7edccee351ea79129fa3e4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 17:04:57 +0100 Subject: [PATCH 114/137] Remove unused import statement --- scrapy/utils/gz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 3e6596b0b..d69fb598d 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,7 +4,6 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO -from io import UnsupportedOperation from gzip import GzipFile import six From e15f361b0580eb85a13f79b2c96417d17b9a0de6 Mon Sep 17 00:00:00 2001 From: carlosp420 Date: Tue, 19 Jan 2016 11:12:43 -0500 Subject: [PATCH 115/137] fixed typo You -> you --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 79f0b3b53..28ed7c064 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -50,7 +50,7 @@ many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU you crawler will have available. A good starting point is ``100``, but the best way to find out is by doing some trials and identifying at what concurrency your Scrapy process gets CPU -bounded. For optimum performance, You should pick a concurrency where CPU usage +bounded. For optimum performance, you should pick a concurrency where CPU usage is at 80-90%. To increase the global concurrency use:: From 176610f91068cb9663be184094e450de0aac77bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 19 Jan 2016 15:34:26 -0300 Subject: [PATCH 116/137] optional_features has been removed --- tests/test_downloader_handlers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a474b75d2..1eb6192ce 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -431,8 +431,12 @@ class HttpDownloadHandlerMock(object): def download_request(self, request, spider): return request + class S3AnonTestCase(unittest.TestCase): - skip = 'boto' not in optional_features and 'missing boto library' + try: + import boto + except ImportError: + skip = 'missing boto library' def setUp(self): self.s3reqh = S3DownloadHandler(Settings(), @@ -448,14 +452,13 @@ class S3AnonTestCase(unittest.TestCase): self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) self.assertEqual(self.s3reqh.conn.anon, True) + class S3TestCase(unittest.TestCase): download_handler_cls = S3DownloadHandler try: - # can't instance without settings, but ignore that - download_handler_cls({}) - except NotConfigured: + import boto + except ImportError: skip = 'missing boto library' - except KeyError: pass # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf From f0cf5463c85bd38243ac1539487d7118003a790e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 20 Jan 2016 12:23:48 +0300 Subject: [PATCH 117/137] cleanup http11 tunneling connection after #1678 - construct string and then convert to bytes --- scrapy/core/downloader/handlers/http11.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a4d5a28c8..ad3285a32 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,11 +92,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = ( - b'CONNECT ' + - to_bytes(self._tunneledHost, encoding='ascii') + b':' + - to_bytes(str(self._tunneledPort)) + - b' HTTP/1.1\r\n') + tunnelReq = to_bytes( + 'CONNECT %s:%s HTTP/1.1\r\n' % ( + self._tunneledHost, self._tunneledPort), encoding='ascii') if self._proxyAuthHeader: tunnelReq += \ b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n' From 29ff84a7920dd27a7b723f21b68ccc6e2076c08a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 20 Jan 2016 12:03:38 +0100 Subject: [PATCH 118/137] Invert PY2/PY3 test for conditional read1() definition --- scrapy/utils/gz.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index d69fb598d..d035f9fdf 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -16,12 +16,12 @@ import six # works here but is only available from Python>=3.3 # - scrapy does not support Python 3.2 # - Python 2.7 GzipFile works fine with standard read() + extrabuf -if six.PY3: - def read1(gzf, size=-1): - return gzf.read1(size) -else: +if six.PY2: def read1(gzf, size=-1): return gzf.read(size) +else: + def read1(gzf, size=-1): + return gzf.read1(size) def gunzip(data): From a32b59ac7507b9b909edfde4cfa9a9effae27641 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 18:54:32 +0300 Subject: [PATCH 119/137] py3: fix EngineTest.test_crawler --- tests/test_engine.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index dad921a60..df68e5281 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -55,11 +55,12 @@ class TestSpider(Spider): def parse_item(self, response): item = self.item_cls() - m = self.name_re.search(response.body) + body = response.body_as_unicode() + m = self.name_re.search(body) if m: item['name'] = m.group(1) item['url'] = response.url - m = self.price_re.search(response.body) + m = self.price_re.search(body) if m: item['price'] = m.group(1) return item @@ -77,8 +78,8 @@ class DictItemsSpider(TestSpider): def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: From 0b08b4bfcfb629aefd4f83cf85e64c74bc1077c3 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 10:07:00 +0300 Subject: [PATCH 120/137] fix engine tests - this callback (spider_closed_callback) should accept one argument, but the error was hidden on py2 --- tests/test_engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index df68e5281..9f2c02bff 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -238,12 +238,12 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_close_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) self.assertEqual(len(e.open_spiders), 1) yield e.close() @@ -251,7 +251,7 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_engine_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() self.assertTrue(e.running) From c1db60188aaf05d3a7f055d9b736669e679aed52 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 10:07:12 +0300 Subject: [PATCH 121/137] py3: unskip passing test_engine --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f7d85ef7..3e147e9e9 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -6,7 +6,6 @@ tests/test_linkextractors_deprecated.py tests/test_crawl.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware_retry.py -tests/test_engine.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py From 47d3c63338c4550341cc7518bcb48d0fc606b67d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:28:22 +0300 Subject: [PATCH 122/137] py3: port fetch and shell commands, and review + enable already passing test_closespider.py and tests/test_utils_template.py --- scrapy/commands/fetch.py | 6 ++++-- scrapy/utils/testsite.py | 12 ++++++------ tests/py3-ignores.txt | 4 ---- tests/test_command_fetch.py | 8 ++++---- tests/test_command_shell.py | 14 +++++++------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index e61eedf50..3888da210 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.python import to_unicode class Command(ScrapyCommand): @@ -30,7 +31,8 @@ class Command(ScrapyCommand): def _print_headers(self, headers, prefix): for key, values in headers.items(): for value in values: - print('%s %s: %s' % (prefix, key, value)) + print('%s %s: %s' % ( + prefix, to_unicode(key), to_unicode(value))) def _print_response(self, response, opts): if opts.headers: @@ -38,7 +40,7 @@ class Command(ScrapyCommand): print('>') self._print_headers(response.headers, '<') else: - print(response.body) + print(to_unicode(response.body)) def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 01508bdb4..ad0375443 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -22,13 +22,13 @@ class SiteTest(object): def test_site(): r = resource.Resource() - r.putChild("text", static.Data("Works", "text/plain")) - r.putChild("html", static.Data("

Works

World

", "text/html")) - r.putChild("enc-gb18030", static.Data("

gb18030 encoding

", "text/html; charset=gb18030")) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) + r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) - + if __name__ == '__main__': port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f7d85ef7..1f0f34c49 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,3 @@ -tests/test_closespider.py -tests/test_command_fetch.py -tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py @@ -12,7 +9,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_template.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 5283852b7..4843a9a2f 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url('/text')]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace('\r', '') # required on win32 - assert 'Server: TwistedWeb' in out - assert 'Content-Type: text/plain' in out + out = out.replace(b'\r', b'') # required on win32 + assert b'Server: TwistedWeb' in out, out + assert b'Content-Type: text/plain' in out diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index a56236d54..105202754 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(['-c', 'item']) - assert '{}' in out + assert b'{}' in out @defer.inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body']) - assert 'Works' in out + assert b'Works' in out @defer.inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)']) - assert 'TextResponse' in out + assert b'TextResponse' in out @defer.inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)']) - assert 'HtmlResponse' in out + assert b'HtmlResponse' in out @defer.inlineCallbacks def test_response_selector_html(self): xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding']) - self.assertEqual(out.strip(), 'gb18030') + self.assertEqual(out.strip(), b'gb18030') @defer.inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) - assert out.strip().endswith('/redirected') + assert out.strip().endswith(b'/redirected') @defer.inlineCallbacks def test_request_replace(self): From c6d013ec85747fe7a0ec57852667bc4e29ec3e9c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:42:34 +0300 Subject: [PATCH 123/137] py3 fetch command: it may actually be better to try to print bytes as-is --- scrapy/commands/fetch.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 3888da210..49fa18ab2 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,11 +1,11 @@ from __future__ import print_function +import sys, six from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError from scrapy.utils.spider import spidercls_for_request, DefaultSpider -from scrapy.utils.python import to_unicode class Command(ScrapyCommand): @@ -29,10 +29,10 @@ class Command(ScrapyCommand): help="print response HTTP headers instead of body") def _print_headers(self, headers, prefix): + prefix = prefix.encode() for key, values in headers.items(): for value in values: - print('%s %s: %s' % ( - prefix, to_unicode(key), to_unicode(value))) + self._print_bytes(prefix + b' ' + key + b': ' + value) def _print_response(self, response, opts): if opts.headers: @@ -40,7 +40,11 @@ class Command(ScrapyCommand): print('>') self._print_headers(response.headers, '<') else: - print(to_unicode(response.body)) + self._print_bytes(response.body) + + def _print_bytes(self, bytes_): + bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer + bytes_writer.write(bytes_ + b'\n') def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): From a5da7531c42bb26af49787aa901c3d19b80abf2f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:06:20 +0300 Subject: [PATCH 124/137] py3 backout skipping test_closespider - it was fixed on another branch --- tests/py3-ignores.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f0f34c49..f8c631827 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,3 +1,4 @@ +tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py From fd24e22442d900170e7a5888b27698c9414b2983 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 20 Jan 2016 23:30:58 +0300 Subject: [PATCH 125/137] use byte constants for prefix instead of encoding it --- scrapy/commands/fetch.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 49fa18ab2..f09a873c1 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -29,16 +29,15 @@ class Command(ScrapyCommand): help="print response HTTP headers instead of body") def _print_headers(self, headers, prefix): - prefix = prefix.encode() for key, values in headers.items(): for value in values: self._print_bytes(prefix + b' ' + key + b': ' + value) def _print_response(self, response, opts): if opts.headers: - self._print_headers(response.request.headers, '>') + self._print_headers(response.request.headers, b'>') print('>') - self._print_headers(response.headers, '<') + self._print_headers(response.headers, b'<') else: self._print_bytes(response.body) From 659715ecd92c3f39ed0b52509adefb73c49fa56c Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Fri, 24 Jul 2015 12:07:59 -0300 Subject: [PATCH 126/137] implements FormRequest.from_response CSS support --- docs/topics/request-response.rst | 8 +++++++- scrapy/http/request/form.py | 10 ++++++++-- tests/test_http_request.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 0abec1f96..8f519b459 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -282,7 +282,7 @@ fields with form data from :class:`Response` objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ```` element contained @@ -310,6 +310,9 @@ fields with form data from :class:`Response` objects. :param formxpath: if given, the first form that matches the xpath will be used. :type formxpath: string + :param formcss: if given, the first form that matches the css selector will be used. + :type formcss: string + :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. :type formnumber: integer @@ -339,6 +342,9 @@ fields with form data from :class:`Response` objects. .. versionadded:: 0.17 The ``formxpath`` parameter. + .. versionadded:: 1.1.5 + The ``formcss`` parameter. + Request usage examples ---------------------- diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f623a5aa3..5501634d3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,8 +34,14 @@ class FormRequest(Request): @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, **kwargs): + clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): + kwargs.setdefault('encoding', response.encoding) + + if formcss is not None: + from parsel.csstranslator import HTMLTranslator + formxpath = HTMLTranslator().css_to_xpath(formcss) + form = _get_form(response, formname, formid, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = _get_form_url(form, kwargs.pop('url', None)) @@ -73,7 +79,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): f = root.xpath('//form[@id="%s"]' % formid) if f: return f[0] - + # Get form element from xpath, if not found, go up if formxpath is not None: nodes = root.xpath(formxpath) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index f82b2de8d..b81d43c41 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -846,6 +846,26 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response) self.assertEqual(req.url, 'http://b.com/test_form') + def test_from_response_css(self): + response = _buildresponse( + """ + + + +
+ + +
""") + r1 = self.request_class.from_response(response, formcss="form[action='post.php']") + fs = _qs(r1) + self.assertEqual(fs[b'one'], [b'1']) + + r1 = self.request_class.from_response(response, formcss="input[name='four']") + fs = _qs(r1) + self.assertEqual(fs[b'three'], [b'3']) + + self.assertRaises(ValueError, self.request_class.from_response, + response, formcss="input[name='abc']") def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) From d4c4ca80624c2540d5dedd218695d38542d92a01 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 09:42:15 -0200 Subject: [PATCH 127/137] fix version number to appear new feature --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8f519b459..ea64d1599 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -342,7 +342,7 @@ fields with form data from :class:`Response` objects. .. versionadded:: 0.17 The ``formxpath`` parameter. - .. versionadded:: 1.1.5 + .. versionadded:: 1.1.0 The ``formcss`` parameter. Request usage examples From 80c55f19a143d8938ced81a599100259509567a1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 21 Jan 2016 18:31:58 +0500 Subject: [PATCH 128/137] PY3 fixed scrapy bench command --- scrapy/utils/benchserver.py | 8 ++++---- tests/test_commands.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 4385d72a9..a9a2c938e 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -16,15 +16,15 @@ class Root(Resource): total = _getarg(request, 'total', 100, int) show = _getarg(request, 'show', 10, int) nlist = [random.randint(1, total) for _ in range(show)] - request.write("") + request.write(b"") args = request.args.copy() for nl in nlist: args['n'] = nl argstr = urlencode(args, doseq=True) request.write("follow {1}
" - .format(argstr, nl)) - request.write("") - return '' + .format(argstr, nl).encode('utf8')) + request.write(b"") + return b'' def _getarg(request, name, default=None, type=str): diff --git a/tests/test_commands.py b/tests/test_commands.py index 5755b3881..057112d12 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -72,7 +72,7 @@ class StartprojectTest(ProjectTest): self.assertEqual(1, self.call('startproject', self.project_name)) self.assertEqual(1, self.call('startproject', 'wrong---project---name')) self.assertEqual(1, self.call('startproject', 'sys')) - + class StartprojectTemplatesTest(ProjectTest): @@ -80,7 +80,7 @@ class StartprojectTemplatesTest(ProjectTest): super(StartprojectTemplatesTest, self).setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') - + def test_startproject_template_override(self): copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) os.mknod(join(self.tmpl_proj, 'root_template')) @@ -276,3 +276,4 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') log = to_native_str(p.stderr.read()) self.assertIn('INFO: Crawled', log) + self.assertNotIn('Unhandled Error', log) From a18dc24471f121ed85d1bee4281d43c3a3728162 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 21 Jan 2016 18:44:37 +0500 Subject: [PATCH 129/137] correctly process arguments for bench server --- scrapy/utils/benchserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index a9a2c938e..5bbda6e27 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -13,8 +13,8 @@ class Root(Resource): return self def render(self, request): - total = _getarg(request, 'total', 100, int) - show = _getarg(request, 'show', 10, int) + total = _getarg(request, b'total', 100, int) + show = _getarg(request, b'show', 10, int) nlist = [random.randint(1, total) for _ in range(show)] request.write(b"") args = request.args.copy() From f042ad0f39594d59a1a2032e6294ff1890638138 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Thu, 10 Dec 2015 23:04:25 -0500 Subject: [PATCH 130/137] py3 fix HttpProxy and Retry Middlewares --- scrapy/downloadermiddlewares/httpproxy.py | 6 +++--- tests/py3-ignores.txt | 4 ---- tests/test_downloadermiddleware_httpproxy.py | 4 ++-- tests/test_downloadermiddleware_retry.py | 8 ++++---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index dda6a3d2a..8c3514fd0 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,7 +9,7 @@ from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured - +from scrapy.utils.python import to_bytes class HttpProxyMiddleware(object): @@ -26,7 +26,7 @@ class HttpProxyMiddleware(object): proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = '%s:%s' % (unquote(user), unquote(password)) + user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password))) creds = base64.b64encode(user_pass).strip() else: creds = None @@ -52,4 +52,4 @@ class HttpProxyMiddleware(object): creds, proxy = self.proxies[scheme] request.meta['proxy'] = proxy if creds: - request.headers['Proxy-Authorization'] = 'Basic ' + creds + request.headers['Proxy-Authorization'] = b'Basic ' + creds diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 8c883ee3c..185a278fb 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -2,8 +2,6 @@ tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware_retry.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py @@ -23,8 +21,6 @@ scrapy/pipelines/files.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py -scrapy/downloadermiddlewares/retry.py -scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 5b9717a89..7676b2a00 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -52,7 +52,7 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz') + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -60,7 +60,7 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjo=') + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 20561e771..3de9399cf 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -21,20 +21,20 @@ class RetryTest(unittest.TestCase): def test_priority_adjust(self): req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority < req.priority def test_404(self): req = Request('http://www.scrapytest.org/404') - rsp = Response('http://www.scrapytest.org/404', body='', status=404) + rsp = Response('http://www.scrapytest.org/404', body=b'', status=404) # dont retry 404s assert self.mw.process_response(req, rsp, self.spider) is rsp def test_dont_retry(self): req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True}) - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) # first retry r = self.mw.process_response(req, rsp, self.spider) @@ -56,7 +56,7 @@ class RetryTest(unittest.TestCase): def test_503(self): req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) # first retry req = self.mw.process_response(req, rsp, self.spider) From a06a5f00f4a62028416ada4919994efd7d439eb2 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Wed, 20 Jan 2016 13:52:52 -0500 Subject: [PATCH 131/137] adding configurable encoding for httpproxy authentication --- docs/topics/downloader-middleware.rst | 12 ++++++++++++ scrapy/downloadermiddlewares/httpproxy.py | 13 +++++++++++-- scrapy/settings/default_settings.py | 2 ++ tests/test_downloadermiddleware_httpproxy.py | 19 +++++++++++++++++-- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3641da231..a97d5a696 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -951,6 +951,18 @@ Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to enable it for :ref:`broad crawls `. +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + .. _DBM: http://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 8c3514fd0..b01bab76d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes + class HttpProxyMiddleware(object): - def __init__(self): + def __init__(self, auth_encoding='latin-1'): + self.auth_encoding = auth_encoding self.proxies = {} for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) @@ -21,12 +23,19 @@ class HttpProxyMiddleware(object): if not self.proxies: raise NotConfigured + @classmethod + def from_crawler(cls, crawler): + auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') + return cls(auth_encoding) + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password))) + user_pass = to_bytes( + '%s:%s' % (unquote(user), unquote(password)), + encoding=self.auth_encoding) creds = base64.b64encode(user_pass).strip() else: creds = None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b151933b6..44e74dc61 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_AUTH_ENCODING = 'latin-1' + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7676b2a00..2b26431a4 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -9,6 +9,7 @@ from scrapy.spiders import Spider spider = Spider('foo') + class TestDefaultHeadersMiddleware(TestCase): failureException = AssertionError @@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + def test_proxy_auth_encoding(self): + # utf-8 encoding + os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + mw = HttpProxyMiddleware(auth_encoding='utf-8') + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + + # default latin-1 encoding + mw = HttpProxyMiddleware(auth_encoding='latin-1') + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() @@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None - def test_no_proxy(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() @@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta - From 20b839b44ba5795aa0a6cf96ac71d4524072fcab Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 12:42:45 +0300 Subject: [PATCH 132/137] py3: pass first crawl test (test_follow_all): fix mock server --- tests/mockserver.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index e7953c4d4..336633f4b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -7,6 +7,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl from scrapy import twisted_version +from scrapy.utils.python import to_bytes if twisted_version < (11, 0, 0): @@ -55,12 +56,12 @@ class LeafResource(Resource): class Follow(LeafResource): def render(self, request): - total = getarg(request, "total", 100, type=int) - show = getarg(request, "show", 1, type=int) - order = getarg(request, "order", "desc") - maxlatency = getarg(request, "maxlatency", 0, type=float) - n = getarg(request, "n", total, type=int) - if order == "rand": + total = getarg(request, b"total", 100, type=int) + show = getarg(request, b"show", 1, type=int) + order = getarg(request, b"order", b"desc") + maxlatency = getarg(request, b"maxlatency", 0, type=float) + n = getarg(request, b"n", total, type=int) + if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" nlist = range(n, max(n - show, 0), -1) @@ -73,19 +74,19 @@ class Follow(LeafResource): s = """ """ args = request.args.copy() for nl in nlist: - args["n"] = [str(nl)] + args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) s += "follow %d
" % (argstr, nl) s += """""" - request.write(s) + request.write(to_bytes(s)) request.finish() class Delay(LeafResource): def render_GET(self, request): - n = getarg(request, "n", 1, type=float) - b = getarg(request, "b", 1, type=int) + n = getarg(request, b"n", 1, type=float) + b = getarg(request, b"b", 1, type=int) if b: # send headers now and delay body request.write('') @@ -93,16 +94,16 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write("Response delayed for %0.3f seconds\n" % n) + request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) request.finish() class Status(LeafResource): def render_GET(self, request): - n = getarg(request, "n", 200, type=int) + n = getarg(request, b"n", 200, type=int) request.setResponseCode(n) - return "" + return b"" class Raw(LeafResource): @@ -114,7 +115,7 @@ class Raw(LeafResource): render_POST = render_GET def _delayedRender(self, request): - raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n') + raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n') request.startedWriting = 1 request.write(raw) request.channel.transport.loseConnection() @@ -128,7 +129,7 @@ class Echo(LeafResource): 'headers': dict(request.requestHeaders.getAllRawHeaders()), 'body': request.content.read(), } - return json.dumps(output) + return to_bytes(json.dumps(output)) class Partial(LeafResource): @@ -146,7 +147,7 @@ class Partial(LeafResource): class Drop(Partial): def _delayedRender(self, request): - abort = getarg(request, "abort", 0, type=int) + abort = getarg(request, b"abort", 0, type=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport try: From 0680950b9898aa77e4c494ab8a318d791ef0d55f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 13:06:31 +0300 Subject: [PATCH 133/137] py3: pass CrawlTestCase.test_referer_header, fixing Echo resource in mockserver and json decoding in test --- tests/mockserver.py | 9 +++++---- tests/test_crawl.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 336633f4b..6877c786c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,13 +1,12 @@ from __future__ import print_function import sys, time, random, os, json -import six from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl from scrapy import twisted_version -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode if twisted_version < (11, 0, 0): @@ -126,8 +125,10 @@ class Echo(LeafResource): def render_GET(self, request): output = { - 'headers': dict(request.requestHeaders.getAllRawHeaders()), - 'body': request.content.read(), + 'headers': dict( + (to_unicode(k), [to_unicode(v) for v in vs]) + for k, vs in request.requestHeaders.getAllRawHeaders()), + 'body': to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..90fd921c8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -8,6 +8,7 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.crawler import CrawlerRunner +from scrapy.utils.python import to_unicode from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider @@ -201,16 +202,16 @@ with multiples lines self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) # start requests doesn't set Referer header - echo0 = json.loads(crawler.spider.meta['responses'][2].body) + echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) self.assertNotIn('Referer', echo0['headers']) # following request sets Referer to start request url - echo1 = json.loads(crawler.spider.meta['responses'][1].body) + echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body)) self.assertEqual(echo1['headers'].get('Referer'), [req0.url]) # next request avoids Referer header - echo2 = json.loads(crawler.spider.meta['responses'][2].body) + echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) self.assertNotIn('Referer', echo2['headers']) # last request explicitly sets a Referer header - echo3 = json.loads(crawler.spider.meta['responses'][3].body) + echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body)) self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) @defer.inlineCallbacks From ad2b3321b90245e59b2e4b99eb7a4296f6c7e768 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 13:13:41 +0300 Subject: [PATCH 134/137] py3 compat: use range, fixes CrawlTestCase.test_start_requests_bug_yielding --- tests/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 516062929..711d80cac 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -119,7 +119,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_before_yield: 1 / 0 - for s in xrange(100): + for s in range(100): qargs = {'total': 10, 'seed': s} url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) From bf5f54fa339b44fec3451c88a78e4620f56c3bc8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:01:30 +0300 Subject: [PATCH 135/137] py3: fix getarg --- tests/mockserver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 6877c786c..365ec81fd 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -30,9 +30,12 @@ else: from twisted.internet.task import deferLater -def getarg(request, name, default=None, type=str): +def getarg(request, name, default=None, type=None): if name in request.args: - return type(request.args[name][0]) + value = request.args[name][0] + if type is not None: + value = type(value) + return value else: return default From 4607f2843e25b559f7483186916a1d91329c9dd8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:01:43 +0300 Subject: [PATCH 136/137] py3: unskip test_crawl --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 185a278fb..c8beea8a3 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,7 +1,6 @@ tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_crawl.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py From 5813de883888a69cf015ad506d6052e6191feb6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:08:05 +0300 Subject: [PATCH 137/137] py3: unskip test_closespider - it passes after fixing mockserver.Follow resouce on py3 --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index c8beea8a3..f189a4c86 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,4 +1,3 @@ -tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_mail.py