From 81a0e3cd93954ecd0a728ef5fcf5dc92a281fdd5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Sep 2016 13:42:28 +0200 Subject: [PATCH 01/62] Raise log level for HttpErrorMiddleware to INFO (from DEBUG) Fixes GH-910 --- scrapy/spidermiddlewares/httperror.py | 2 +- tests/test_spidermiddleware_httperror.py | 27 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 6b7c50fce..e34c265c1 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -46,7 +46,7 @@ class HttpErrorMiddleware(object): def process_spider_exception(self, response, exception, spider): if isinstance(exception, HttpError): - logger.debug( + logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", {'response': response}, extra={'spider': spider}, ) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index a64400482..319746350 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,3 +1,4 @@ +import logging from unittest import TestCase from testfixtures import LogCapture @@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): self.assertIn('Ignoring response <500', str(log)) self.assertNotIn('Ignoring response <200', str(log)) self.assertNotIn('Ignoring response <402', str(log)) + + @defer.inlineCallbacks + def test_logging_level(self): + # HttpError logs ignored responses with level INFO + crawler = get_crawler(_HttpErrorSpider) + with LogCapture(level=logging.INFO) as log: + yield crawler.crawl() + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + + self.assertIn('Ignoring response <402', str(log)) + self.assertIn('Ignoring response <404', str(log)) + self.assertIn('Ignoring response <500', str(log)) + self.assertNotIn('Ignoring response <200', str(log)) + + # with level WARNING, we shouldn't capture anything from HttpError + crawler = get_crawler(_HttpErrorSpider) + with LogCapture(level=logging.WARNING) as log: + yield crawler.crawl() + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + + self.assertNotIn('Ignoring response <402', str(log)) + self.assertNotIn('Ignoring response <404', str(log)) + self.assertNotIn('Ignoring response <500', str(log)) + self.assertNotIn('Ignoring response <200', str(log)) From 45e95b79ce17eef0b3b4d324daa8b0d220fdb2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Tue, 18 Oct 2016 11:06:55 -0300 Subject: [PATCH 02/62] (fixes #2272) using arg_to_iter() to wrap single values and list() to avoid consuming from generators. --- scrapy/mail.py | 6 ++++++ tests/test_mail.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index c6339f25b..283c3b8e1 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -21,6 +21,8 @@ else: from twisted.internet import defer, reactor, ssl +from utils.misc import arg_to_iter + logger = logging.getLogger(__name__) @@ -48,6 +50,10 @@ class MailSender(object): msg = MIMEMultipart() else: msg = MIMENonMultipart(*mimetype.split('/', 1)) + + to = list(arg_to_iter(to)) + cc = list(arg_to_iter(cc)) + msg['From'] = self.mailfrom msg['To'] = COMMASPACE.join(to) msg['Date'] = formatdate(localtime=True) diff --git a/tests/test_mail.py b/tests/test_mail.py index bd7e49621..b139e98d8 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -10,7 +10,8 @@ class MailSenderTest(unittest.TestCase): def test_send(self): mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', _callback=self._catch_mail_sent) + mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', + _callback=self._catch_mail_sent) assert self.catched_msg @@ -24,9 +25,16 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get_payload(), 'body') self.assertEqual(msg.get('Content-Type'), 'text/plain') + def test_send_single_values_to_and_cc(self): + mailsender = MailSender(debug=True) + mailsender.send(to='test@scrapy.org', subject='subject', body='body', + cc='test@scrapy.org', _callback=self._catch_mail_sent) + def test_send_html(self): mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', body='

body

', mimetype='text/html', _callback=self._catch_mail_sent) + mailsender.send(to=['test@scrapy.org'], subject='subject', + body='

body

', mimetype='text/html', + _callback=self._catch_mail_sent) msg = self.catched_msg['msg'] self.assertEqual(msg.get_payload(), '

body

') @@ -90,7 +98,8 @@ class MailSenderTest(unittest.TestCase): mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, - attachs=attachs, charset='utf-8', _callback=self._catch_mail_sent) + attachs=attachs, charset='utf-8', + _callback=self._catch_mail_sent) assert self.catched_msg self.assertEqual(self.catched_msg['subject'], subject) @@ -99,7 +108,8 @@ class MailSenderTest(unittest.TestCase): msg = self.catched_msg['msg'] self.assertEqual(msg['subject'], subject) self.assertEqual(msg.get_charset(), Charset('utf-8')) - self.assertEqual(msg.get('Content-Type'), 'multipart/mixed; charset="utf-8"') + self.assertEqual(msg.get('Content-Type'), + 'multipart/mixed; charset="utf-8"') payload = msg.get_payload() assert isinstance(payload, list) From 3fb6e52457a7f24d580ed32c07aa12d9547d2970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Tue, 18 Oct 2016 12:24:11 -0300 Subject: [PATCH 03/62] fixes import for py35 env. --- scrapy/mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 283c3b8e1..0bb395521 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -21,7 +21,7 @@ else: from twisted.internet import defer, reactor, ssl -from utils.misc import arg_to_iter +from .utils.misc import arg_to_iter logger = logging.getLogger(__name__) From 3cd56da0cc68a1c1d0ac4ca38dabc5b8aab3e461 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 8 Nov 2016 20:52:32 -0300 Subject: [PATCH 04/62] Ignore explicitly compiled python files. This avoids to include compiled files in the templates directory. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 04b3e1fb9..94de4f3bf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,3 +12,4 @@ prune docs/build recursive-include extras * recursive-include bin * recursive-include tests * +global-exclude __pycache__ *.py[cod] From 28155dfcccc0cd7883ffde714570e8c13b0920a6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 9 Nov 2016 12:20:06 +0100 Subject: [PATCH 05/62] Parse robots.txt content as native str Fixes #2373 --- scrapy/downloadermiddlewares/robotstxt.py | 5 ++++- tests/test_downloadermiddleware_robotstxt.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 698f394ad..e26c22a09 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.python import to_native_str logger = logging.getLogger(__name__) @@ -94,7 +95,9 @@ class RobotsTxtMiddleware(object): # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. pass - rp.parse(body.splitlines()) + # stdlib's robotparser expects native 'str' ; + # with unicode input, non-ASCII encoded bytes decoding fails in Python2 + rp.parse(to_native_str(body).splitlines()) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f2e94e171..95208c41f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import re from twisted.internet import reactor, error @@ -30,11 +31,15 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = re.sub(b'^\s+(?m)', b'', b''' + ROBOTS = re.sub(b'^\s+(?m)', b'', u''' User-Agent: * Disallow: /admin/ Disallow: /static/ - ''') + + # taken from https://en.wikipedia.org/robots.txt + Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: + Disallow: /wiki/Käyttäjä: + '''.encode('utf-8')) response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): deferred = Deferred() @@ -48,7 +53,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): return DeferredList([ self.assertNotIgnored(Request('http://site.local/allowed'), middleware), self.assertIgnored(Request('http://site.local/admin/main'), middleware), - self.assertIgnored(Request('http://site.local/static/'), middleware) + self.assertIgnored(Request('http://site.local/static/'), middleware), + self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), + self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): From 6cd35c77da6601f218b5042f4b9a9919e642e15c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 15 Nov 2016 17:38:32 +0100 Subject: [PATCH 06/62] Pass user-agent as native str when checking URLs against robots.txt --- scrapy/downloadermiddlewares/robotstxt.py | 3 ++- tests/test_downloadermiddleware_robotstxt.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index e26c22a09..c3dfa7819 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -41,7 +41,8 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch(self._useragent, request.url): + if rp is not None and not rp.can_fetch( + to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) raise IgnoreRequest() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 95208c41f..60306eacb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -39,6 +39,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): # taken from https://en.wikipedia.org/robots.txt Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: + + User-Agent: UnicödeBöt + Disallow: /some/randome/page.html '''.encode('utf-8')) response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): From 01142e2ae5e3b82d5f8701858931ba0354808720 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 22 Nov 2016 14:48:33 +0100 Subject: [PATCH 07/62] Print more dependencies versions in "scrapy version" verbose output --- scrapy/commands/version.py | 13 +++++++++++++ tests/test_command_version.py | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 4bf085c9e..a9954edb0 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -26,12 +26,25 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: + import cssselect + import parsel import lxml.etree + import w3lib + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + print("Scrapy : %s" % scrapy.__version__) print("lxml : %s" % lxml_version) print("libxml2 : %s" % libxml2_version) + print("cssselect : %s" % cssselect.__version__) + print("parsel : %s" % parsel.__version__) + print("w3lib : %s" % w3lib_version) print("Twisted : %s" % twisted.version.short()) print("Python : %s" % sys.version.replace("\n", "- ")) print("pyOpenSSL : %s" % self._get_openssl_version()) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 37e1f2543..2789d207c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -25,5 +25,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute(['-v']) headers = [l.partition(":")[0].strip() for l in out.strip().decode(encoding).splitlines()] - self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'Twisted', - 'Python', 'pyOpenSSL', 'Platform']) + self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', + 'cssselect', 'parsel', 'w3lib', + 'Twisted', 'Python', 'pyOpenSSL', + 'Platform']) From 35b655d2f84d652440c393166f6e19d7384b4f1c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 12:23:22 +0100 Subject: [PATCH 08/62] Handle redirects transparently by default in shell and fetch Adds --no-status-aware command line option to have previous behaviour --- scrapy/commands/fetch.py | 5 ++++- scrapy/commands/shell.py | 4 +++- scrapy/shell.py | 12 ++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index f09a873c1..a157b19f8 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,6 +27,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not handle status codes like redirects and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -50,7 +52,8 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + if opts.no_status_aware: + request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7be7f7256..bc0203d89 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,6 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not transparently handle status codes like redirects") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -68,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url) + shell.start(url=url, handle_statuses=opts.no_status_aware) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 183ee1f70..966003f17 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -40,11 +40,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None): + def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider) + self.fetch(url, spider, handle_statuses=handle_statuses) elif request: self.fetch(request, spider) elif response: @@ -98,14 +98,14 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None): + def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): if isinstance(request_or_url, Request): request = request_or_url - url = request.url else: url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + request = Request(url, dont_filter=True, **kwargs) + if handle_statuses: + request.meta['handle_httpstatus_all'] = True response = None try: response, spider = threads.blockingCallFromThread( From 9aefc0a886ca571a49f087e5349e2557fd78d943 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 13:41:51 +0100 Subject: [PATCH 09/62] Add test for fetch command with redirections disabled --- scrapy/utils/testsite.py | 8 ++++++++ tests/test_command_fetch.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ad0375443..e50a989b3 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -20,12 +20,20 @@ class SiteTest(object): return urljoin(self.baseurl, path) +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request): + content = util.Redirect.render(self, request) + return content.replace(b'http-equiv=\"refresh\"', + b'http-no-equiv=\"do-not-refresh-me\"') + + def test_site(): r = resource.Resource() r.putChild(b"text", static.Data(b"Works", "text/plain")) r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 4843a9a2f..45d03a129 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/text')]) self.assertEqual(out.strip(), b'Works') + @defer.inlineCallbacks + def test_redirect_default(self): + _, out, _ = yield self.execute([self.url('/redirect')]) + self.assertEqual(out.strip(), b'Redirected here') + + @defer.inlineCallbacks + def test_redirect_disabled(self): + _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')]) + err = err.strip() + self.assertIn(b'downloader/response_status_count/302', err, err) + self.assertNotIn(b'downloader/response_status_count/200', err, err) + @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) From 03cf5f1bd2019127933ea4a6358a7d47743efcf6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:18:57 +0100 Subject: [PATCH 10/62] Remove ChunkedTransferMiddleware from default settings --- docs/topics/downloader-middleware.rst | 8 +++++++- docs/topics/settings.rst | 1 - scrapy/settings/default_settings.py | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15069e56e..dca5ec6a0 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -665,7 +665,13 @@ ChunkedTransferMiddleware .. class:: ChunkedTransferMiddleware - This middleware adds support for `chunked transfer encoding`_ + This middleware adds support for `chunked transfer encoding`_. + +.. note:: + This middleware is not enabled nor used by Scrapy downloader anymore. + In fact, Scrapy downloader has built-in support for chunked transfers, + so this middleware has no effect in practice. + HttpProxyMiddleware ------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..2195e4233 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -468,7 +468,6 @@ Default:: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, } diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..61f4bd567 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -102,7 +102,6 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, # Downloader side From e6f174b01535810fffeae2d020bc66a6a25ba4e8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:33:27 +0100 Subject: [PATCH 11/62] Add deprecation warning for ChunkedTransfer middleware --- scrapy/downloadermiddlewares/chunked.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 57e97e4d2..fd90aab2a 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -1,6 +1,14 @@ +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer +warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` " + "is deprecated, chunked transfers are supported by default.", + ScrapyDeprecationWarning, stacklevel=2) + + class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding From 8cffb4bbefcebd10ec12ee2678fd490edf149576 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:50:21 +0100 Subject: [PATCH 12/62] Update warning wording --- scrapy/downloadermiddlewares/chunked.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index fd90aab2a..64d94c489 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -4,8 +4,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer -warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` " - "is deprecated, chunked transfers are supported by default.", +warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " + "chunked transfers are supported by default.", ScrapyDeprecationWarning, stacklevel=2) From 059085b5b4a54901c4fd0730e43b156a6d8eeb51 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 18:23:34 +0100 Subject: [PATCH 13/62] Remove docs for deprecated ChunkedTransfer middleware --- docs/topics/downloader-middleware.rst | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dca5ec6a0..29d9b0298 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -657,22 +657,6 @@ Default: ``True`` Whether the Compression middleware will be enabled. -ChunkedTransferMiddleware -------------------------- - -.. module:: scrapy.downloadermiddlewares.chunked - :synopsis: Chunked Transfer Middleware - -.. class:: ChunkedTransferMiddleware - - This middleware adds support for `chunked transfer encoding`_. - -.. note:: - This middleware is not enabled nor used by Scrapy downloader anymore. - In fact, Scrapy downloader has built-in support for chunked transfers, - so this middleware has no effect in practice. - - HttpProxyMiddleware ------------------- @@ -976,4 +960,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html -.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding From f98ffb53b66cf2e5b38b393f811f59ccc1d68992 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 29 Nov 2016 16:52:54 +0100 Subject: [PATCH 14/62] add docs for some scheduler settings --- docs/faq.rst | 2 +- docs/topics/settings.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 415331515..ad11b071b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -317,6 +317,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. .. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/LIFO +.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..91f3b37c9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1033,6 +1033,35 @@ Example entry in logs:: (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) + +.. setting:: SCHEDULER_DISK_QUEUE + + SCHEDULER_DISK_QUEUE +-------------------- + +Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` + +Type of disk queue that will be used by scheduler. Other available types are +``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +``scrapy.squeues.MarshalLifoDiskQueue``. + +.. setting:: SCHEDULER_MEMORY_QUEUE + +SCHEDULER_MEMORY_QUEUE +---------------------- +Default: ``'scrapy.squeues.LifoMemoryQueue'`` + +Type of in-memory queue used by scheduler. Other available type is: +``scrapy.squeues.FifoMemoryQueue``. + +.. setting:: SCHEDULER_PRIORITY_QUEUE + +SCHEDULER_PRIORITY_QUEUE +------------------------ +Default: ``'queuelib.PriorityQueue'`` + +Type of priority queue used by scheduler. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS From 624284e85145aa62b0e7fc21bf0d61bda2cbbefa Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 29 Nov 2016 18:18:59 +0100 Subject: [PATCH 15/62] Fix indent --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 91f3b37c9..fe3767a53 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1036,7 +1036,7 @@ Example entry in logs:: .. setting:: SCHEDULER_DISK_QUEUE - SCHEDULER_DISK_QUEUE +SCHEDULER_DISK_QUEUE -------------------- Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` From 27606aad1166afe3eebbee8dc365b1a122639e3c Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 30 Nov 2016 09:47:02 +0200 Subject: [PATCH 16/62] Fix #396 re-triggered issue The InteractiveShellEmbed class is a singleton and we need to drop the instance with its clear_instance() method to rebuild the instance from scratch with fresh environment for each subsequent Scrapy's shell drop in. --- scrapy/utils/console.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 567fd51bc..1888d9599 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -15,6 +15,9 @@ def _embed_ipython_shell(namespace={}, banner=''): config = load_default_config() # Always use .instace() to ensure _instance propagation to all parents # this is needed for completion works well for new imports + # and clear the instance to always have the fresh env + # on repeated breaks like with inspect_response() + InteractiveShellEmbed.clear_instance() shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config) shell() From 5ff64ad015aff872ffae78193b67525dd53ae80d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Nov 2016 11:47:25 -0300 Subject: [PATCH 17/62] handle relative sitemap urls in robots.txt --- scrapy/spiders/sitemap.py | 2 +- scrapy/utils/sitemap.py | 7 +++++-- tests/test_spider.py | 6 +++++- tests/test_utils_sitemap.py | 9 +++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 89d96c330..9e45637c3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -32,7 +32,7 @@ class SitemapSpider(Spider): def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): - for url in sitemap_urls_from_robots(response.text): + for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) else: body = self._get_sitemap_body(response) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 008196435..4742b3e13 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,9 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + import lxml.etree +from six.moves.urllib.parse import urljoin class Sitemap(object): @@ -34,10 +36,11 @@ class Sitemap(object): yield d -def sitemap_urls_from_robots(robots_text): +def sitemap_urls_from_robots(robots_text, base_url=None): """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith('sitemap:'): - yield line.split(':', 1)[1].strip() + url = line.split(':', 1)[1].strip() + yield urljoin(base_url, url) diff --git a/tests/test_spider.py b/tests/test_spider.py index 1d22c1212..079734a69 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -332,13 +332,17 @@ class SitemapSpiderTest(SpiderTest): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml """ r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://example.com/sitemap.xml', - 'http://example.com/sitemap-product-index.xml']) + 'http://example.com/sitemap-product-index.xml', + 'http://example.com/sitemap-uppercase.xml', + 'http://www.example.com/sitemap-relative-url.xml']) class BaseSpiderDeprecationTest(unittest.TestCase): diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index bd2677956..716bb44eb 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -119,13 +119,18 @@ Disallow: /s*/*tags # Sitemap files Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml # Forums Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots)), - ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')), + ['http://example.com/sitemap.xml', + 'http://example.com/sitemap-product-index.xml', + 'http://example.com/sitemap-uppercase.xml', + 'http://example.com/sitemap-relative-url.xml']) def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before tag""" From d9f43e21ba510ee3c95c89c3d551bc30a8d0f2c9 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 1 Dec 2016 11:56:33 -0300 Subject: [PATCH 18/62] TST: Fix duplicated test name. --- tests/test_spiderloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 68dca2e98..83c3a3670 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -66,7 +66,7 @@ class SpiderLoaderTest(unittest.TestCase): self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 1 - def test_load_spider_module(self): + def test_load_spider_module_multiple(self): prefix = 'tests.test_spiderloader.test_spiders.' module = ','.join(prefix + s for s in ('spider1', 'spider2')) settings = Settings({'SPIDER_MODULES': module}) From 6431e7a1386e03fff203ae85b6c47a4f0f4cf797 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 1 Dec 2016 13:24:12 -0300 Subject: [PATCH 19/62] DOC State explicitly that spiders are loaded recursively. --- docs/topics/api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index d470a0d41..985cc0433 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -171,7 +171,8 @@ SpiderLoader API This class method is used by Scrapy to create an instance of the class. It's called with the current project settings, and it loads the spiders - found in the modules of the :setting:`SPIDER_MODULES` setting. + found recursively in the modules of the :setting:`SPIDER_MODULES` + setting. :param settings: project settings :type settings: :class:`~scrapy.settings.Settings` instance From 923b974f0a10c3eb007ee4d183aa63a41d6a6db2 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 1 Dec 2016 12:52:52 -0300 Subject: [PATCH 20/62] TST Include nested a nested spider in spider loader test. --- tests/test_spiderloader/__init__.py | 6 +++--- tests/test_spiderloader/test_spiders/nested/__init__.py | 0 tests/test_spiderloader/test_spiders/nested/spider4.py | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 tests/test_spiderloader/test_spiders/nested/__init__.py create mode 100644 tests/test_spiderloader/test_spiders/nested/spider4.py diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 68dca2e98..74e461215 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -9,6 +9,7 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy +import tempfile from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings @@ -22,8 +23,7 @@ class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = self.mktemp() - os.mkdir(self.tmpdir) + self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) @@ -40,7 +40,7 @@ class SpiderLoaderTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spider_loader.list()), - set(['spider1', 'spider2', 'spider3'])) + set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_load(self): spider1 = self.spider_loader.load("spider1") diff --git a/tests/test_spiderloader/test_spiders/nested/__init__.py b/tests/test_spiderloader/test_spiders/nested/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py new file mode 100644 index 000000000..35b71870a --- /dev/null +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -0,0 +1,9 @@ +from scrapy.spiders import Spider + +class Spider4(Spider): + name = "spider4" + allowed_domains = ['spider4.com'] + + @classmethod + def handles_request(cls, request): + return request.url == 'http://spider4.com/onlythis' From e1ea0c433a96c2949b0a7c26cde65aea13fb79b4 Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 22:02:10 +0000 Subject: [PATCH 21/62] Strip xlib.tx code of Twisted 10 --- scrapy/xlib/tx/__init__.py | 12 ++++----- scrapy/xlib/tx/_newclient.py | 31 +++++++++++++++++------- scrapy/xlib/tx/client.py | 28 +++++++++++++++------ scrapy/xlib/tx/endpoints.py | 26 +++++++++++++++----- scrapy/xlib/tx/interfaces.py | 47 ++++++++++++++++++++++++++---------- scrapy/xlib/tx/iweb.py | 18 ++++++++++---- 6 files changed, 115 insertions(+), 47 deletions(-) diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index 1ac4e0108..1c9bf09e5 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -15,9 +15,9 @@ else: client = endpoints = _Mock() -Agent = client.Agent -ProxyAgent = client.ProxyAgent -ResponseDone = client.ResponseDone -ResponseFailed = client.ResponseFailed -HTTPConnectionPool = client.HTTPConnectionPool -TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint +Agent = client.Agent # since < 11.1 +ProxyAgent = client.ProxyAgent # since 11.1 +ResponseDone = client.ResponseDone # since 11.1 +ResponseFailed = client.ResponseFailed # since 11.1 +HTTPConnectionPool = client.HTTPConnectionPool # since 12.1 +TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index 16d0ca6b4..e902d6683 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder -from .iweb import IResponse, UNKNOWN_LENGTH +from twisted.web._newclient import ( + BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, + RequestGenerationFailed, RequestTransmissionFailed, + WrongBodyLength, ResponseDone, RequestNotSent, + LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder, + TransportProxyProducer, +) +# newer than 10.0.0 +#from twisted.web._newclient import ( +# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser, +# HTTPClientParser, Request, Response, HTTP11ClientProtocol, +#) +from .iweb import IResponse # States HTTPParser can be in STATUS = 'STATUS' @@ -52,7 +65,7 @@ HEADER = 'HEADER' BODY = 'BODY' DONE = 'DONE' - +''' {{{ class BadHeaders(Exception): """ Headers passed to L{Request} were in some way invalid. @@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException): @ivar reasons: A C{list} of one or more L{Failure} instances giving the reasons the request transmission was considered to have failed. """ - +}}} ''' class ConnectionAborted(Exception): @@ -126,7 +139,7 @@ class ConnectionAborted(Exception): """ - +''' {{{ class WrongBodyLength(Exception): """ An L{IBodyProducer} declared the number of bytes it was going to @@ -142,7 +155,7 @@ class ResponseDone(Exception): protocol passed to L{Response.deliverBody} and indicates that the entire response has been delivered. """ - +}}} ''' class ResponseFailed(_WrapperException): @@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed): """ - +''' {{{ class RequestNotSent(Exception): """ L{RequestNotSent} indicates that an attempt was made to issue a request but @@ -178,7 +191,7 @@ class RequestNotSent(Exception): to send a request using a protocol which is no longer connected to a server. """ - +}}} ''' def _callAppFunction(function): @@ -764,7 +777,7 @@ class Request: _callAppFunction(self.bodyProducer.stopProducing) - +''' {{{ class LengthEnforcingConsumer: """ An L{IConsumer} proxy which enforces an exact length requirement on the @@ -1188,7 +1201,7 @@ class TransportProxyProducer: """ if self._producer is not None: self._producer.pauseProducing() - +}}} ''' class HTTP11ClientProtocol(Protocol): diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index c3830dc47..396115985 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocol from twisted.python import failure from twisted.python.components import proxyForInterface from twisted.web import error +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from twisted.web.http_headers import Headers +from twisted.web.client import ( + PartialDownloadError, +) +# newer than 10.0.0 +#from twisted.web.client import ( +# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer, +# HTTPConnectionPool, Agent, ProxyAgent, +#) + from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer - +from .iweb import IResponse +''' {{{ class PartialDownloadError(error.Error): """ Page was only partially downloaded, we got disconnected in middle. @ivar response: All of the response body which was downloaded. """ - +}}} ''' class _URL(tuple): """ @@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, from twisted.web.error import SchemeNotSupported from ._newclient import Request, Response, HTTP11ClientProtocol -from ._newclient import ResponseDone, ResponseFailed -from ._newclient import RequestNotSent, RequestTransmissionFailed +from twisted.web._newclient import ResponseDone +from ._newclient import ResponseFailed +from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed +from twisted.web._newclient import ( + PotentialDataLoss, _WrapperException) from ._newclient import ( - ResponseNeverReceived, PotentialDataLoss, _WrapperException) + ResponseNeverReceived) try: from twisted.internet.ssl import ClientContextFactory @@ -1161,8 +1174,7 @@ def readBody(response): __all__ = [ - 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader', - 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage', + 'PartialDownloadError', 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index d8a92ccd0..21a467433 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions. from __future__ import division, absolute_import import os -import socket +#import socket from zope.interface import implementer, directlyProvides import warnings -from twisted.internet import interfaces, defer, error, fdesc, threads +from twisted.internet import interfaces, defer, error, fdesc from twisted.internet.protocol import ( - ClientFactory, Protocol, ProcessProtocol, Factory) + ClientFactory, Protocol, Factory) +#from twisted.internet import threads, ProcessProtocol from twisted.internet.interfaces import IStreamServerEndpointStringParser from twisted.internet.interfaces import IStreamClientEndpointStringParser from twisted.python.filepath import FilePath -from twisted.python.failure import Failure -from twisted.python import log +#from twisted.python.failure import Failure +#from twisted.python import log from twisted.python.components import proxyForInterface from twisted.plugin import IPlugin, getPlugins -from twisted.internet import stdio +#from twisted.internet import stdio + +# newer than 10.0.0 +#from twisted.internet.endpoints import ( +# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint, +# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol, +# quoteStringArgument, +# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers +# clientFromString, #> using newer _clientParsers +# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint, +# _parseTCP, _parseUNIX, _loadCAsFromDir, +# _parseSSL, _tokenize, +# _parseClientTCP, _parseClientSSL, _parseClientUNIX, +#) from .interfaces import IFileDescriptorReceiver diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index f3e4ed5d8..a715d4a05 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -11,7 +11,28 @@ from __future__ import division, absolute_import from zope.interface import Interface, Attribute +from twisted.internet.interfaces import ( + IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, + IReactorUDP, IReactorMulticast, IReactorProcess, + IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, + IReactorPluggableResolver, IReactorFDSet, + IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, + IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, + ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, + IProtocol, IProcessProtocol, IHalfCloseableProtocol, + IProtocolFactory, ITransport, IProcessTransport, IServiceCollection, + IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, + IMulticastTransport, +) +# newer than 10.0.0 +#from twisted.internet.interfaces import ( +# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket, +# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport, +# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint, +# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, +#) +''' {{{ class IAddress(Interface): """ An address, e.g. a TCP C{(host, port)}. @@ -74,7 +95,7 @@ class IResolverSimple(Interface): @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) if the name cannot be resolved within the specified timeout period. """ - +}}} ''' class IResolver(IResolverSimple): @@ -614,7 +635,7 @@ class IResolver(IResolverSimple): """ - +''' {{{ class IReactorTCP(Interface): def listenTCP(port, factory, backlog=50, interface=''): @@ -701,7 +722,7 @@ class IReactorSSL(Interface): @param interface: the hostname to bind to, defaults to '' (all) """ - +}}} ''' class IReactorUNIX(Interface): @@ -829,7 +850,7 @@ class IReactorWin32Events(Interface): """ - +''' {{{ class IReactorUDP(Interface): """ UDP socket methods. @@ -868,7 +889,7 @@ class IReactorMulticast(Interface): @see: L{twisted.internet.interfaces.IMulticastTransport} @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} """ - +}}} ''' class IReactorSocket(Interface): @@ -970,7 +991,7 @@ class IReactorSocket(Interface): """ - +''' {{{ class IReactorProcess(Interface): def spawnProcess(processProtocol, executable, args=(), env={}, path=None, @@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface): @return: The previously installed resolver. """ - +}}} ''' class IReactorDaemonize(Interface): """ @@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface): """ - +''' {{{ class IReactorFDSet(Interface): """ Implement me to be able to use L{IFileDescriptor} type resources. @@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface): This will never be called for TCP connections as TCP does not support notification of this type of half-close. """ - +}}} ''' class IFileDescriptorReceiver(Interface): @@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface): """ - +''' {{{ class IProtocolFactory(Interface): """ Interface for protocol factories. @@ -1974,7 +1995,7 @@ class ITransport(Interface): @return: An L{IAddress} provider. """ - +}}} ''' class ITCPTransport(ITransport): """ @@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport): Return an object with the peer's certificate info. """ - +''' {{{ class IProcessTransport(ITransport): """ A process transport. @@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface): """ Leave multicast group, return L{Deferred} of success. """ - +}}} ''' class IStreamClientEndpoint(Interface): """ diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index ddcb6ed7a..57a111411 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}. from zope.interface import Interface, Attribute -from twisted.internet.interfaces import IPushProducer +#from twisted.internet.interfaces import IPushProducer +from twisted.web.iweb import ( + ICredentialFactory, IBodyProducer, + UNKNOWN_LENGTH, +) +# newer than 10.0.0 +#from twisted.web.iweb import ( +# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, +#) class IRequest(Interface): """ @@ -320,7 +328,7 @@ class IRequest(Interface): """ - +''' {{{ class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of @@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer): L{Deferred} returned by C{startProducing} is never fired. """ - +}}} ''' class IRenderable(Interface): @@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface): """ - +''' {{{ UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" - +}}} ''' __all__ = [ "ICredentialFactory", "IRequest", "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", From c8cf1a303d2272aa44422035fd1e3e6048cb5606 Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 22:02:11 +0000 Subject: [PATCH 22/62] Bump Twisted dependency to 13.1.0 (released June 2013) --- requirements.txt | 2 +- scrapy/xlib/tx/_newclient.py | 36 +++++++++++++--------------- scrapy/xlib/tx/client.py | 38 +++++++++++++---------------- scrapy/xlib/tx/endpoints.py | 38 +++++++++++++---------------- scrapy/xlib/tx/interfaces.py | 46 +++++++++++++++++------------------- scrapy/xlib/tx/iweb.py | 15 +++++------- setup.py | 2 +- 7 files changed, 79 insertions(+), 98 deletions(-) diff --git a/requirements.txt b/requirements.txt index cfa907050..64b6e771c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=10.0.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index e902d6683..d20eda34f 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDone from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol -from twisted.protocols.basic import LineReceiver -from twisted.web.iweb import UNKNOWN_LENGTH -from twisted.web.http_headers import Headers +#from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH, IResponse +#from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder from twisted.web._newclient import ( BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, - RequestGenerationFailed, RequestTransmissionFailed, - WrongBodyLength, ResponseDone, RequestNotSent, - LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder, - TransportProxyProducer, + RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted, + WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent, + ResponseNeverReceived, HTTPParser, HTTPClientParser, Request, + LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder, + TransportProxyProducer, HTTP11ClientProtocol ) -# newer than 10.0.0 -#from twisted.web._newclient import ( -# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser, -# HTTPClientParser, Request, Response, HTTP11ClientProtocol, -#) -from .iweb import IResponse # States HTTPParser can be in STATUS = 'STATUS' @@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException): @ivar reasons: A C{list} of one or more L{Failure} instances giving the reasons the request transmission was considered to have failed. """ -}}} ''' + class ConnectionAborted(Exception): @@ -139,7 +134,7 @@ class ConnectionAborted(Exception): """ -''' {{{ + class WrongBodyLength(Exception): """ An L{IBodyProducer} declared the number of bytes it was going to @@ -155,7 +150,7 @@ class ResponseDone(Exception): protocol passed to L{Response.deliverBody} and indicates that the entire response has been delivered. """ -}}} ''' + class ResponseFailed(_WrapperException): @@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed): """ -''' {{{ + class RequestNotSent(Exception): """ L{RequestNotSent} indicates that an attempt was made to issue a request but @@ -191,7 +186,7 @@ class RequestNotSent(Exception): to send a request using a protocol which is no longer connected to a server. """ -}}} ''' + def _callAppFunction(function): @@ -777,7 +772,7 @@ class Request: _callAppFunction(self.bodyProducer.stopProducing) -''' {{{ + class LengthEnforcingConsumer: """ An L{IConsumer} proxy which enforces an exact length requirement on the @@ -1201,7 +1196,7 @@ class TransportProxyProducer: """ if self._producer is not None: self._producer.pauseProducing() -}}} ''' + class HTTP11ClientProtocol(Protocol): @@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol): d = Deferred() self._abortDeferreds.append(d) return d +}}} ''' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index 396115985..8e0b1df8b 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -29,23 +29,18 @@ from twisted.python.failure import Failure from twisted.web import http from twisted.internet import defer, protocol, task, reactor from twisted.internet.interfaces import IProtocol +from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint from twisted.python import failure from twisted.python.components import proxyForInterface from twisted.web import error -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse from twisted.web.http_headers import Headers from twisted.web.client import ( - PartialDownloadError, + PartialDownloadError, FileBodyProducer, + CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, + Agent, ProxyAgent, HTTPConnectionPool, readBody, ) -# newer than 10.0.0 -#from twisted.web.client import ( -# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer, -# HTTPConnectionPool, Agent, ProxyAgent, -#) - -from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse ''' {{{ class PartialDownloadError(error.Error): @@ -54,7 +49,7 @@ class PartialDownloadError(error.Error): @ivar response: All of the response body which was downloaded. """ -}}} ''' + class _URL(tuple): """ @@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, else: reactor.connectTCP(host, port, factory) return factory - +}}} ''' # The code which follows is based on the new HTTP client implementation. It # should be significantly better than anything above, though it is not yet # feature equivalent. -from twisted.web.error import SchemeNotSupported -from ._newclient import Request, Response, HTTP11ClientProtocol -from twisted.web._newclient import ResponseDone -from ._newclient import ResponseFailed -from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed -from twisted.web._newclient import ( - PotentialDataLoss, _WrapperException) -from ._newclient import ( - ResponseNeverReceived) +#from twisted.web.error import SchemeNotSupported +from twisted.web._newclient import Response +#from twisted.web._newclient import Request, HTTP11ClientProtocol +from twisted.web._newclient import ResponseDone, ResponseFailed +#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed +#from twisted.web._newclient import ( +# ResponseNeverReceived, PotentialDataLoss, _WrapperException) +''' {{{ try: from twisted.internet.ssl import ClientContextFactory except ImportError: @@ -1170,7 +1164,7 @@ def readBody(response): d = defer.Deferred() response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) return d - +}}} ''' __all__ = [ diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index 21a467433..3f4704064 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvides import warnings from twisted.internet import interfaces, defer, error, fdesc -from twisted.internet.protocol import ( - ClientFactory, Protocol, Factory) +#from twisted.internet.protocol import ( +# ClientFactory, Protocol) +from twisted.internet.protocol import Factory #from twisted.internet import threads, ProcessProtocol from twisted.internet.interfaces import IStreamServerEndpointStringParser -from twisted.internet.interfaces import IStreamClientEndpointStringParser +#from twisted.internet.interfaces import IStreamClientEndpointStringParser from twisted.python.filepath import FilePath #from twisted.python.failure import Failure #from twisted.python import log -from twisted.python.components import proxyForInterface +#from twisted.python.components import proxyForInterface from twisted.plugin import IPlugin, getPlugins #from twisted.internet import stdio -# newer than 10.0.0 -#from twisted.internet.endpoints import ( -# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint, -# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol, -# quoteStringArgument, -# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers -# clientFromString, #> using newer _clientParsers -# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint, -# _parseTCP, _parseUNIX, _loadCAsFromDir, -# _parseSSL, _tokenize, -# _parseClientTCP, _parseClientSSL, _parseClientUNIX, -#) - -from .interfaces import IFileDescriptorReceiver +from twisted.internet.endpoints import ( + clientFromString, serverFromString, quoteStringArgument, + TCP4ServerEndpoint, TCP6ServerEndpoint, + TCP4ClientEndpoint, TCP6ClientEndpoint, + UNIXServerEndpoint, UNIXClientEndpoint, + SSL4ServerEndpoint, SSL4ClientEndpoint, + AdoptedStreamServerEndpoint, connectProtocol, +) __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - +''' {{{ class _WrappingProtocol(Protocol): """ Wrap another protocol in order to notify my user when a connection has @@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol): self._wrappedProtocol = wrappedProtocol for iface in [interfaces.IHalfCloseableProtocol, - IFileDescriptorReceiver]: + interfaces.IFileDescriptorReceiver]: if iface.providedBy(self._wrappedProtocol): directlyProvides(self, iface) @@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object): + def _parseTCP(factory, port, interface="", backlog=50): """ Internal parser function for L{_parseServer} to convert the string @@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol): def buildProtocol(self, addr): return protocol return endpoint.connect(OneShotFactory()) - +}}} ''' diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index a715d4a05..7b2a78632 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -13,24 +13,21 @@ from zope.interface import Interface, Attribute from twisted.internet.interfaces import ( IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, - IReactorUDP, IReactorMulticast, IReactorProcess, + IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, - IReactorPluggableResolver, IReactorFDSet, + IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet, IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, IProtocol, IProcessProtocol, IHalfCloseableProtocol, - IProtocolFactory, ITransport, IProcessTransport, IServiceCollection, + IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport, + IUNIXTransport, + ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection, IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, - IMulticastTransport, + IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint, + IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, + IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver ) -# newer than 10.0.0 -#from twisted.internet.interfaces import ( -# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket, -# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport, -# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint, -# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, -#) ''' {{{ class IAddress(Interface): @@ -95,7 +92,7 @@ class IResolverSimple(Interface): @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) if the name cannot be resolved within the specified timeout period. """ -}}} ''' + class IResolver(IResolverSimple): @@ -635,7 +632,7 @@ class IResolver(IResolverSimple): """ -''' {{{ + class IReactorTCP(Interface): def listenTCP(port, factory, backlog=50, interface=''): @@ -722,7 +719,7 @@ class IReactorSSL(Interface): @param interface: the hostname to bind to, defaults to '' (all) """ -}}} ''' + class IReactorUNIX(Interface): @@ -850,7 +847,7 @@ class IReactorWin32Events(Interface): """ -''' {{{ + class IReactorUDP(Interface): """ UDP socket methods. @@ -889,7 +886,7 @@ class IReactorMulticast(Interface): @see: L{twisted.internet.interfaces.IMulticastTransport} @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} """ -}}} ''' + class IReactorSocket(Interface): @@ -991,7 +988,7 @@ class IReactorSocket(Interface): """ -''' {{{ + class IReactorProcess(Interface): def spawnProcess(processProtocol, executable, args=(), env={}, path=None, @@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface): @return: The previously installed resolver. """ -}}} ''' + class IReactorDaemonize(Interface): """ @@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface): """ -''' {{{ + class IReactorFDSet(Interface): """ Implement me to be able to use L{IFileDescriptor} type resources. @@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface): This will never be called for TCP connections as TCP does not support notification of this type of half-close. """ -}}} ''' + class IFileDescriptorReceiver(Interface): @@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface): """ -''' {{{ + class IProtocolFactory(Interface): """ Interface for protocol factories. @@ -1995,7 +1992,7 @@ class ITransport(Interface): @return: An L{IAddress} provider. """ -}}} ''' + class ITCPTransport(ITransport): """ @@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport): Return an object with the peer's certificate info. """ -''' {{{ + class IProcessTransport(ITransport): """ A process transport. @@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface): """ Leave multicast group, return L{Deferred} of success. """ -}}} ''' + class IStreamClientEndpoint(Interface): """ @@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface): @return: a client endpoint @rtype: L{IStreamClientEndpoint} """ +}}} ''' diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index 57a111411..32c88ff2d 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute #from twisted.internet.interfaces import IPushProducer from twisted.web.iweb import ( - ICredentialFactory, IBodyProducer, - UNKNOWN_LENGTH, + IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, + IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, ) -# newer than 10.0.0 -#from twisted.web.iweb import ( -# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, -#) +''' {{{ class IRequest(Interface): """ An HTTP request. @@ -328,7 +325,7 @@ class IRequest(Interface): """ -''' {{{ + class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of @@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer): L{Deferred} returned by C{startProducing} is never fired. """ -}}} ''' + class IRenderable(Interface): @@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface): """ -''' {{{ + UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" }}} ''' __all__ = [ diff --git a/setup.py b/setup.py index 92c114a7a..5e32d4240 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], install_requires=[ - 'Twisted>=10.0.0', + 'Twisted>=13.1.0', 'w3lib>=1.15.0', 'queuelib', 'lxml', From 985755d1fe0aabf922bcdb0e8bc22d67948820cb Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 23:44:15 +0000 Subject: [PATCH 23/62] Remove obsolete xlib code for Twisted 13.1.0 --- scrapy/xlib/tx/_newclient.py | 1466 -------------------- scrapy/xlib/tx/client.py | 1116 ---------------- scrapy/xlib/tx/endpoints.py | 1253 ----------------- scrapy/xlib/tx/interfaces.py | 2433 ---------------------------------- scrapy/xlib/tx/iweb.py | 569 -------- 5 files changed, 6837 deletions(-) diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index d20eda34f..39cd20f95 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -38,9 +38,7 @@ from twisted.internet.error import ConnectionDone from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol -#from twisted.protocols.basic import LineReceiver from twisted.web.iweb import UNKNOWN_LENGTH, IResponse -#from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder @@ -59,1467 +57,3 @@ STATUS = 'STATUS' HEADER = 'HEADER' BODY = 'BODY' DONE = 'DONE' - -''' {{{ -class BadHeaders(Exception): - """ - Headers passed to L{Request} were in some way invalid. - """ - - - -class ExcessWrite(Exception): - """ - The body L{IBodyProducer} for a request tried to write data after - indicating it had finished writing data. - """ - - -class ParseError(Exception): - """ - Some received data could not be parsed. - - @ivar data: The string which could not be parsed. - """ - def __init__(self, reason, data): - Exception.__init__(self, reason, data) - self.data = data - - - -class BadResponseVersion(ParseError): - """ - The version string in a status line was unparsable. - """ - - - -class _WrapperException(Exception): - """ - L{_WrapperException} is the base exception type for exceptions which - include one or more other exceptions as the low-level causes. - - @ivar reasons: A list of exceptions. See subclass documentation for more - details. - """ - def __init__(self, reasons): - Exception.__init__(self, reasons) - self.reasons = reasons - - - -class RequestGenerationFailed(_WrapperException): - """ - There was an error while creating the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request generation was considered to have failed. - """ - - - -class RequestTransmissionFailed(_WrapperException): - """ - There was an error while sending the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request transmission was considered to have failed. - """ - - - -class ConnectionAborted(Exception): - """ - The connection was explicitly aborted by application code. - """ - - - -class WrongBodyLength(Exception): - """ - An L{IBodyProducer} declared the number of bytes it was going to - produce (via its C{length} attribute) and then produced a different number - of bytes. - """ - - - -class ResponseDone(Exception): - """ - L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the - protocol passed to L{Response.deliverBody} and indicates that the entire - response has been delivered. - """ - - - -class ResponseFailed(_WrapperException): - """ - L{ResponseFailed} indicates that all of the response to a request was not - received for some reason. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the response was considered to have failed. - - @ivar response: If specified, the L{Response} received from the server (and - in particular the status code and the headers). - """ - - def __init__(self, reasons, response=None): - _WrapperException.__init__(self, reasons) - self.response = response - - - -class ResponseNeverReceived(ResponseFailed): - """ - A L{ResponseFailed} that knows no response bytes at all have been received. - """ - - - -class RequestNotSent(Exception): - """ - L{RequestNotSent} indicates that an attempt was made to issue a request but - for reasons unrelated to the details of the request itself, the request - could not be sent. For example, this may indicate that an attempt was made - to send a request using a protocol which is no longer connected to a - server. - """ - - - -def _callAppFunction(function): - """ - Call C{function}. If it raises an exception, log it with a minimal - description of the source. - - @return: C{None} - """ - try: - function() - except: - log.err(None, "Unexpected exception from %s" % ( - fullyQualifiedName(function),)) - - - -class HTTPParser(LineReceiver): - """ - L{HTTPParser} handles the parsing side of HTTP processing. With a suitable - subclass, it can parse either the client side or the server side of the - connection. - - @ivar headers: All of the non-connection control message headers yet - received. - - @ivar state: State indicator for the response parsing state machine. One - of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}. - - @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline - header while that header is being received. - """ - - # NOTE: According to HTTP spec, we're supposed to eat the - # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that - # doesn't sound like a good idea to me, because it makes it impossible to - # have a non-authenticating transparent proxy in front of an authenticating - # proxy. An authenticating proxy can eat them itself. -jknight - # - # Further, quoting - # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html - # regarding the 'Proxy-Connection' header: - # - # The Proxy-Connection: header is a mistake in how some web browsers - # use HTTP. Its name is the result of a false analogy. It is not a - # standard part of the protocol. There is a different standard - # protocol mechanism for doing what it does. And its existence - # imposes a requirement upon HTTP servers such that no proxy HTTP - # server can be standards-conforming in practice. - # - # -exarkun - - # Some servers (like http://news.ycombinator.com/) return status lines and - # HTTP headers delimited by \n instead of \r\n. - delimiter = '\n' - - CONNECTION_CONTROL_HEADERS = set([ - 'content-length', 'connection', 'keep-alive', 'te', 'trailers', - 'transfer-encoding', 'upgrade', 'proxy-connection']) - - def connectionMade(self): - self.headers = Headers() - self.connHeaders = Headers() - self.state = STATUS - self._partialHeader = None - - - def switchToBodyMode(self, decoder): - """ - Switch to body parsing mode - interpret any more bytes delivered as - part of the message body and deliver them to the given decoder. - """ - if self.state == BODY: - raise RuntimeError("already in body mode") - - self.bodyDecoder = decoder - self.state = BODY - self.setRawMode() - - - def lineReceived(self, line): - """ - Handle one line from a response. - """ - # Handle the normal CR LF case. - if line[-1:] == '\r': - line = line[:-1] - - if self.state == STATUS: - self.statusReceived(line) - self.state = HEADER - elif self.state == HEADER: - if not line or line[0] not in ' \t': - if self._partialHeader is not None: - header = ''.join(self._partialHeader) - name, value = header.split(':', 1) - value = value.strip() - self.headerReceived(name, value) - if not line: - # Empty line means the header section is over. - self.allHeadersReceived() - else: - # Line not beginning with LWS is another header. - self._partialHeader = [line] - else: - # A line beginning with LWS is a continuation of a header - # begun on a previous line. - self._partialHeader.append(line) - - - def rawDataReceived(self, data): - """ - Pass data from the message body to the body decoder object. - """ - self.bodyDecoder.dataReceived(data) - - - def isConnectionControlHeader(self, name): - """ - Return C{True} if the given lower-cased name is the name of a - connection control header (rather than an entity header). - - According to RFC 2616, section 14.10, the tokens in the Connection - header are probably relevant here. However, I am not sure what the - practical consequences of either implementing or ignoring that are. - So I leave it unimplemented for the time being. - """ - return name in self.CONNECTION_CONTROL_HEADERS - - - def statusReceived(self, status): - """ - Callback invoked whenever the first line of a new message is received. - Override this. - - @param status: The first line of an HTTP request or response message - without trailing I{CR LF}. - @type status: C{str} - """ - - - def headerReceived(self, name, value): - """ - Store the given header in C{self.headers}. - """ - name = name.lower() - if self.isConnectionControlHeader(name): - headers = self.connHeaders - else: - headers = self.headers - headers.addRawHeader(name, value) - - - def allHeadersReceived(self): - """ - Callback invoked after the last header is passed to C{headerReceived}. - Override this to change to the C{BODY} or C{DONE} state. - """ - self.switchToBodyMode(None) - - - -class HTTPClientParser(HTTPParser): - """ - An HTTP parser which only handles HTTP responses. - - @ivar request: The request with which the expected response is associated. - @type request: L{Request} - - @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a - body. - - @ivar finisher: A callable to invoke when this response is fully parsed. - - @ivar _responseDeferred: A L{Deferred} which will be called back with the - response when all headers in the response have been received. - Thereafter, C{None}. - - @ivar _everReceivedData: C{True} if any bytes have been received. - """ - NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED]) - - _transferDecoders = { - 'chunked': _ChunkedTransferDecoder, - } - - bodyDecoder = None - - def __init__(self, request, finisher): - self.request = request - self.finisher = finisher - self._responseDeferred = Deferred() - self._everReceivedData = False - - - def dataReceived(self, data): - """ - Override so that we know if any response has been received. - """ - self._everReceivedData = True - HTTPParser.dataReceived(self, data) - - - def parseVersion(self, strversion): - """ - Parse version strings of the form Protocol '/' Major '.' Minor. E.g. - 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError - on bad syntax. - """ - try: - proto, strnumber = strversion.split('/') - major, minor = strnumber.split('.') - major, minor = int(major), int(minor) - except ValueError as e: - raise BadResponseVersion(str(e), strversion) - if major < 0 or minor < 0: - raise BadResponseVersion("version may not be negative", strversion) - return (proto, major, minor) - - - def statusReceived(self, status): - """ - Parse the status line into its components and create a response object - to keep track of this response's state. - """ - parts = status.split(' ', 2) - if len(parts) != 3: - raise ParseError("wrong number of parts", status) - - try: - statusCode = int(parts[1]) - except ValueError: - raise ParseError("non-integer status code", status) - - self.response = Response( - self.parseVersion(parts[0]), - statusCode, - parts[2], - self.headers, - self.transport) - - - def _finished(self, rest): - """ - Called to indicate that an entire response has been received. No more - bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are - passed up and the state of this L{HTTPClientParser} is set to I{DONE}. - - @param rest: A C{str} giving any extra bytes delivered to this - L{HTTPClientParser} which are not part of the response being - parsed. - """ - self.state = DONE - self.finisher(rest) - - - def isConnectionControlHeader(self, name): - """ - Content-Length in the response to a HEAD request is an entity header, - not a connection control header. - """ - if self.request.method == 'HEAD' and name == 'content-length': - return False - return HTTPParser.isConnectionControlHeader(self, name) - - - def allHeadersReceived(self): - """ - Figure out how long the response body is going to be by examining - headers and stuff. - """ - if (self.response.code in self.NO_BODY_CODES - or self.request.method == 'HEAD'): - self.response.length = 0 - self._finished(self.clearLineBuffer()) - else: - transferEncodingHeaders = self.connHeaders.getRawHeaders( - 'transfer-encoding') - if transferEncodingHeaders: - - # This could be a KeyError. However, that would mean we do not - # know how to decode the response body, so failing the request - # is as good a behavior as any. Perhaps someday we will want - # to normalize/document/test this specifically, but failing - # seems fine to me for now. - transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()] - - # If anyone ever invents a transfer encoding other than - # chunked (yea right), and that transfer encoding can predict - # the length of the response body, it might be sensible to - # allow the transfer decoder to set the response object's - # length attribute. - else: - contentLengthHeaders = self.connHeaders.getRawHeaders('content-length') - if contentLengthHeaders is None: - contentLength = None - elif len(contentLengthHeaders) == 1: - contentLength = int(contentLengthHeaders[0]) - self.response.length = contentLength - else: - # "HTTP Message Splitting" or "HTTP Response Smuggling" - # potentially happening. Or it's just a buggy server. - raise ValueError( - "Too many Content-Length headers; response is invalid") - - if contentLength == 0: - self._finished(self.clearLineBuffer()) - transferDecoder = None - else: - transferDecoder = lambda x, y: _IdentityTransferDecoder( - contentLength, x, y) - - if transferDecoder is None: - self.response._bodyDataFinished() - else: - # Make sure as little data as possible from the response body - # gets delivered to the response object until the response - # object actually indicates it is ready to handle bytes - # (probably because an application gave it a way to interpret - # them). - self.transport.pauseProducing() - self.switchToBodyMode(transferDecoder( - self.response._bodyDataReceived, - self._finished)) - - # This must be last. If it were first, then application code might - # change some state (for example, registering a protocol to receive the - # response body). Then the pauseProducing above would be wrong since - # the response is ready for bytes and nothing else would ever resume - # the transport. - self._responseDeferred.callback(self.response) - del self._responseDeferred - - - def connectionLost(self, reason): - if self.bodyDecoder is not None: - try: - try: - self.bodyDecoder.noMoreData() - except PotentialDataLoss: - self.response._bodyDataFinished(Failure()) - except _DataLoss: - self.response._bodyDataFinished( - Failure(ResponseFailed([reason, Failure()], - self.response))) - else: - self.response._bodyDataFinished() - except: - # Handle exceptions from both the except suites and the else - # suite. Those functions really shouldn't raise exceptions, - # but maybe there's some buggy application code somewhere - # making things difficult. - log.err() - elif self.state != DONE: - if self._everReceivedData: - exceptionClass = ResponseFailed - else: - exceptionClass = ResponseNeverReceived - self._responseDeferred.errback(Failure(exceptionClass([reason]))) - del self._responseDeferred - - - -class Request: - """ - A L{Request} instance describes an HTTP request to be sent to an HTTP - server. - - @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD', - 'POST', etc. - @type method: C{str} - - @ivar uri: The relative URI of the resource to request. For example, - C{'/foo/bar?baz=quux'}. - @type uri: C{str} - - @ivar headers: Headers to be sent to the server. It is important to - note that this object does not create any implicit headers. So it - is up to the HTTP Client to add required headers such as 'Host'. - @type headers: L{twisted.web.http_headers.Headers} - - @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which - produces the content body to send to the remote HTTP server. - - @ivar persistent: Set to C{True} when you use HTTP persistent connection. - @type persistent: C{bool} - """ - def __init__(self, method, uri, headers, bodyProducer, persistent=False): - self.method = method - self.uri = uri - self.headers = headers - self.bodyProducer = bodyProducer - self.persistent = persistent - - - def _writeHeaders(self, transport, TEorCL): - hosts = self.headers.getRawHeaders('host', ()) - if len(hosts) != 1: - raise BadHeaders("Exactly one Host header required") - - # In the future, having the protocol version be a parameter to this - # method would probably be good. It would be nice if this method - # weren't limited to issuing HTTP/1.1 requests. - requestLines = [] - requestLines.append( - '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) - if not self.persistent: - requestLines.append('Connection: close\r\n') - if TEorCL is not None: - requestLines.append(TEorCL) - for name, values in self.headers.getAllRawHeaders(): - requestLines.extend(['%s: %s\r\n' % (name, v) for v in values]) - requestLines.append('\r\n') - transport.writeSequence(requestLines) - - - def _writeToChunked(self, transport): - """ - Write this request to the given transport using chunked - transfer-encoding to frame the body. - """ - self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n') - encoder = ChunkedEncoder(transport) - encoder.registerProducer(self.bodyProducer, True) - d = self.bodyProducer.startProducing(encoder) - - def cbProduced(ignored): - encoder.unregisterProducer() - def ebProduced(err): - encoder._allowNoMoreWrites() - # Don't call the encoder's unregisterProducer because it will write - # a zero-length chunk. This would indicate to the server that the - # request body is complete. There was an error, though, so we - # don't want to do that. - transport.unregisterProducer() - return err - d.addCallbacks(cbProduced, ebProduced) - return d - - - def _writeToContentLength(self, transport): - """ - Write this request to the given transport using content-length to frame - the body. - """ - self._writeHeaders( - transport, - 'Content-Length: %d\r\n' % (self.bodyProducer.length,)) - - # This Deferred is used to signal an error in the data written to the - # encoder below. It can only errback and it will only do so before too - # many bytes have been written to the encoder and before the producer - # Deferred fires. - finishedConsuming = Deferred() - - # This makes sure the producer writes the correct number of bytes for - # the request body. - encoder = LengthEnforcingConsumer( - self.bodyProducer, transport, finishedConsuming) - - transport.registerProducer(self.bodyProducer, True) - - finishedProducing = self.bodyProducer.startProducing(encoder) - - def combine(consuming, producing): - # This Deferred is returned and will be fired when the first of - # consuming or producing fires. If it's cancelled, forward that - # cancellation to the producer. - def cancelConsuming(ign): - finishedProducing.cancel() - ultimate = Deferred(cancelConsuming) - - # Keep track of what has happened so far. This initially - # contains None, then an integer uniquely identifying what - # sequence of events happened. See the callbacks and errbacks - # defined below for the meaning of each value. - state = [None] - - def ebConsuming(err): - if state == [None]: - # The consuming Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. The - # producing Deferred should not fire later (because the - # consumer should have called stopProducing on the - # producer), but if it does, a callback will be ignored - # and an errback will be logged. - state[0] = 1 - ultimate.errback(err) - else: - # The consuming Deferred errbacked after the producing - # Deferred fired. This really shouldn't ever happen. - # If it does, I goofed. Log the error anyway, just so - # there's a chance someone might notice and complain. - log.err( - err, - "Buggy state machine in %r/[%d]: " - "ebConsuming called" % (self, state[0])) - - def cbProducing(result): - if state == [None]: - # The producing Deferred succeeded first. Nothing will - # ever happen to the consuming Deferred. Tell the - # encoder we're done so it can check what the producer - # wrote and make sure it was right. - state[0] = 2 - try: - encoder._noMoreWritesExpected() - except: - # Fail the overall writeTo Deferred - something the - # producer did was wrong. - ultimate.errback() - else: - # Success - succeed the overall writeTo Deferred. - ultimate.callback(None) - # Otherwise, the consuming Deferred already errbacked. The - # producing Deferred wasn't supposed to fire, but it did - # anyway. It's buggy, but there's not really anything to be - # done about it. Just ignore this result. - - def ebProducing(err): - if state == [None]: - # The producing Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. - # Tell the encoder that we're done so it knows to reject - # further writes from the producer (which should not - # happen, but the producer may be buggy). - state[0] = 3 - encoder._allowNoMoreWrites() - ultimate.errback(err) - else: - # The producing Deferred failed after the consuming - # Deferred failed. It shouldn't have, so it's buggy. - # Log the exception in case anyone who can fix the code - # is watching. - log.err(err, "Producer is buggy") - - consuming.addErrback(ebConsuming) - producing.addCallbacks(cbProducing, ebProducing) - - return ultimate - - d = combine(finishedConsuming, finishedProducing) - def f(passthrough): - # Regardless of what happens with the overall Deferred, once it - # fires, the producer registered way up above the definition of - # combine should be unregistered. - transport.unregisterProducer() - return passthrough - d.addBoth(f) - return d - - - def writeTo(self, transport): - """ - Format this L{Request} as an HTTP/1.1 request and write it to the given - transport. If bodyProducer is not None, it will be associated with an - L{IConsumer}. - - @return: A L{Deferred} which fires with C{None} when the request has - been completely written to the transport or with a L{Failure} if - there is any problem generating the request bytes. - """ - if self.bodyProducer is not None: - if self.bodyProducer.length is UNKNOWN_LENGTH: - return self._writeToChunked(transport) - else: - return self._writeToContentLength(transport) - else: - self._writeHeaders(transport, None) - return succeed(None) - - - def stopWriting(self): - """ - Stop writing this request to the transport. This can only be called - after C{writeTo} and before the L{Deferred} returned by C{writeTo} - fires. It should cancel any asynchronous task started by C{writeTo}. - The L{Deferred} returned by C{writeTo} need not be fired if this method - is called. - """ - # If bodyProducer is None, then the Deferred returned by writeTo has - # fired already and this method cannot be called. - _callAppFunction(self.bodyProducer.stopProducing) - - - -class LengthEnforcingConsumer: - """ - An L{IConsumer} proxy which enforces an exact length requirement on the - total data written to it. - - @ivar _length: The number of bytes remaining to be written. - - @ivar _producer: The L{IBodyProducer} which is writing to this - consumer. - - @ivar _consumer: The consumer to which at most C{_length} bytes will be - forwarded. - - @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too - many bytes are written to this consumer. - """ - def __init__(self, producer, consumer, finished): - self._length = producer.length - self._producer = producer - self._consumer = consumer - self._finished = finished - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self._finished = None - - - def write(self, bytes): - """ - Write C{bytes} to the underlying consumer unless - C{_noMoreWritesExpected} has been called or there are/have been too - many bytes. - """ - if self._finished is None: - # No writes are supposed to happen any more. Try to convince the - # calling code to stop calling this method by calling its - # stopProducing method and then throwing an exception at it. This - # exception isn't documented as part of the API because you're - # never supposed to expect it: only buggy code will ever receive - # it. - self._producer.stopProducing() - raise ExcessWrite() - - if len(bytes) <= self._length: - self._length -= len(bytes) - self._consumer.write(bytes) - else: - # No synchronous exception is raised in *this* error path because - # we still have _finished which we can use to report the error to a - # better place than the direct caller of this method (some - # arbitrary application code). - _callAppFunction(self._producer.stopProducing) - self._finished.errback(WrongBodyLength("too many bytes written")) - self._allowNoMoreWrites() - - - def _noMoreWritesExpected(self): - """ - Called to indicate no more bytes will be written to this consumer. - Check to see that the correct number have been written. - - @raise WrongBodyLength: If not enough bytes have been written. - """ - if self._finished is not None: - self._allowNoMoreWrites() - if self._length: - raise WrongBodyLength("too few bytes written") - - - -def makeStatefulDispatcher(name, template): - """ - Given a I{dispatch} name and a function, return a function which can be - used as a method and which, when called, will call another method defined - on the instance and return the result. The other method which is called is - determined by the value of the C{_state} attribute of the instance. - - @param name: A string which is used to construct the name of the subsidiary - method to invoke. The subsidiary method is named like C{'_%s_%s' % - (name, _state)}. - - @param template: A function object which is used to give the returned - function a docstring. - - @return: The dispatcher function. - """ - def dispatcher(self, *args, **kwargs): - func = getattr(self, '_' + name + '_' + self._state, None) - if func is None: - raise RuntimeError( - "%r has no %s method in state %s" % (self, name, self._state)) - return func(*args, **kwargs) - dispatcher.__doc__ = template.__doc__ - return dispatcher - - - -class Response: - """ - A L{Response} instance describes an HTTP response received from an HTTP - server. - - L{Response} should not be subclassed or instantiated. - - @ivar _transport: The transport which is delivering this response. - - @ivar _bodyProtocol: The L{IProtocol} provider to which the body is - delivered. C{None} before one has been registered with - C{deliverBody}. - - @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived} - before C{deliverBody} is called. C{None} afterwards. - - @ivar _state: Indicates what state this L{Response} instance is in, - particularly with respect to delivering bytes from the response body - to an application-suppled protocol object. This may be one of - C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'}, - with the following meanings: - - - INITIAL: This is the state L{Response} objects start in. No - protocol has yet been provided and the underlying transport may - still have bytes to deliver to it. - - - DEFERRED_CLOSE: If the underlying transport indicates all bytes - have been delivered but no application-provided protocol is yet - available, the L{Response} moves to this state. Data is - buffered and waiting for a protocol to be delivered to. - - - CONNECTED: If a protocol is provided when the state is INITIAL, - the L{Response} moves to this state. Any buffered data is - delivered and any data which arrives from the transport - subsequently is given directly to the protocol. - - - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state, - the L{Response} moves to this state after delivering all - buffered data to the protocol. Otherwise, if the L{Response} is - in the CONNECTED state, if the transport indicates there is no - more data, the L{Response} moves to this state. Nothing else - can happen once the L{Response} is in this state. - """ - implements(IResponse) - - length = UNKNOWN_LENGTH - - _bodyProtocol = None - _bodyFinished = False - - def __init__(self, version, code, phrase, headers, _transport): - self.version = version - self.code = code - self.phrase = phrase - self.headers = headers - self._transport = _transport - self._bodyBuffer = [] - self._state = 'INITIAL' - - - def deliverBody(self, protocol): - """ - Dispatch the given L{IProtocol} depending of the current state of the - response. - """ - deliverBody = makeStatefulDispatcher('deliverBody', deliverBody) - - - def _deliverBody_INITIAL(self, protocol): - """ - Deliver any buffered data to C{protocol} and prepare to deliver any - future data to it. Move to the C{'CONNECTED'} state. - """ - # Now that there's a protocol to consume the body, resume the - # transport. It was previously paused by HTTPClientParser to avoid - # reading too much data before it could be handled. - self._transport.resumeProducing() - - protocol.makeConnection(self._transport) - self._bodyProtocol = protocol - for data in self._bodyBuffer: - self._bodyProtocol.dataReceived(data) - self._bodyBuffer = None - self._state = 'CONNECTED' - - - def _deliverBody_CONNECTED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol when it is - already being delivered to another protocol. - """ - raise RuntimeError( - "Response already has protocol %r, cannot deliverBody " - "again" % (self._bodyProtocol,)) - - - def _deliverBody_DEFERRED_CLOSE(self, protocol): - """ - Deliver any buffered data to C{protocol} and then disconnect the - protocol. Move to the C{'FINISHED'} state. - """ - # Unlike _deliverBody_INITIAL, there is no need to resume the - # transport here because all of the response data has been received - # already. Some higher level code may want to resume the transport if - # that code expects further data to be received over it. - - protocol.makeConnection(self._transport) - - for data in self._bodyBuffer: - protocol.dataReceived(data) - self._bodyBuffer = None - protocol.connectionLost(self._reason) - self._state = 'FINISHED' - - - def _deliverBody_FINISHED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol after the - response body has been delivered to another protocol. - """ - raise RuntimeError( - "Response already finished, cannot deliverBody now.") - - - def _bodyDataReceived(self, data): - """ - Called by HTTPClientParser with chunks of data from the response body. - They will be buffered or delivered to the protocol passed to - deliverBody. - """ - _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived', - _bodyDataReceived) - - - def _bodyDataReceived_INITIAL(self, data): - """ - Buffer any data received for later delivery to a protocol passed to - C{deliverBody}. - - Little or no data should be buffered by this method, since the - transport has been paused and will not be resumed until a protocol - is supplied. - """ - self._bodyBuffer.append(data) - - - def _bodyDataReceived_CONNECTED(self, data): - """ - Deliver any data received to the protocol to which this L{Response} - is connected. - """ - self._bodyProtocol.dataReceived(data) - - - def _bodyDataReceived_DEFERRED_CLOSE(self, data): - """ - It is invalid for data to be delivered after it has been indicated - that the response body has been completely delivered. - """ - raise RuntimeError("Cannot receive body data after _bodyDataFinished") - - - def _bodyDataReceived_FINISHED(self, data): - """ - It is invalid for data to be delivered after the response body has - been delivered to a protocol. - """ - raise RuntimeError("Cannot receive body data after protocol disconnected") - - - def _bodyDataFinished(self, reason=None): - """ - Called by HTTPClientParser when no more body data is available. If the - optional reason is supplied, this indicates a problem or potential - problem receiving all of the response body. - """ - _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished', - _bodyDataFinished) - - - def _bodyDataFinished_INITIAL(self, reason=None): - """ - Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to - which to deliver the response body. - """ - self._state = 'DEFERRED_CLOSE' - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._reason = reason - - - def _bodyDataFinished_CONNECTED(self, reason=None): - """ - Disconnect the protocol and move to the C{'FINISHED'} state. - """ - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._bodyProtocol.connectionLost(reason) - self._bodyProtocol = None - self._state = 'FINISHED' - - - def _bodyDataFinished_DEFERRED_CLOSE(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data more than once") - - - def _bodyDataFinished_FINISHED(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data after protocol disconnected") - - - -class ChunkedEncoder: - """ - Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol} - for streaming request bodies to the server. - """ - implements(IConsumer) - - def __init__(self, transport): - self.transport = transport - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self.transport = None - - - def registerProducer(self, producer, streaming): - """ - Register the given producer with C{self.transport}. - """ - self.transport.registerProducer(producer, streaming) - - - def write(self, data): - """ - Write the given request body bytes to the transport using chunked - encoding. - - @type data: C{str} - """ - if self.transport is None: - raise ExcessWrite() - self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n")) - - - def unregisterProducer(self): - """ - Indicate that the request body is complete and finish the request. - """ - self.write('') - self.transport.unregisterProducer() - self._allowNoMoreWrites() - - - -class TransportProxyProducer: - """ - An L{IPushProducer} implementation which wraps another such thing and - proxies calls to it until it is told to stop. - - @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after - this proxy has been stopped. - """ - implements(IPushProducer) - - # LineReceiver uses this undocumented attribute of transports to decide - # when to stop calling lineReceived or rawDataReceived (if it finds it to - # be true, it doesn't bother to deliver any more data). Set disconnecting - # to False here and never change it to true so that all data is always - # delivered to us and so that LineReceiver doesn't fail with an - # AttributeError. - disconnecting = False - - def __init__(self, producer): - self._producer = producer - - - def _stopProxying(self): - """ - Stop forwarding calls of L{IPushProducer} methods to the underlying - L{IPushProvider} provider. - """ - self._producer = None - - - def stopProducing(self): - """ - Proxy the stoppage to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.stopProducing() - - - def resumeProducing(self): - """ - Proxy the resumption to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.resumeProducing() - - - def pauseProducing(self): - """ - Proxy the pause to the underlying producer, unless this proxy has been - stopped. - """ - if self._producer is not None: - self._producer.pauseProducing() - - - -class HTTP11ClientProtocol(Protocol): - """ - L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client - protocol. It supports as few features as possible. - - @ivar _parser: After a request is issued, the L{HTTPClientParser} to - which received data making up the response to that request is - delivered. - - @ivar _finishedRequest: After a request is issued, the L{Deferred} which - will fire when a L{Response} object corresponding to that request is - available. This allows L{HTTP11ClientProtocol} to fail the request - if there is a connection or parsing problem. - - @ivar _currentRequest: After a request is issued, the L{Request} - instance used to make that request. This allows - L{HTTP11ClientProtocol} to stop request generation if necessary (for - example, if the connection is lost). - - @ivar _transportProxy: After a request is issued, the - L{TransportProxyProducer} to which C{_parser} is connected. This - allows C{_parser} to pause and resume the transport in a way which - L{HTTP11ClientProtocol} can exert some control over. - - @ivar _responseDeferred: After a request is issued, the L{Deferred} from - C{_parser} which will fire with a L{Response} when one has been - received. This is eventually chained with C{_finishedRequest}, but - only in certain cases to avoid double firing that Deferred. - - @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance - is in with respect to transmission of a request and reception of a - response. This may be one of the following strings: - - - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances - start in. Nothing is happening: no request is being sent and no - response is being received or expected. - - - TRANSMITTING: When a request is made (via L{request}), the - instance moves to this state. L{Request.writeTo} has been used - to start to send a request but it has not yet finished. - - - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a - complete response but the request has not yet been fully sent - yet. The instance will remain in this state until the request - is fully sent. - - - GENERATION_FAILED: There was an error while the request. The - request was not fully sent to the network. - - - WAITING: The request was fully sent to the network. The - instance is now waiting for the response to be fully received. - - - ABORTING: Application code has requested that the HTTP connection - be aborted. - - - CONNECTION_LOST: The connection has been lost. - - @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when - the connection is lost. - """ - _state = 'QUIESCENT' - _parser = None - _finishedRequest = None - _currentRequest = None - _transportProxy = None - _responseDeferred = None - - - def __init__(self, quiescentCallback=lambda c: None): - self._quiescentCallback = quiescentCallback - self._abortDeferreds = [] - - - @property - def state(self): - return self._state - - - def request(self, request): - """ - Issue C{request} over C{self.transport} and return a L{Deferred} which - will fire with a L{Response} instance or an error. - - @param request: The object defining the parameters of the request to - issue. - @type request: L{Request} - - @rtype: L{Deferred} - @return: The deferred may errback with L{RequestGenerationFailed} if - the request was not fully written to the transport due to a local - error. It may errback with L{RequestTransmissionFailed} if it was - not fully written to the transport due to a network error. It may - errback with L{ResponseFailed} if the request was sent (not - necessarily received) but some or all of the response was lost. It - may errback with L{RequestNotSent} if it is not possible to send - any more requests using this L{HTTP11ClientProtocol}. - """ - if self._state != 'QUIESCENT': - return fail(RequestNotSent()) - - self._state = 'TRANSMITTING' - _requestDeferred = maybeDeferred(request.writeTo, self.transport) - - def cancelRequest(ign): - # Explicitly cancel the request's deferred if it's still trying to - # write when this request is cancelled. - if self._state in ( - 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'): - _requestDeferred.cancel() - else: - self.transport.abortConnection() - self._disconnectParser(Failure(CancelledError())) - self._finishedRequest = Deferred(cancelRequest) - - # Keep track of the Request object in case we need to call stopWriting - # on it. - self._currentRequest = request - - self._transportProxy = TransportProxyProducer(self.transport) - self._parser = HTTPClientParser(request, self._finishResponse) - self._parser.makeConnection(self._transportProxy) - self._responseDeferred = self._parser._responseDeferred - - def cbRequestWrotten(ignored): - if self._state == 'TRANSMITTING': - self._state = 'WAITING' - self._responseDeferred.chainDeferred(self._finishedRequest) - - def ebRequestWriting(err): - if self._state == 'TRANSMITTING': - self._state = 'GENERATION_FAILED' - self.transport.abortConnection() - self._finishedRequest.errback( - Failure(RequestGenerationFailed([err]))) - else: - log.err(err, 'Error writing request, but not in valid state ' - 'to finalize request: %s' % self._state) - - _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) - - return self._finishedRequest - - - def _finishResponse(self, rest): - """ - Called by an L{HTTPClientParser} to indicate that it has parsed a - complete response. - - @param rest: A C{str} giving any trailing bytes which were given to - the L{HTTPClientParser} which were not part of the response it - was parsing. - """ - _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse) - - - def _finishResponse_WAITING(self, rest): - # Currently the rest parameter is ignored. Don't forget to use it if - # we ever add support for pipelining. And maybe check what trailers - # mean. - if self._state == 'WAITING': - self._state = 'QUIESCENT' - else: - # The server sent the entire response before we could send the - # whole request. That sucks. Oh well. Fire the request() - # Deferred with the response. But first, make sure that if the - # request does ever finish being written that it won't try to fire - # that Deferred. - self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' - self._responseDeferred.chainDeferred(self._finishedRequest) - - # This will happen if we're being called due to connection being lost; - # if so, no need to disconnect parser again, or to call - # _quiescentCallback. - if self._parser is None: - return - - reason = ConnectionDone("synthetic!") - connHeaders = self._parser.connHeaders.getRawHeaders('connection', ()) - if (('close' in connHeaders) or self._state != "QUIESCENT" or - not self._currentRequest.persistent): - self._giveUp(Failure(reason)) - else: - # We call the quiescent callback first, to ensure connection gets - # added back to connection pool before we finish the request. - try: - self._quiescentCallback(self) - except: - # If callback throws exception, just log it and disconnect; - # keeping persistent connections around is an optimisation: - log.err() - self.transport.loseConnection() - self._disconnectParser(reason) - - - _finishResponse_TRANSMITTING = _finishResponse_WAITING - - - def _disconnectParser(self, reason): - """ - If there is still a parser, call its C{connectionLost} method with the - given reason. If there is not, do nothing. - - @type reason: L{Failure} - """ - if self._parser is not None: - parser = self._parser - self._parser = None - self._currentRequest = None - self._finishedRequest = None - self._responseDeferred = None - - # The parser is no longer allowed to do anything to the real - # transport. Stop proxying from the parser's transport to the real - # transport before telling the parser it's done so that it can't do - # anything. - self._transportProxy._stopProxying() - self._transportProxy = None - parser.connectionLost(reason) - - - def _giveUp(self, reason): - """ - Lose the underlying connection and disconnect the parser with the given - L{Failure}. - - Use this method instead of calling the transport's loseConnection - method directly otherwise random things will break. - """ - self.transport.loseConnection() - self._disconnectParser(reason) - - - def dataReceived(self, bytes): - """ - Handle some stuff from some place. - """ - try: - self._parser.dataReceived(bytes) - except: - self._giveUp(Failure()) - - - def connectionLost(self, reason): - """ - The underlying transport went away. If appropriate, notify the parser - object. - """ - connectionLost = makeStatefulDispatcher('connectionLost', connectionLost) - - - def _connectionLost_QUIESCENT(self, reason): - """ - Nothing is currently happening. Move to the C{'CONNECTION_LOST'} - state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_GENERATION_FAILED(self, reason): - """ - The connection was in an inconsistent state. Move to the - C{'CONNECTION_LOST'} state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_TRANSMITTING(self, reason): - """ - Fail the L{Deferred} for the current request, notify the request - object that it does not need to continue transmitting itself, and - move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - self._finishedRequest.errback( - Failure(RequestTransmissionFailed([reason]))) - del self._finishedRequest - - # Tell the request that it should stop bothering now. - self._currentRequest.stopWriting() - - - def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason): - """ - Move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_WAITING(self, reason): - """ - Disconnect the response parser so that it can propagate the event as - necessary (for example, to call an application protocol's - C{connectionLost} method, or to fail a request L{Deferred}) and move - to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(reason) - self._state = 'CONNECTION_LOST' - - - def _connectionLost_ABORTING(self, reason): - """ - Disconnect the response parser with a L{ConnectionAborted} failure, and - move to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(Failure(ConnectionAborted())) - self._state = 'CONNECTION_LOST' - for d in self._abortDeferreds: - d.callback(None) - self._abortDeferreds = [] - - - def abort(self): - """ - Close the connection and cause all outstanding L{request} L{Deferred}s - to fire with an error. - """ - if self._state == "CONNECTION_LOST": - return succeed(None) - self.transport.loseConnection() - self._state = 'ABORTING' - d = Deferred() - self._abortDeferreds.append(d) - return d -}}} ''' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index 8e0b1df8b..c2d50648a 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -42,1129 +42,13 @@ from twisted.web.client import ( Agent, ProxyAgent, HTTPConnectionPool, readBody, ) -''' {{{ -class PartialDownloadError(error.Error): - """ - Page was only partially downloaded, we got disconnected in middle. - - @ivar response: All of the response body which was downloaded. - """ - - -class _URL(tuple): - """ - A parsed URL. - - At some point this should be replaced with a better URL implementation. - """ - def __new__(self, scheme, host, port, path): - return tuple.__new__(_URL, (scheme, host, port, path)) - - - def __init__(self, scheme, host, port, path): - self.scheme = scheme - self.host = host - self.port = port - self.path = path - - -def _parse(url, defaultPort=None): - """ - Split the given URL into the scheme, host, port, and path. - - @type url: C{bytes} - @param url: An URL to parse. - - @type defaultPort: C{int} or C{None} - @param defaultPort: An alternate value to use as the port if the URL does - not include one. - - @return: A four-tuple of the scheme, host, port, and path of the URL. All - of these are C{bytes} instances except for port, which is an C{int}. - """ - url = url.strip() - parsed = http.urlparse(url) - scheme = parsed[0] - path = urlunparse((b'', b'') + parsed[2:]) - - if defaultPort is None: - if scheme == b'https': - defaultPort = 443 - else: - defaultPort = 80 - - host, port = parsed[1], defaultPort - if b':' in host: - host, port = host.split(b':') - try: - port = int(port) - except ValueError: - port = defaultPort - - if path == b'': - path = b'/' - - return _URL(scheme, host, port, path) - - -def _makeGetterFactory(url, factoryFactory, contextFactory=None, - *args, **kwargs): - """ - Create and connect an HTTP page getting factory. - - Any additional positional or keyword arguments are used when calling - C{factoryFactory}. - - @param factoryFactory: Factory factory that is called with C{url}, C{args} - and C{kwargs} to produce the getter - - @param contextFactory: Context factory to use when creating a secure - connection, defaulting to C{None} - - @return: The factory created by C{factoryFactory} - """ - scheme, host, port, path = _parse(url) - factory = factoryFactory(url, *args, **kwargs) - if scheme == b'https': - from twisted.internet import ssl - if contextFactory is None: - contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(host, port, factory, contextFactory) - else: - reactor.connectTCP(host, port, factory) - return factory -}}} ''' # The code which follows is based on the new HTTP client implementation. It # should be significantly better than anything above, though it is not yet # feature equivalent. -#from twisted.web.error import SchemeNotSupported from twisted.web._newclient import Response -#from twisted.web._newclient import Request, HTTP11ClientProtocol from twisted.web._newclient import ResponseDone, ResponseFailed -#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed -#from twisted.web._newclient import ( -# ResponseNeverReceived, PotentialDataLoss, _WrapperException) - -''' {{{ -try: - from twisted.internet.ssl import ClientContextFactory -except ImportError: - class WebClientContextFactory(object): - """ - A web context factory which doesn't work because the necessary SSL - support is missing. - """ - def getContext(self, hostname, port): - raise NotImplementedError("SSL support unavailable") -else: - class WebClientContextFactory(ClientContextFactory): - """ - A web context factory which ignores the hostname and port and does no - certificate verification. - """ - def getContext(self, hostname, port): - return ClientContextFactory.getContext(self) - - - -class _WebToNormalContextFactory(object): - """ - Adapt a web context factory to a normal context factory. - - @ivar _webContext: A web context factory which accepts a hostname and port - number to its C{getContext} method. - - @ivar _hostname: The hostname which will be passed to - C{_webContext.getContext}. - - @ivar _port: The port number which will be passed to - C{_webContext.getContext}. - """ - def __init__(self, webContext, hostname, port): - self._webContext = webContext - self._hostname = hostname - self._port = port - - - def getContext(self): - """ - Called the wrapped web context factory's C{getContext} method with a - hostname and port number and return the resulting context object. - """ - return self._webContext.getContext(self._hostname, self._port) - - - -@implementer(IBodyProducer) -class FileBodyProducer(object): - """ - L{FileBodyProducer} produces bytes from an input file object incrementally - and writes them to a consumer. - - Since file-like objects cannot be read from in an event-driven manner, - L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from - the file. This process is also paused and resumed based on notifications - from the L{IConsumer} provider being written to. - - The file is closed after it has been read, or if the producer is stopped - early. - - @ivar _inputFile: Any file-like object, bytes read from which will be - written to a consumer. - - @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to - schedule all reads. - - @ivar _readSize: The number of bytes to read from C{_inputFile} at a time. - """ - - # Python 2.4 doesn't have these symbolic constants - _SEEK_SET = getattr(os, 'SEEK_SET', 0) - _SEEK_END = getattr(os, 'SEEK_END', 2) - - def __init__(self, inputFile, cooperator=task, readSize=2 ** 16): - self._inputFile = inputFile - self._cooperate = cooperator.cooperate - self._readSize = readSize - self.length = self._determineLength(inputFile) - - - def _determineLength(self, fObj): - """ - Determine how many bytes can be read out of C{fObj} (assuming it is not - modified from this point on). If the determination cannot be made, - return C{UNKNOWN_LENGTH}. - """ - try: - seek = fObj.seek - tell = fObj.tell - except AttributeError: - return UNKNOWN_LENGTH - originalPosition = tell() - seek(0, self._SEEK_END) - end = tell() - seek(originalPosition, self._SEEK_SET) - return end - originalPosition - - - def stopProducing(self): - """ - Permanently stop writing bytes from the file to the consumer by - stopping the underlying L{CooperativeTask}. - """ - self._inputFile.close() - self._task.stop() - - - def startProducing(self, consumer): - """ - Start a cooperative task which will read bytes from the input file and - write them to C{consumer}. Return a L{Deferred} which fires after all - bytes have been written. - - @param consumer: Any L{IConsumer} provider - """ - self._task = self._cooperate(self._writeloop(consumer)) - d = self._task.whenDone() - def maybeStopped(reason): - # IBodyProducer.startProducing's Deferred isn't support to fire if - # stopProducing is called. - reason.trap(task.TaskStopped) - return defer.Deferred() - d.addCallbacks(lambda ignored: None, maybeStopped) - return d - - - def _writeloop(self, consumer): - """ - Return an iterator which reads one chunk of bytes from the input file - and writes them to the consumer for each time it is iterated. - """ - while True: - bytes = self._inputFile.read(self._readSize) - if not bytes: - self._inputFile.close() - break - consumer.write(bytes) - yield None - - - def pauseProducing(self): - """ - Temporarily suspend copying bytes from the input file to the consumer - by pausing the L{CooperativeTask} which drives that activity. - """ - self._task.pause() - - - def resumeProducing(self): - """ - Undo the effects of a previous C{pauseProducing} and resume copying - bytes to the consumer by resuming the L{CooperativeTask} which drives - the write activity. - """ - self._task.resume() - - - -class _HTTP11ClientFactory(protocol.Factory): - """ - A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}. - - @ivar _quiescentCallback: The quiescent callback to be passed to protocol - instances, used to return them to the connection pool. - - @since: 11.1 - """ - def __init__(self, quiescentCallback): - self._quiescentCallback = quiescentCallback - - - def buildProtocol(self, addr): - return HTTP11ClientProtocol(self._quiescentCallback) - - - -class _RetryingHTTP11ClientProtocol(object): - """ - A wrapper for L{HTTP11ClientProtocol} that automatically retries requests. - - @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}. - - @ivar _newConnection: A callable that creates a new connection for a - retry. - """ - - def __init__(self, clientProtocol, newConnection): - self._clientProtocol = clientProtocol - self._newConnection = newConnection - - - def _shouldRetry(self, method, exception, bodyProducer): - """ - Indicate whether request should be retried. - - Only returns C{True} if method is idempotent, no response was - received, the reason for the failed request was not due to - user-requested cancellation, and no body was sent. The latter - requirement may be relaxed in the future, and PUT added to approved - method list. - """ - if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"): - return False - if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed, - ResponseNeverReceived)): - return False - if isinstance(exception, _WrapperException): - for failure in exception.reasons: - if failure.check(defer.CancelledError): - return False - if bodyProducer is not None: - return False - return True - - - def request(self, request): - """ - Do a request, and retry once (with a new connection) it it fails in - a retryable manner. - - @param request: A L{Request} instance that will be requested using the - wrapped protocol. - """ - d = self._clientProtocol.request(request) - - def failed(reason): - if self._shouldRetry(request.method, reason.value, - request.bodyProducer): - return self._newConnection().addCallback( - lambda connection: connection.request(request)) - else: - return reason - d.addErrback(failed) - return d - - - -class HTTPConnectionPool(object): - """ - A pool of persistent HTTP connections. - - Features: - - Cached connections will eventually time out. - - Limits on maximum number of persistent connections. - - Connections are stored using keys, which should be chosen such that any - connections stored under a given key can be used interchangeably. - - Failed requests done using previously cached connections will be retried - once if they use an idempotent method (e.g. GET), in case the HTTP server - timed them out. - - @ivar persistent: Boolean indicating whether connections should be - persistent. Connections are persistent by default. - - @ivar maxPersistentPerHost: The maximum number of cached persistent - connections for a C{host:port} destination. - @type maxPersistentPerHost: C{int} - - @ivar cachedConnectionTimeout: Number of seconds a cached persistent - connection will stay open before disconnecting. - - @ivar retryAutomatically: C{boolean} indicating whether idempotent - requests should be retried once if no response was received. - - @ivar _factory: The factory used to connect to the proxy. - - @ivar _connections: Map (scheme, host, port) to lists of - L{HTTP11ClientProtocol} instances. - - @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a - C{IDelayedCall} instance of their timeout. - - @since: 12.1 - """ - - _factory = _HTTP11ClientFactory - maxPersistentPerHost = 2 - cachedConnectionTimeout = 240 - retryAutomatically = True - - def __init__(self, reactor, persistent=True): - self._reactor = reactor - self.persistent = persistent - self._connections = {} - self._timeouts = {} - - - def getConnection(self, key, endpoint): - """ - Supply a connection, newly created or retrieved from the pool, to be - used for one HTTP request. - - The connection will remain out of the pool (not available to be - returned from future calls to this method) until one HTTP request has - been completed over it. - - Afterwards, if the connection is still open, it will automatically be - added to the pool. - - @param key: A unique key identifying connections that can be used - interchangeably. - - @param endpoint: An endpoint that can be used to open a new connection - if no cached connection is available. - - @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol} - (or a wrapper) that can be used to send a single HTTP request. - """ - # Try to get cached version: - connections = self._connections.get(key) - while connections: - connection = connections.pop(0) - # Cancel timeout: - self._timeouts[connection].cancel() - del self._timeouts[connection] - if connection.state == "QUIESCENT": - if self.retryAutomatically: - newConnection = lambda: self._newConnection(key, endpoint) - connection = _RetryingHTTP11ClientProtocol( - connection, newConnection) - return defer.succeed(connection) - - return self._newConnection(key, endpoint) - - - def _newConnection(self, key, endpoint): - """ - Create a new connection. - - This implements the new connection code path for L{getConnection}. - """ - def quiescentCallback(protocol): - self._putConnection(key, protocol) - factory = self._factory(quiescentCallback) - return endpoint.connect(factory) - - - def _removeConnection(self, key, connection): - """ - Remove a connection from the cache and disconnect it. - """ - connection.transport.loseConnection() - self._connections[key].remove(connection) - del self._timeouts[connection] - - - def _putConnection(self, key, connection): - """ - Return a persistent connection to the pool. This will be called by - L{HTTP11ClientProtocol} when the connection becomes quiescent. - """ - if connection.state != "QUIESCENT": - # Log with traceback for debugging purposes: - try: - raise RuntimeError( - "BUG: Non-quiescent protocol added to connection pool.") - except: - log.err() - return - connections = self._connections.setdefault(key, []) - if len(connections) == self.maxPersistentPerHost: - dropped = connections.pop(0) - dropped.transport.loseConnection() - self._timeouts[dropped].cancel() - del self._timeouts[dropped] - connections.append(connection) - cid = self._reactor.callLater(self.cachedConnectionTimeout, - self._removeConnection, - key, connection) - self._timeouts[connection] = cid - - - def closeCachedConnections(self): - """ - Close all persistent connections and remove them from the pool. - - @return: L{defer.Deferred} that fires when all connections have been - closed. - """ - results = [] - for protocols in self._connections.itervalues(): - for p in protocols: - results.append(p.abort()) - self._connections = {} - for dc in self._timeouts.values(): - dc.cancel() - self._timeouts = {} - return defer.gatherResults(results).addCallback(lambda ign: None) - - - -class _AgentBase(object): - """ - Base class offering common facilities for L{Agent}-type classes. - - @ivar _reactor: The C{IReactorTime} implementation which will be used by - the pool, and perhaps by subclasses as well. - - @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections. - """ - - def __init__(self, reactor, pool): - if pool is None: - pool = HTTPConnectionPool(reactor, False) - self._reactor = reactor - self._pool = pool - - - def _computeHostValue(self, scheme, host, port): - """ - Compute the string to use for the value of the I{Host} header, based on - the given scheme, host name, and port number. - """ - if (scheme, port) in (('http', 80), ('https', 443)): - return host - return '%s:%d' % (host, port) - - - def _requestWithEndpoint(self, key, endpoint, method, parsedURI, - headers, bodyProducer, requestPath): - """ - Issue a new request, given the endpoint and the path sent as part of - the request. - """ - # Create minimal headers, if necessary: - if headers is None: - headers = Headers() - if not headers.hasHeader('host'): - #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us - headers.addRawHeader( - 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, - parsedURI.port)) - - d = self._pool.getConnection(key, endpoint) - def cbConnected(proto): - return proto.request( - Request(method, requestPath, headers, bodyProducer, - persistent=self._pool.persistent)) - d.addCallback(cbConnected) - return d - - - -class Agent(_AgentBase): - """ - L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS} - scheme URIs (but performs no certificate checking by default). - - @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a - non-persistent L{HTTPConnectionPool} instance will be created. - - @ivar _contextFactory: A web context factory which will be used to create - SSL context objects for any SSL connections the agent needs to make. - - @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP} - or C{connectSSL} for specifying the connection timeout. - - @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or - C{connectSSL} for specifying the local address to bind to. - - @since: 9.0 - """ - - def __init__(self, reactor, contextFactory=WebClientContextFactory(), - connectTimeout=None, bindAddress=None, - pool=None): - _AgentBase.__init__(self, reactor, pool) - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - - - def _wrapContextFactory(self, host, port): - """ - Create and return a normal context factory wrapped around - C{self._contextFactory} in such a way that C{self._contextFactory} will - have the host and port information passed to it. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: A context factory suitable to be passed to - C{reactor.connectSSL}. - """ - return _WebToNormalContextFactory(self._contextFactory, host, port) - - - def _getEndpoint(self, scheme, host, port): - """ - Get an endpoint for the given host and port, using a transport - selected based on scheme. - - @param scheme: A string like C{'http'} or C{'https'} (the only two - supported values) to use to determine how to establish the - connection. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: An endpoint which can be used to connect to given address. - """ - kwargs = {} - if self._connectTimeout is not None: - kwargs['timeout'] = self._connectTimeout - kwargs['bindAddress'] = self._bindAddress - if scheme == 'http': - return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) - elif scheme == 'https': - return SSL4ClientEndpoint(self._reactor, host, port, - self._wrapContextFactory(host, port), - **kwargs) - else: - raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,)) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request. - - @param method: The request method to send. - @type method: C{str} - - @param uri: The request URI send. - @type uri: C{str} - - @param headers: The request headers to send. If no I{Host} header is - included, one will be added based on the request URI. - @type headers: L{Headers} - - @param bodyProducer: An object which will produce the request body or, - if the request body is to be empty, L{None}. - @type bodyProducer: L{IBodyProducer} provider - - @return: A L{Deferred} which fires with the result of the request (a - L{twisted.web.iweb.IResponse} provider), or fails if there is a - problem setting up a connection over which to issue the request. - It may also fail with L{SchemeNotSupported} if the scheme of the - given URI is not supported. - @rtype: L{Deferred} - """ - parsedURI = _parse(uri) - try: - endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host, - parsedURI.port) - except SchemeNotSupported: - return defer.fail(Failure()) - key = (parsedURI.scheme, parsedURI.host, parsedURI.port) - return self._requestWithEndpoint(key, endpoint, method, parsedURI, - headers, bodyProducer, parsedURI.path) - - - -class ProxyAgent(_AgentBase): - """ - An HTTP agent able to cross HTTP proxies. - - @ivar _proxyEndpoint: The endpoint used to connect to the proxy. - - @since: 11.1 - """ - - def __init__(self, endpoint, reactor=None, pool=None): - if reactor is None: - from twisted.internet import reactor - _AgentBase.__init__(self, reactor, pool) - self._proxyEndpoint = endpoint - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request via the configured proxy. - """ - # Cache *all* connections under the same key, since we are only - # connecting to a single destination, the proxy: - key = ("http-proxy", self._proxyEndpoint) - - # To support proxying HTTPS via CONNECT, we will use key - # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that - # wraps _proxyEndpoint with an additional callback to do the CONNECT. - return self._requestWithEndpoint(key, self._proxyEndpoint, method, - _parse(uri), headers, bodyProducer, - uri) - - - -class _FakeUrllib2Request(object): - """ - A fake C{urllib2.Request} object for C{cookielib} to work with. - - @see: U{http://docs.python.org/library/urllib2.html#request-objects} - - @type uri: C{str} - @ivar uri: Request URI. - - @type headers: L{twisted.web.http_headers.Headers} - @ivar headers: Request headers. - - @type type: C{str} - @ivar type: The scheme of the URI. - - @type host: C{str} - @ivar host: The host[:port] of the URI. - - @since: 11.1 - """ - def __init__(self, uri): - self.uri = uri - self.headers = Headers() - self.type, rest = splittype(self.uri) - self.host, rest = splithost(rest) - - - def has_header(self, header): - return self.headers.hasHeader(header) - - - def add_unredirected_header(self, name, value): - self.headers.addRawHeader(name, value) - - - def get_full_url(self): - return self.uri - - - def get_header(self, name, default=None): - headers = self.headers.getRawHeaders(name, default) - if headers is not None: - return headers[0] - return None - - - def get_host(self): - return self.host - - - def get_type(self): - return self.type - - - def is_unverifiable(self): - # In theory this shouldn't be hardcoded. - return False - - - -class _FakeUrllib2Response(object): - """ - A fake C{urllib2.Response} object for C{cookielib} to work with. - - @type response: C{twisted.web.iweb.IResponse} - @ivar response: Underlying Twisted Web response. - - @since: 11.1 - """ - def __init__(self, response): - self.response = response - - - def info(self): - class _Meta(object): - def getheaders(zelf, name): - return self.response.headers.getRawHeaders(name, []) - return _Meta() - - - -class CookieAgent(object): - """ - L{CookieAgent} extends the basic L{Agent} to add RFC-compliant - handling of HTTP cookies. Cookies are written to and extracted - from a C{cookielib.CookieJar} instance. - - The same cookie jar instance will be used for any requests through this - agent, mutating it whenever a I{Set-Cookie} header appears in a response. - - @type _agent: L{twisted.web.client.Agent} - @ivar _agent: Underlying Twisted Web agent to issue requests through. - - @type cookieJar: C{cookielib.CookieJar} - @ivar cookieJar: Initialized cookie jar to read cookies from and store - cookies to. - - @since: 11.1 - """ - def __init__(self, agent, cookieJar): - self._agent = agent - self.cookieJar = cookieJar - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request to the wrapped L{Agent}. - - Send a I{Cookie} header if a cookie for C{uri} is stored in - L{CookieAgent.cookieJar}. Cookies are automatically extracted and - stored from requests. - - If a C{'cookie'} header appears in C{headers} it will override the - automatic cookie header obtained from the cookie jar. - - @see: L{Agent.request} - """ - if headers is None: - headers = Headers() - lastRequest = _FakeUrllib2Request(uri) - # Setting a cookie header explicitly will disable automatic request - # cookies. - if not headers.hasHeader('cookie'): - self.cookieJar.add_cookie_header(lastRequest) - cookieHeader = lastRequest.get_header('Cookie', None) - if cookieHeader is not None: - headers = headers.copy() - headers.addRawHeader('cookie', cookieHeader) - - d = self._agent.request(method, uri, headers, bodyProducer) - d.addCallback(self._extractCookies, lastRequest) - return d - - - def _extractCookies(self, response, request): - """ - Extract response cookies and store them in the cookie jar. - - @type response: L{twisted.web.iweb.IResponse} - @param response: Twisted Web response. - - @param request: A urllib2 compatible request object. - """ - resp = _FakeUrllib2Response(response) - self.cookieJar.extract_cookies(resp, request) - return response - - - -class GzipDecoder(proxyForInterface(IResponse)): - """ - A wrapper for a L{Response} instance which handles gzip'ed body. - - @ivar original: The original L{Response} object. - - @since: 11.1 - """ - - def __init__(self, response): - self.original = response - self.length = UNKNOWN_LENGTH - - - def deliverBody(self, protocol): - """ - Override C{deliverBody} to wrap the given C{protocol} with - L{_GzipProtocol}. - """ - self.original.deliverBody(_GzipProtocol(protocol, self.original)) - - - -class _GzipProtocol(proxyForInterface(IProtocol)): - """ - A L{Protocol} implementation which wraps another one, transparently - decompressing received data. - - @ivar _zlibDecompress: A zlib decompress object used to decompress the data - stream. - - @ivar _response: A reference to the original response, in case of errors. - - @since: 11.1 - """ - - def __init__(self, protocol, response): - self.original = protocol - self._response = response - self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS) - - - def dataReceived(self, data): - """ - Decompress C{data} with the zlib decompressor, forwarding the raw data - to the original protocol. - """ - try: - rawData = self._zlibDecompress.decompress(data) - except zlib.error: - raise ResponseFailed([failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - - - def connectionLost(self, reason): - """ - Forward the connection lost event, flushing remaining data from the - decompressor if any. - """ - try: - rawData = self._zlibDecompress.flush() - except zlib.error: - raise ResponseFailed([reason, failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - self.original.connectionLost(reason) - - - -class ContentDecoderAgent(object): - """ - An L{Agent} wrapper to handle encoded content. - - It takes care of declaring the support for content in the - I{Accept-Encoding} header, and automatically decompresses the received data - if it's effectively using compression. - - @param decoders: A list or tuple of (name, decoder) objects. The name - declares which decoding the decoder supports, and the decoder must - return a response object when called/instantiated. For example, - C{(('gzip', GzipDecoder))}. The order determines how the decoders are - going to be advertized to the server. - - @since: 11.1 - """ - - def __init__(self, agent, decoders): - self._agent = agent - self._decoders = dict(decoders) - self._supported = ','.join([decoder[0] for decoder in decoders]) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request which declares supporting compressed content. - - @see: L{Agent.request}. - """ - if headers is None: - headers = Headers() - else: - headers = headers.copy() - headers.addRawHeader('accept-encoding', self._supported) - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback(self._handleResponse) - - - def _handleResponse(self, response): - """ - Check if the response is encoded, and wrap it to handle decompression. - """ - contentEncodingHeaders = response.headers.getRawHeaders( - 'content-encoding', []) - contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',') - while contentEncodingHeaders: - name = contentEncodingHeaders.pop().strip() - decoder = self._decoders.get(name) - if decoder is not None: - response = decoder(response) - else: - # Add it back - contentEncodingHeaders.append(name) - break - if contentEncodingHeaders: - response.headers.setRawHeaders( - 'content-encoding', [','.join(contentEncodingHeaders)]) - else: - response.headers.removeHeader('content-encoding') - return response - - - -class RedirectAgent(object): - """ - An L{Agent} wrapper which handles HTTP redirects. - - The implementation is rather strict: 301 and 302 behaves like 307, not - redirecting automatically on methods different from C{GET} and C{HEAD}. - - @param redirectLimit: The maximum number of times the agent is allowed to - follow redirects before failing with a L{error.InfiniteRedirection}. - - @since: 11.1 - """ - - def __init__(self, agent, redirectLimit=20): - self._agent = agent - self._redirectLimit = redirectLimit - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request following HTTP redirects. - - @see: L{Agent.request}. - """ - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback( - self._handleResponse, method, uri, headers, 0) - - - def _handleRedirect(self, response, method, uri, headers, redirectCount): - """ - Handle a redirect response, checking the number of redirects already - followed, and extracting the location header fields. - """ - if redirectCount >= self._redirectLimit: - err = error.InfiniteRedirection( - response.code, - 'Infinite redirection detected', - location=uri) - raise ResponseFailed([failure.Failure(err)], response) - locationHeaders = response.headers.getRawHeaders('location', []) - if not locationHeaders: - err = error.RedirectWithNoLocation( - response.code, 'No location header field', uri) - raise ResponseFailed([failure.Failure(err)], response) - location = locationHeaders[0] - deferred = self._agent.request(method, location, headers) - return deferred.addCallback( - self._handleResponse, method, uri, headers, redirectCount + 1) - - - def _handleResponse(self, response, method, uri, headers, redirectCount): - """ - Handle the response, making another request if it indicates a redirect. - """ - if response.code in (http.MOVED_PERMANENTLY, http.FOUND, - http.TEMPORARY_REDIRECT): - if method not in ('GET', 'HEAD'): - err = error.PageRedirect(response.code, location=uri) - raise ResponseFailed([failure.Failure(err)], response) - return self._handleRedirect(response, method, uri, headers, - redirectCount) - elif response.code == http.SEE_OTHER: - return self._handleRedirect(response, 'GET', uri, headers, - redirectCount) - return response - - - -class _ReadBodyProtocol(protocol.Protocol): - """ - Protocol that collects data sent to it. - - This is a helper for L{IResponse.deliverBody}, which collects the body and - fires a deferred with it. - - @ivar deferred: See L{__init__}. - @ivar status: See L{__init__}. - @ivar message: See L{__init__}. - - @ivar dataBuffer: list of byte-strings received - @type dataBuffer: L{list} of L{bytes} - """ - - def __init__(self, status, message, deferred): - """ - @param status: Status of L{IResponse} - @ivar status: L{int} - - @param message: Message of L{IResponse} - @type message: L{bytes} - - @param deferred: deferred to fire when response is complete - @type deferred: L{Deferred} firing with L{bytes} - """ - self.deferred = deferred - self.status = status - self.message = message - self.dataBuffer = [] - - - def dataReceived(self, data): - """ - Accumulate some more bytes from the response. - """ - self.dataBuffer.append(data) - - - def connectionLost(self, reason): - """ - Deliver the accumulated response bytes to the waiting L{Deferred}, if - the response body has been completely received without error. - """ - if reason.check(ResponseDone): - self.deferred.callback(b''.join(self.dataBuffer)) - elif reason.check(PotentialDataLoss): - self.deferred.errback( - PartialDownloadError(self.status, self.message, - b''.join(self.dataBuffer))) - else: - self.deferred.errback(reason) - - - -def readBody(response): - """ - Get the body of an L{IResponse} and return it as a byte string. - - This is a helper function for clients that don't want to incrementally - receive the body of an HTTP response. - - @param response: The HTTP response for which the body will be read. - @type response: L{IResponse} provider - - @return: A L{Deferred} which will fire with the body of the response. - """ - d = defer.Deferred() - response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) - return d -}}} ''' __all__ = [ diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index 3f4704064..197e43ed3 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -14,27 +14,6 @@ parsed by the L{clientFromString} and L{serverFromString} functions. from __future__ import division, absolute_import -import os -#import socket - -from zope.interface import implementer, directlyProvides -import warnings - -from twisted.internet import interfaces, defer, error, fdesc -#from twisted.internet.protocol import ( -# ClientFactory, Protocol) -from twisted.internet.protocol import Factory -#from twisted.internet import threads, ProcessProtocol -from twisted.internet.interfaces import IStreamServerEndpointStringParser -#from twisted.internet.interfaces import IStreamClientEndpointStringParser -from twisted.python.filepath import FilePath -#from twisted.python.failure import Failure -#from twisted.python import log -#from twisted.python.components import proxyForInterface - -from twisted.plugin import IPlugin, getPlugins -#from twisted.internet import stdio - from twisted.internet.endpoints import ( clientFromString, serverFromString, quoteStringArgument, TCP4ServerEndpoint, TCP6ServerEndpoint, @@ -44,1236 +23,4 @@ from twisted.internet.endpoints import ( AdoptedStreamServerEndpoint, connectProtocol, ) - __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - -''' {{{ -class _WrappingProtocol(Protocol): - """ - Wrap another protocol in order to notify my user when a connection has - been made. - """ - - def __init__(self, connectedDeferred, wrappedProtocol): - """ - @param connectedDeferred: The L{Deferred} that will callback - with the C{wrappedProtocol} when it is connected. - - @param wrappedProtocol: An L{IProtocol} provider that will be - connected. - """ - self._connectedDeferred = connectedDeferred - self._wrappedProtocol = wrappedProtocol - - for iface in [interfaces.IHalfCloseableProtocol, - interfaces.IFileDescriptorReceiver]: - if iface.providedBy(self._wrappedProtocol): - directlyProvides(self, iface) - - - def logPrefix(self): - """ - Transparently pass through the wrapped protocol's log prefix. - """ - if interfaces.ILoggingContext.providedBy(self._wrappedProtocol): - return self._wrappedProtocol.logPrefix() - return self._wrappedProtocol.__class__.__name__ - - - def connectionMade(self): - """ - Connect the C{self._wrappedProtocol} to our C{self.transport} and - callback C{self._connectedDeferred} with the C{self._wrappedProtocol} - """ - self._wrappedProtocol.makeConnection(self.transport) - self._connectedDeferred.callback(self._wrappedProtocol) - - - def dataReceived(self, data): - """ - Proxy C{dataReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.dataReceived(data) - - - def fileDescriptorReceived(self, descriptor): - """ - Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.fileDescriptorReceived(descriptor) - - - def connectionLost(self, reason): - """ - Proxy C{connectionLost} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.connectionLost(reason) - - - def readConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.readConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.readConnectionLost() - - - def writeConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.writeConnectionLost() - - - -class _WrappingFactory(ClientFactory): - """ - Wrap a factory in order to wrap the protocols it builds. - - @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol - method will be called and whose resulting protocol will be wrapped. - - @ivar _onConnection: A L{Deferred} that fires when the protocol is - connected - - @ivar _connector: A L{connector } - that is managing the current or previous connection attempt. - """ - protocol = _WrappingProtocol - - def __init__(self, wrappedFactory): - """ - @param wrappedFactory: A provider of I{IProtocolFactory} whose - buildProtocol method will be called and whose resulting protocol - will be wrapped. - """ - self._wrappedFactory = wrappedFactory - self._onConnection = defer.Deferred(canceller=self._canceller) - - - def startedConnecting(self, connector): - """ - A connection attempt was started. Remember the connector which started - said attempt, for use later. - """ - self._connector = connector - - - def _canceller(self, deferred): - """ - The outgoing connection attempt was cancelled. Fail that L{Deferred} - with an L{error.ConnectingCancelledError}. - - @param deferred: The L{Deferred } that was cancelled; - should be the same as C{self._onConnection}. - @type deferred: L{Deferred } - - @note: This relies on startedConnecting having been called, so it may - seem as though there's a race condition where C{_connector} may not - have been set. However, using public APIs, this condition is - impossible to catch, because a connection API - (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a - L{_WrappingFactory}'s L{Deferred } is returned to - C{connect()}'s caller. - - @return: C{None} - """ - deferred.errback( - error.ConnectingCancelledError( - self._connector.getDestination())) - self._connector.stopConnecting() - - - def doStart(self): - """ - Start notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStart() - - - def doStop(self): - """ - Stop notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStop() - - - def buildProtocol(self, addr): - """ - Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback - the C{self._onConnection} L{Deferred}. - - @return: An instance of L{_WrappingProtocol} or C{None} - """ - try: - proto = self._wrappedFactory.buildProtocol(addr) - except: - self._onConnection.errback() - else: - return self.protocol(self._onConnection, proto) - - - def clientConnectionFailed(self, connector, reason): - """ - Errback the C{self._onConnection} L{Deferred} when the - client connection fails. - """ - if not self._onConnection.called: - self._onConnection.errback(reason) - - - - - -@implementer(interfaces.ITransport) -class _ProcessEndpointTransport(proxyForInterface( - interfaces.IProcessTransport, '_process')): - """ - An L{ITransport} provider for the L{IProtocol} instance passed to the - process endpoint. - - @ivar _process: An active process transport which will be used by write - methods on this object to write data to a child process. - @type _process: L{interfaces.IProcessTransport} provider - """ - - def write(self, data): - """ - Write to the child process's standard input. - - @param data: The data to write on stdin. - """ - self._process.writeToChild(0, data) - - - def writeSequence(self, data): - """ - Write a list of strings to child process's stdin. - - @param data: The list of chunks to write on stdin. - """ - for chunk in data: - self._process.writeToChild(0, chunk) - - -@implementer(interfaces.IStreamServerEndpoint) -class _TCPServerEndpoint(object): - """ - A TCP server endpoint interface - """ - - def __init__(self, reactor, port, backlog, interface): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - self._reactor = reactor - self._port = port - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a TCP - socket - """ - return defer.execute(self._reactor.listenTCP, - self._port, - protocolFactory, - backlog=self._backlog, - interface=self._interface) - - - -class TCP4ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv4 configuration - """ - def __init__(self, reactor, port, backlog=50, interface=''): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -class TCP6ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv6 configuration - """ - def __init__(self, reactor, port, backlog=50, interface='::'): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class TCP4ClientEndpoint(object): - """ - TCP client endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, host, port, timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorTCP} provider - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param timeout: The number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectTCP( - self._host, self._port, wf, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - - -@implementer(interfaces.IStreamServerEndpoint) -class SSL4ServerEndpoint(object): - """ - SSL secured TCP server endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, port, sslContextFactory, - backlog=50, interface=''): - """ - @param reactor: An L{IReactorSSL} provider. - - @param port: The port number used for listening - @type port: int - - @param sslContextFactory: An instance of - L{twisted.internet.ssl.ContextFactory}. - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - self._reactor = reactor - self._port = port - self._sslContextFactory = sslContextFactory - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen for SSL on a - TCP socket. - """ - return defer.execute(self._reactor.listenSSL, self._port, - protocolFactory, - contextFactory=self._sslContextFactory, - backlog=self._backlog, - interface=self._interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class SSL4ClientEndpoint(object): - """ - SSL secured TCP client endpoint with an IPv4 configuration - """ - - def __init__(self, reactor, host, port, sslContextFactory, - timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorSSL} provider. - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param sslContextFactory: SSL Configuration information as an instance - of L{twisted.internet.ssl.ContextFactory}. - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._sslContextFactory = sslContextFactory - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect with SSL over - TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectSSL( - self._host, self._port, wf, self._sslContextFactory, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class UNIXServerEndpoint(object): - """ - UnixSocket server endpoint. - """ - def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - @param address: The path to the Unix socket file, used when listening - @param backlog: number of connections to allow in backlog. - @param mode: mode to set on the unix socket. This parameter is - deprecated. Permissions should be set on the directory which - contains the UNIX socket. - @param wantPID: If True, create a pidfile for the socket. - """ - self._reactor = reactor - self._address = address - self._backlog = backlog - self._mode = mode - self._wantPID = wantPID - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket. - """ - return defer.execute(self._reactor.listenUNIX, self._address, - protocolFactory, - backlog=self._backlog, - mode=self._mode, - wantPID=self._wantPID) - - - -@implementer(interfaces.IStreamClientEndpoint) -class UNIXClientEndpoint(object): - """ - UnixSocket client endpoint. - """ - def __init__(self, reactor, path, timeout=30, checkPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - - @param path: The path to the Unix socket file, used when connecting - @type path: str - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param checkPID: If True, check for a pid file to verify that a server - is listening. - @type checkPID: bool - """ - self._reactor = reactor - self._path = path - self._timeout = timeout - self._checkPID = checkPID - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via a - UNIX Socket - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectUNIX( - self._path, wf, - timeout=self._timeout, - checkPID=self._checkPID) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class AdoptedStreamServerEndpoint(object): - """ - An endpoint for listening on a file descriptor initialized outside of - Twisted. - - @ivar _used: A C{bool} indicating whether this endpoint has been used to - listen with a factory yet. C{True} if so. - """ - _close = os.close - _setNonBlocking = staticmethod(fdesc.setNonBlocking) - - def __init__(self, reactor, fileno, addressFamily): - """ - @param reactor: An L{IReactorSocket} provider. - - @param fileno: An integer file descriptor corresponding to a listening - I{SOCK_STREAM} socket. - - @param addressFamily: The address family of the socket given by - C{fileno}. - """ - self.reactor = reactor - self.fileno = fileno - self.addressFamily = addressFamily - self._used = False - - - def listen(self, factory): - """ - Implement L{IStreamServerEndpoint.listen} to start listening on, and - then close, C{self._fileno}. - """ - if self._used: - return defer.fail(error.AlreadyListened()) - self._used = True - - try: - self._setNonBlocking(self.fileno) - port = self.reactor.adoptStreamPort( - self.fileno, self.addressFamily, factory) - self._close(self.fileno) - except: - return defer.fail() - return defer.succeed(port) - - - - -def _parseTCP(factory, port, interface="", backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a TCP(IPv4) stream endpoint into the structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments - to L{TCP4ServerEndpoint}. - """ - return (int(port), factory), {'interface': interface, - 'backlog': int(backlog)} - - - -def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the - structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param address: the pathname of the unix socket - @type address: C{str} - - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param lockfile: A string '0' or '1', mapping to True and False - respectively. See the C{wantPID} argument to C{listenUNIX} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory, - arguments to L{UNIXServerEndpoint}. - """ - return ( - (address, factory), - {'mode': int(mode, 8), 'backlog': int(backlog), - 'wantPID': bool(int(lockfile))}) - - - -def _parseSSL(factory, port, privateKey="server.pem", certKey=None, - sslmethod=None, interface='', backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for an SSL (over TCP/IPv4) stream endpoint into the structured - arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param privateKey: The file name of a PEM format private key file. - @type privateKey: C{str} - - @param certKey: The file name of a PEM format certificate file. - @type certKey: C{str} - - @param sslmethod: The string name of an SSL method, based on the name of a - constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD", - "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD". - @type sslmethod: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments - to L{SSL4ServerEndpoint}. - """ - from twisted.internet import ssl - if certKey is None: - certKey = privateKey - kw = {} - if sslmethod is not None: - kw['method'] = getattr(ssl.SSL, sslmethod) - else: - kw['method'] = ssl.SSL.SSLv23_METHOD - certPEM = FilePath(certKey).getContent() - keyPEM = FilePath(privateKey).getContent() - privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM) - cf = ssl.CertificateOptions( - privateKey=privateCertificate.privateKey.original, - certificate=privateCertificate.original, - **kw - ) - return ((int(port), factory, cf), - {'interface': interface, 'backlog': int(backlog)}) - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _StandardIOParser(object): - """ - Stream server endpoint string parser for the Standard I/O type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "stdio" - - def _parseServer(self, reactor): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{StandardIOEndpoint} - - @param reactor: Reactor for the endpoint - """ - return StandardIOEndpoint(reactor) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor) - - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _TCP6ServerParser(object): - """ - Stream server endpoint string parser for the TCP6ServerEndpoint type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type - - def _parseServer(self, reactor, port, backlog=50, interface='::'): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{TCP6ServerEndpoint} - - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - port = int(port) - backlog = int(backlog) - return TCP6ServerEndpoint(reactor, port, backlog, interface) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor, *args, **kwargs) - - - -_serverParsers = {"tcp": _parseTCP, - "unix": _parseUNIX, - "ssl": _parseSSL, - } - -_OP, _STRING = range(2) - -def _tokenize(description): - """ - Tokenize a strports string and yield each token. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples - starting with L{_OP} will contain a second element of either ':' (i.e. - 'next parameter') or '=' (i.e. 'assign parameter value'). For example, - the string 'hello:greet\=ing=world' would result in a generator - yielding these values:: - - _STRING, 'hello' - _OP, ':' - _STRING, 'greet=ing' - _OP, '=' - _STRING, 'world' - """ - current = '' - ops = ':=' - nextOps = {':': ':=', '=': ':'} - description = iter(description) - for n in description: - if n in ops: - yield _STRING, current - yield _OP, n - current = '' - ops = nextOps[n] - elif n == '\\': - current += next(description) - else: - current += n - yield _STRING, current - - - -def _parse(description): - """ - Convert a description string into a list of positional and keyword - parameters, using logic vaguely like what Python does. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all - ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of - all C{str}s which do contain an '='. For example, the result of - C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}. - """ - args, kw = [], {} - def add(sofar): - if len(sofar) == 1: - args.append(sofar[0]) - else: - kw[sofar[0]] = sofar[1] - sofar = () - for (type, value) in _tokenize(description): - if type is _STRING: - sofar += (value,) - elif value == ':': - add(sofar) - sofar = () - add(sofar) - return args, kw - - -# Mappings from description "names" to endpoint constructors. -_endpointServerFactories = { - 'TCP': TCP4ServerEndpoint, - 'SSL': SSL4ServerEndpoint, - 'UNIX': UNIXServerEndpoint, - } - -_endpointClientFactories = { - 'TCP': TCP4ClientEndpoint, - 'SSL': SSL4ClientEndpoint, - 'UNIX': UNIXClientEndpoint, - } - - -_NO_DEFAULT = object() - -def _parseServer(description, factory, default=None): - """ - Parse a stports description into a 2-tuple of arguments and keyword values. - - @param description: A description in the format explained by - L{serverFromString}. - @type description: C{str} - - @param factory: A 'factory' argument; this is left-over from - twisted.application.strports, it's not really used. - @type factory: L{IProtocolFactory} or L{None} - - @param default: Deprecated argument, specifying the default parser mode to - use for unqualified description strings (those which do not have a ':' - and prefix). - @type default: C{str} or C{NoneType} - - @return: a 3-tuple of (plugin or name, arguments, keyword arguments) - """ - args, kw = _parse(description) - if not args or (len(args) == 1 and not kw): - deprecationMessage = ( - "Unqualified strport description passed to 'service'." - "Use qualified endpoint descriptions; for example, 'tcp:%s'." - % (description,)) - if default is None: - default = 'tcp' - warnings.warn( - deprecationMessage, category=DeprecationWarning, stacklevel=4) - elif default is _NO_DEFAULT: - raise ValueError(deprecationMessage) - # If the default has been otherwise specified, the user has already - # been warned. - args[0:0] = [default] - endpointType = args[0] - parser = _serverParsers.get(endpointType) - if parser is None: - # If the required parser is not found in _server, check if - # a plugin exists for the endpointType - for plugin in getPlugins(IStreamServerEndpointStringParser): - if plugin.prefix == endpointType: - return (plugin, args[1:], kw) - raise ValueError("Unknown endpoint type: '%s'" % (endpointType,)) - return (endpointType.upper(),) + parser(factory, *args[1:], **kw) - - - -def _serverFromStringLegacy(reactor, description, default): - """ - Underlying implementation of L{serverFromString} which avoids exposing the - deprecated 'default' argument to anything but L{strports.service}. - """ - nameOrPlugin, args, kw = _parseServer(description, None, default) - if type(nameOrPlugin) is not str: - plugin = nameOrPlugin - return plugin.parseStreamServer(reactor, *args, **kw) - else: - name = nameOrPlugin - # Chop out the factory. - args = args[:1] + args[2:] - return _endpointServerFactories[name](reactor, *args, **kw) - - - -def serverFromString(reactor, description): - """ - Construct a stream server endpoint from an endpoint description string. - - The format for server endpoint descriptions is a simple string. It is a - prefix naming the type of endpoint, then a colon, then the arguments for - that endpoint. - - For example, you can call it like this to create an endpoint that will - listen on TCP port 80:: - - serverFromString(reactor, "tcp:80") - - Additional arguments may be specified as keywords, separated with colons. - For example, you can specify the interface for a TCP server endpoint to - bind to like this:: - - serverFromString(reactor, "tcp:80:interface=127.0.0.1") - - SSL server endpoints may be specified with the 'ssl' prefix, and the - private key and certificate files may be specified by the C{privateKey} and - C{certKey} arguments:: - - serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem") - - If a private key file name (C{privateKey}) isn't provided, a "server.pem" - file is assumed to exist which contains the private key. If the certificate - file name (C{certKey}) isn't provided, the private key file is assumed to - contain the certificate as well. - - You may escape colons in arguments with a backslash, which you will need to - use if you want to specify a full pathname argument on Windows:: - - serverFromString(reactor, - "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem") - - finally, the 'unix' prefix may be used to specify a filesystem UNIX socket, - optionally with a 'mode' argument to specify the mode of the socket file - created by C{listen}:: - - serverFromString(reactor, "unix:/var/run/finger") - serverFromString(reactor, "unix:/var/run/finger:mode=660") - - This function is also extensible; new endpoint types may be registered as - L{IStreamServerEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The server endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to listen with the parameters - given by by C{description}. - - @rtype: L{IStreamServerEndpoint} - - @raise ValueError: when the 'description' string cannot be parsed. - - @since: 10.2 - """ - return _serverFromStringLegacy(reactor, description, _NO_DEFAULT) - - - -def quoteStringArgument(argument): - """ - Quote an argument to L{serverFromString} and L{clientFromString}. Since - arguments are separated with colons and colons are escaped with - backslashes, some care is necessary if, for example, you have a pathname, - you may be tempted to interpolate into a string like this:: - - serverFromString("ssl:443:privateKey=%s" % (myPathName,)) - - This may appear to work, but will have portability issues (Windows - pathnames, for example). Usually you should just construct the appropriate - endpoint type rather than interpolating strings, which in this case would - be L{SSL4ServerEndpoint}. There are some use-cases where you may need to - generate such a string, though; for example, a tool to manipulate a - configuration file which has strports descriptions in it. To be correct in - those cases, do this instead:: - - serverFromString("ssl:443:privateKey=%s" % - (quoteStringArgument(myPathName),)) - - @param argument: The part of the endpoint description string you want to - pass through. - - @type argument: C{str} - - @return: The quoted argument. - - @rtype: C{str} - """ - return argument.replace('\\', '\\\\').replace(':', '\\:') - - - -def _parseClientTCP(*args, **kwargs): - """ - Perform any argument value coercion necessary for TCP client parameters. - - Valid positional arguments to this function are host and port. - - Valid keyword arguments to this function are all L{IReactorTCP.connectTCP} - arguments. - - @return: The coerced values as a C{dict}. - """ - - if len(args) == 2: - kwargs['port'] = int(args[1]) - kwargs['host'] = args[0] - elif len(args) == 1: - if 'host' in kwargs: - kwargs['port'] = int(args[0]) - else: - kwargs['host'] = args[0] - - try: - kwargs['port'] = int(kwargs['port']) - except KeyError: - pass - - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - - - -def _loadCAsFromDir(directoryPath): - """ - Load certificate-authority certificate objects in a given directory. - - @param directoryPath: a L{FilePath} pointing at a directory to load .pem - files from. - - @return: a C{list} of L{OpenSSL.crypto.X509} objects. - """ - from twisted.internet import ssl - - caCerts = {} - for child in directoryPath.children(): - if not child.basename().split('.')[-1].lower() == 'pem': - continue - try: - data = child.getContent() - except IOError: - # Permission denied, corrupt disk, we don't care. - continue - try: - theCert = ssl.Certificate.loadPEM(data) - except ssl.SSL.Error: - # Duplicate certificate, invalid certificate, etc. We don't care. - pass - else: - caCerts[theCert.digest()] = theCert.original - return caCerts.values() - - - -def _parseClientSSL(*args, **kwargs): - """ - Perform any argument value coercion necessary for SSL client parameters. - - Valid keyword arguments to this function are all L{IReactorSSL.connectSSL} - arguments except for C{contextFactory}. Instead, C{certKey} (the path name - of the certificate file) C{privateKey} (the path name of the private key - associated with the certificate) are accepted and used to construct a - context factory. - - Valid positional arguments to this function are host and port. - - @param caCertsDir: The one parameter which is not part of - L{IReactorSSL.connectSSL}'s signature, this is a path name used to - construct a list of certificate authority certificates. The directory - will be scanned for files ending in C{.pem}, all of which will be - considered valid certificate authorities for this connection. - - @type caCertsDir: C{str} - - @return: The coerced values as a C{dict}. - """ - from twisted.internet import ssl - kwargs = _parseClientTCP(*args, **kwargs) - certKey = kwargs.pop('certKey', None) - privateKey = kwargs.pop('privateKey', None) - caCertsDir = kwargs.pop('caCertsDir', None) - if certKey is not None: - certx509 = ssl.Certificate.loadPEM( - FilePath(certKey).getContent()).original - else: - certx509 = None - if privateKey is not None: - privateKey = ssl.PrivateCertificate.loadPEM( - FilePath(privateKey).getContent()).privateKey.original - else: - privateKey = None - if caCertsDir is not None: - verify = True - caCerts = _loadCAsFromDir(FilePath(caCertsDir)) - else: - verify = False - caCerts = None - kwargs['sslContextFactory'] = ssl.CertificateOptions( - method=ssl.SSL.SSLv23_METHOD, - certificate=certx509, - privateKey=privateKey, - verify=verify, - caCerts=caCerts - ) - return kwargs - - - -def _parseClientUNIX(*args, **kwargs): - """ - Perform any argument value coercion necessary for UNIX client parameters. - - Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX} - keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted - and has the same meaning. Also C{path} is used instead of C{address}. - - Valid positional arguments to this function are C{path}. - - @return: The coerced values as a C{dict}. - """ - if len(args) == 1: - kwargs['path'] = args[0] - - try: - kwargs['checkPID'] = bool(int(kwargs.pop('lockfile'))) - except KeyError: - pass - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - -_clientParsers = { - 'TCP': _parseClientTCP, - 'SSL': _parseClientSSL, - 'UNIX': _parseClientUNIX, - } - - - -def clientFromString(reactor, description): - """ - Construct a client endpoint from a description string. - - Client description strings are much like server description strings, - although they take all of their arguments as keywords, aside from host and - port. - - You can create a TCP client endpoint with the 'host' and 'port' arguments, - like so:: - - clientFromString(reactor, "tcp:host=www.example.com:port=80") - - or, without specifying host and port keywords:: - - clientFromString(reactor, "tcp:www.example.com:80") - - Or you can specify only one or the other, as in the following 2 examples:: - - clientFromString(reactor, "tcp:host=www.example.com:80") - clientFromString(reactor, "tcp:www.example.com:port=80") - - or an SSL client endpoint with those arguments, plus the arguments used by - the server SSL, for a client certificate:: - - clientFromString(reactor, "ssl:web.example.com:443:" - "privateKey=foo.pem:certKey=foo.pem") - - to specify your certificate trust roots, you can identify a directory with - PEM files in it with the C{caCertsDir} argument:: - - clientFromString(reactor, "ssl:host=web.example.com:port=443:" - "caCertsDir=/etc/ssl/certs") - - You can create a UNIX client endpoint with the 'path' argument and optional - 'lockfile' and 'timeout' arguments:: - - clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9") - - or, with the path as a positional argument with or without optional - arguments as in the following 2 examples:: - - clientFromString(reactor, "unix:/var/foo/bar") - clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9") - - This function is also extensible; new endpoint types may be registered as - L{IStreamClientEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The client endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to connect with the parameters - given by by C{description}. - @rtype: L{IStreamClientEndpoint} - - @since: 10.2 - """ - args, kwargs = _parse(description) - aname = args.pop(0) - name = aname.upper() - for plugin in getPlugins(IStreamClientEndpointStringParser): - if plugin.prefix.upper() == name: - return plugin.parseStreamClient(*args, **kwargs) - if name not in _clientParsers: - raise ValueError("Unknown endpoint type: %r" % (aname,)) - kwargs = _clientParsers[name](*args, **kwargs) - return _endpointClientFactories[name](reactor, **kwargs) - - - -def connectProtocol(endpoint, protocol): - """ - Connect a protocol instance to an endpoint. - - This allows using a client endpoint without having to create a factory. - - @param endpoint: A client endpoint to connect to. - - @param protocol: A protocol instance. - - @return: The result of calling C{connect} on the endpoint, i.e. a - L{Deferred} that will fire with the protocol when connected, or an - appropriate error. - """ - class OneShotFactory(Factory): - def buildProtocol(self, addr): - return protocol - return endpoint.connect(OneShotFactory()) -}}} ''' diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index 7b2a78632..fdcbf3977 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -9,8 +9,6 @@ Maintainer: Itamar Shtull-Trauring from __future__ import division, absolute_import -from zope.interface import Interface, Attribute - from twisted.internet.interfaces import ( IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, @@ -28,2434 +26,3 @@ from twisted.internet.interfaces import ( IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver ) - -''' {{{ -class IAddress(Interface): - """ - An address, e.g. a TCP C{(host, port)}. - - Default implementations are in L{twisted.internet.address}. - """ - -### Reactor Interfaces - -class IConnector(Interface): - """ - Object used to interface between connections and protocols. - - Each L{IConnector} manages one connection. - """ - - def stopConnecting(): - """ - Stop attempting to connect. - """ - - def disconnect(): - """ - Disconnect regardless of the connection state. - - If we are connected, disconnect, if we are trying to connect, - stop trying. - """ - - def connect(): - """ - Try to connect to remote address. - """ - - def getDestination(): - """ - Return destination this will try to connect to. - - @return: An object which provides L{IAddress}. - """ - - - -class IResolverSimple(Interface): - def getHostByName(name, timeout = (1, 3, 11, 45)): - """ - Resolve the domain name C{name} into an IP address. - - @type name: C{str} - @type timeout: C{tuple} - @rtype: L{twisted.internet.defer.Deferred} - @return: The callback of the Deferred that is returned will be - passed a string that represents the IP address of the specified - name, or the errback will be called if the lookup times out. If - multiple types of address records are associated with the name, - A6 records will be returned in preference to AAAA records, which - will be returned in preference to A records. If there are multiple - records of the type to be returned, one will be selected at random. - - @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) - if the name cannot be resolved within the specified timeout period. - """ - - - -class IResolver(IResolverSimple): - def query(query, timeout=None): - """ - Dispatch C{query} to the method which can handle its type. - - @type query: L{twisted.names.dns.Query} - @param query: The DNS query being issued, to which a response is to be - generated. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress(name, timeout=None): - """ - Perform an A record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress6(name, timeout=None): - """ - Perform an A6 record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupIPV6Address(name, timeout=None): - """ - Perform an AAAA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailExchange(name, timeout=None): - """ - Perform an MX record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNameservers(name, timeout=None): - """ - Perform an NS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupCanonicalName(name, timeout=None): - """ - Perform a CNAME record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailBox(name, timeout=None): - """ - Perform an MB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailGroup(name, timeout=None): - """ - Perform an MG record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailRename(name, timeout=None): - """ - Perform an MR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupPointer(name, timeout=None): - """ - Perform a PTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAuthority(name, timeout=None): - """ - Perform an SOA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNull(name, timeout=None): - """ - Perform a NULL record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupWellKnownServices(name, timeout=None): - """ - Perform a WKS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupHostInfo(name, timeout=None): - """ - Perform a HINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailboxInfo(name, timeout=None): - """ - Perform an MINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupText(name, timeout=None): - """ - Perform a TXT record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupResponsibility(name, timeout=None): - """ - Perform an RP record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAFSDatabase(name, timeout=None): - """ - Perform an AFSDB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupService(name, timeout=None): - """ - Perform an SRV record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAllRecords(name, timeout=None): - """ - Perform an ALL_RECORD lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupSenderPolicy(name, timeout= 10): - """ - Perform a SPF record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNamingAuthorityPointer(name, timeout=None): - """ - Perform a NAPTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupZone(name, timeout=None): - """ - Perform an AXFR record lookup. - - NB This is quite different from other DNS requests. See - U{http://cr.yp.to/djbdns/axfr-notes.html} for more - information. - - NB Unlike other C{lookup*} methods, the timeout here is not a - list of ints, it is a single int. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: C{int} - @param timeout: When this timeout expires, the query is - considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. - The first element of the tuple gives answers. - The second and third elements are always empty. - The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - -class IReactorTCP(Interface): - - def listenTCP(port, factory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param backlog: size of the listen queue - - @param interface: The local IPv4 or IPv6 address to which to bind; - defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6 - addresses, you must call this method twice. - - @return: an object that provides L{IListeningPort}. - - @raise CannotListenError: as defined here - L{twisted.internet.error.CannotListenError}, - if it cannot listen on this port (e.g., it - cannot bind to the required port number) - """ - - def connectTCP(host, port, factory, timeout=30, bindAddress=None): - """ - Connect a TCP client. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind - to, or None. - - @return: An object which provides L{IConnector}. This connector will - call various callbacks on the factory when a connection is - made, failed, or lost - see - L{ClientFactory} - docs for details. - """ - -class IReactorSSL(Interface): - - def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None): - """ - Connect a client Protocol to a remote SSL socket. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object. - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind to, - or C{None}. - - @return: An object which provides L{IConnector}. - """ - - def listenSSL(port, factory, contextFactory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - The connection is a SSL one, using contexts created by the context - factory. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance - - @param backlog: size of the listen queue - - @param interface: the hostname to bind to, defaults to '' (all) - """ - - - -class IReactorUNIX(Interface): - """ - UNIX socket methods. - """ - - def connectUNIX(address, factory, timeout=30, checkPID=0): - """ - Connect a client protocol to a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the connection - has failed. - - @param checkPID: if True, check for a pid file to verify that a server - is listening. If C{address} is a Linux abstract namespace path, - this must be C{False}. - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0): - """ - Listen on a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.Factory} instance. - - @param backlog: number of connections to allow in backlog. - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param wantPID: if True, create a pidfile for the socket. If C{address} - is a Linux abstract namespace path, this must be C{False}. - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorUNIXDatagram(Interface): - """ - Datagram UNIX socket methods. - """ - - def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None): - """ - Connect a client protocol to a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param bindAddress: address to bind to - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666): - """ - Listen on a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance. - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorWin32Events(Interface): - """ - Win32 Event API methods - - @since: 10.2 - """ - - def addEvent(event, fd, action): - """ - Add a new win32 event to the event loop. - - @param event: a Win32 event object created using win32event.CreateEvent() - - @param fd: an instance of L{twisted.internet.abstract.FileDescriptor} - - @param action: a string that is a method name of the fd instance. - This method is called in response to the event. - - @return: None - """ - - - def removeEvent(event): - """ - Remove an event. - - @param event: a Win32 event object added using L{IReactorWin32Events.addEvent} - - @return: None - """ - - - -class IReactorUDP(Interface): - """ - UDP socket methods. - """ - - def listenUDP(port, protocol, interface='', maxPacketSize=8192): - """ - Connects a given DatagramProtocol to the given numeric UDP port. - - @return: object which provides L{IListeningPort}. - """ - - - -class IReactorMulticast(Interface): - """ - UDP socket methods that support multicast. - - IMPORTANT: This is an experimental new interface. It may change - without backwards compatibility. Suggestions are welcome. - """ - - def listenMulticast(port, protocol, interface='', maxPacketSize=8192, - listenMultiple=False): - """ - Connects a given - L{DatagramProtocol} to the - given numeric UDP port. - - @param listenMultiple: If set to True, allows multiple sockets to - bind to the same address and port number at the same time. - @type listenMultiple: C{bool} - - @returns: An object which provides L{IListeningPort}. - - @see: L{twisted.internet.interfaces.IMulticastTransport} - @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} - """ - - - -class IReactorSocket(Interface): - """ - Methods which allow a reactor to use externally created sockets. - - For example, to use C{adoptStreamPort} to implement behavior equivalent - to that of L{IReactorTCP.listenTCP}, you might write code like this:: - - from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket - portSocket = socket(AF_INET, SOCK_STREAM) - # Set FD_CLOEXEC on port, left as an exercise. Then make it into a - # non-blocking listening port: - portSocket.setblocking(False) - portSocket.bind(('192.168.1.2', 12345)) - portSocket.listen(SOMAXCONN) - - # Now have the reactor use it as a TCP port - port = reactor.adoptStreamPort( - portSocket.fileno(), AF_INET, YourFactory()) - - # portSocket itself is no longer necessary, and needs to be cleaned - # up by us. - portSocket.close() - - # Whenever the server is no longer needed, stop it as usual. - stoppedDeferred = port.stopListening() - - Another potential use is to inherit a listening descriptor from a parent - process (for example, systemd or launchd), or to receive one over a UNIX - domain socket. - - Some plans for extending this interface exist. See: - - - U{http://twistedmatrix.com/trac/ticket/5570}: established connections - - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports - - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets - """ - - def adoptStreamPort(fileDescriptor, addressFamily, factory): - """ - Add an existing listening I{SOCK_STREAM} socket to the reactor to - monitor for new connections to accept and handle. - - @param fileDescriptor: A file descriptor associated with a socket which - is already bound to an address and marked as listening. The socket - must be set non-blocking. Any additional flags (for example, - close-on-exec) must also be set by application code. Application - code is responsible for closing the file descriptor, which may be - done as soon as C{adoptStreamPort} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create new - protocols to handle connections accepted via this socket. - - @return: An object providing L{IListeningPort}. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - def adoptStreamConnection(fileDescriptor, addressFamily, factory): - """ - Add an existing connected I{SOCK_STREAM} socket to the reactor to - monitor for data. - - Note that the given factory won't have its C{startFactory} and - C{stopFactory} methods called, as there is no sensible time to call - them in this situation. - - @param fileDescriptor: A file descriptor associated with a socket which - is already connected. The socket must be set non-blocking. Any - additional flags (for example, close-on-exec) must also be set by - application code. Application code is responsible for closing the - file descriptor, which may be done as soon as - C{adoptStreamConnection} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create a new - protocol to handle the connection via this socket. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - -class IReactorProcess(Interface): - - def spawnProcess(processProtocol, executable, args=(), env={}, path=None, - uid=None, gid=None, usePTY=0, childFDs=None): - """ - Spawn a process, with a process protocol. - - @type processProtocol: L{IProcessProtocol} provider - @param processProtocol: An object which will be notified of all - events related to the created process. - - @param executable: the file name to spawn - the full path should be - used. - - @param args: the command line arguments to pass to the process; a - sequence of strings. The first string should be the - executable's name. - - @type env: a C{dict} mapping C{str} to C{str}, or C{None}. - @param env: the environment variables to pass to the child process. The - resulting behavior varies between platforms. If - - C{env} is not set: - - On POSIX: pass an empty environment. - - On Windows: pass C{os.environ}. - - C{env} is C{None}: - - On POSIX: pass C{os.environ}. - - On Windows: pass C{os.environ}. - - C{env} is a C{dict}: - - On POSIX: pass the key/value pairs in C{env} as the - complete environment. - - On Windows: update C{os.environ} with the key/value - pairs in the C{dict} before passing it. As a - consequence of U{bug #1640 - }, passing - keys with empty values in an effort to unset - environment variables I{won't} unset them. - - @param path: the path to run the subprocess in - defaults to the - current directory. - - @param uid: user ID to run the subprocess as. (Only available on - POSIX systems.) - - @param gid: group ID to run the subprocess as. (Only available on - POSIX systems.) - - @param usePTY: if true, run this process in a pseudo-terminal. - optionally a tuple of C{(masterfd, slavefd, ttyname)}, - in which case use those file descriptors. - (Not available on all systems.) - - @param childFDs: A dictionary mapping file descriptors in the new child - process to an integer or to the string 'r' or 'w'. - - If the value is an integer, it specifies a file - descriptor in the parent process which will be mapped - to a file descriptor (specified by the key) in the - child process. This is useful for things like inetd - and shell-like file redirection. - - If it is the string 'r', a pipe will be created and - attached to the child at that file descriptor: the - child will be able to write to that file descriptor - and the parent will receive read notification via the - L{IProcessProtocol.childDataReceived} callback. This - is useful for the child's stdout and stderr. - - If it is the string 'w', similar setup to the previous - case will occur, with the pipe being readable by the - child instead of writeable. The parent process can - write to that file descriptor using - L{IProcessTransport.writeToChild}. This is useful for - the child's stdin. - - If childFDs is not passed, the default behaviour is to - use a mapping that opens the usual stdin/stdout/stderr - pipes. - - @see: L{twisted.internet.protocol.ProcessProtocol} - - @return: An object which provides L{IProcessTransport}. - - @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are - insufficient system resources to create a new process. - """ - -class IReactorTime(Interface): - """ - Time methods that a Reactor should implement. - """ - - def seconds(): - """ - Get the current time in seconds. - - @return: A number-like object of some sort. - """ - - - def callLater(delay, callable, *args, **kw): - """ - Call a function later. - - @type delay: C{float} - @param delay: the number of seconds to wait. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: An object which provides L{IDelayedCall} and can be used to - cancel the scheduled call, by calling its C{cancel()} method. - It also may be rescheduled by calling its C{delay()} or - C{reset()} methods. - """ - - - def getDelayedCalls(): - """ - Retrieve all currently scheduled delayed calls. - - @return: A tuple of all L{IDelayedCall} providers representing all - currently scheduled calls. This is everything that has been - returned by C{callLater} but not yet called or canceled. - """ - - -class IDelayedCall(Interface): - """ - A scheduled call. - - There are probably other useful methods we can add to this interface; - suggestions are welcome. - """ - - def getTime(): - """ - Get time when delayed call will happen. - - @return: time in seconds since epoch (a float). - """ - - def cancel(): - """ - Cancel the scheduled call. - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def delay(secondsLater): - """ - Delay the scheduled call. - - @param secondsLater: how many seconds from its current firing time to delay - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def reset(secondsFromNow): - """ - Reset the scheduled call's timer. - - @param secondsFromNow: how many seconds from now it should fire, - equivalent to C{.cancel()} and then doing another - C{reactor.callLater(secondsLater, ...)} - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def active(): - """ - @return: True if this call is still active, False if it has been - called or cancelled. - """ - -class IReactorThreads(Interface): - """ - Dispatch methods to be run in threads. - - Internally, this should use a thread pool and dispatch methods to them. - """ - - def getThreadPool(): - """ - Return the threadpool used by L{callInThread}. Create it first if - necessary. - - @rtype: L{twisted.python.threadpool.ThreadPool} - """ - - - def callInThread(callable, *args, **kwargs): - """ - Run the callable object in a separate thread. - """ - - - def callFromThread(callable, *args, **kw): - """ - Cause a function to be executed by the reactor thread. - - Use this method when you want to run a function in the reactor's thread - from another thread. Calling L{callFromThread} should wake up the main - thread (where L{reactor.run()} is executing) and run the - given callable in that thread. - - If you're writing a multi-threaded application the C{callable} may need - to be thread safe, but this method doesn't require it as such. If you - want to call a function in the next mainloop iteration, but you're in - the same thread, use L{callLater} with a delay of 0. - """ - - - def suggestThreadPoolSize(size): - """ - Suggest the size of the internal threadpool used to dispatch functions - passed to L{callInThread}. - """ - - -class IReactorCore(Interface): - """ - Core methods that a Reactor must implement. - """ - - running = Attribute( - "A C{bool} which is C{True} from I{during startup} to " - "I{during shutdown} and C{False} the rest of the time.") - - - def resolve(name, timeout=10): - """ - Return a L{twisted.internet.defer.Deferred} that will resolve a hostname. - """ - - def run(): - """ - Fire 'startup' System Events, move the reactor to the 'running' - state, then run the main loop until it is stopped with C{stop()} or - C{crash()}. - """ - - def stop(): - """ - Fire 'shutdown' System Events, which will move the reactor to the - 'stopped' state and cause C{reactor.run()} to exit. - """ - - def crash(): - """ - Stop the main loop *immediately*, without firing any system events. - - This is named as it is because this is an extremely "rude" thing to do; - it is possible to lose data and put your system in an inconsistent - state by calling this. However, it is necessary, as sometimes a system - can become wedged in a pre-shutdown call. - """ - - def iterate(delay=0): - """ - Run the main loop's I/O polling function for a period of time. - - This is most useful in applications where the UI is being drawn "as - fast as possible", such as games. All pending L{IDelayedCall}s will - be called. - - The reactor must have been started (via the C{run()} method) prior to - any invocations of this method. It must also be stopped manually - after the last call to this method (via the C{stop()} method). This - method is not re-entrant: you must not call it recursively; in - particular, you must not call it while the reactor is running. - """ - - def fireSystemEvent(eventType): - """ - Fire a system-wide event. - - System-wide events are things like 'startup', 'shutdown', and - 'persist'. - """ - - def addSystemEventTrigger(phase, eventType, callable, *args, **kw): - """ - Add a function to be called when a system event occurs. - - Each "system event" in Twisted, such as 'startup', 'shutdown', and - 'persist', has 3 phases: 'before', 'during', and 'after' (in that - order, of course). These events will be fired internally by the - Reactor. - - An implementor of this interface must only implement those events - described here. - - Callbacks registered for the "before" phase may return either None or a - Deferred. The "during" phase will not execute until all of the - Deferreds from the "before" phase have fired. - - Once the "during" phase is running, all of the remaining triggers must - execute; their return values must be ignored. - - @param phase: a time to call the event -- either the string 'before', - 'after', or 'during', describing when to call it - relative to the event's execution. - - @param eventType: this is a string describing the type of event. - - @param callable: the object to call before shutdown. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: an ID that can be used to remove this call with - removeSystemEventTrigger. - """ - - def removeSystemEventTrigger(triggerID): - """ - Removes a trigger added with addSystemEventTrigger. - - @param triggerID: a value returned from addSystemEventTrigger. - - @raise KeyError: If there is no system event trigger for the given - C{triggerID}. - - @raise ValueError: If there is no system event trigger for the given - C{triggerID}. - - @raise TypeError: If there is no system event trigger for the given - C{triggerID}. - """ - - def callWhenRunning(callable, *args, **kw): - """ - Call a function when the reactor is running. - - If the reactor has not started, the callable will be scheduled - to run when it does start. Otherwise, the callable will be invoked - immediately. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: None if the callable was invoked, otherwise a system - event id for the scheduled call. - """ - - -class IReactorPluggableResolver(Interface): - """ - A reactor with a pluggable name resolver interface. - """ - - def installResolver(resolver): - """ - Set the internal resolver to use to for name lookups. - - @type resolver: An object implementing the L{IResolverSimple} interface - @param resolver: The new resolver to use. - - @return: The previously installed resolver. - """ - - -class IReactorDaemonize(Interface): - """ - A reactor which provides hooks that need to be called before and after - daemonization. - - Notes: - - This interface SHOULD NOT be called by applications. - - This interface should only be implemented by reactors as a workaround - (in particular, it's implemented currently only by kqueue()). - For details please see the comments on ticket #1918. - """ - - def beforeDaemonize(): - """ - Hook to be called immediately before daemonization. No reactor methods - may be called until L{afterDaemonize} is called. - - @return: C{None}. - """ - - - def afterDaemonize(): - """ - Hook to be called immediately after daemonization. This may only be - called after L{beforeDaemonize} had been called previously. - - @return: C{None}. - """ - - - -class IReactorFDSet(Interface): - """ - Implement me to be able to use L{IFileDescriptor} type resources. - - This assumes that your main-loop uses UNIX-style numeric file descriptors - (or at least similarly opaque IDs returned from a .fileno() method) - """ - - def addReader(reader): - """ - I add reader to the set of file descriptors to get read events for. - - @param reader: An L{IReadDescriptor} provider that will be checked for - read events until it is removed from the reactor with - L{removeReader}. - - @return: C{None}. - """ - - def addWriter(writer): - """ - I add writer to the set of file descriptors to get write events for. - - @param writer: An L{IWriteDescriptor} provider that will be checked for - write events until it is removed from the reactor with - L{removeWriter}. - - @return: C{None}. - """ - - def removeReader(reader): - """ - Removes an object previously added with L{addReader}. - - @return: C{None}. - """ - - def removeWriter(writer): - """ - Removes an object previously added with L{addWriter}. - - @return: C{None}. - """ - - def removeAll(): - """ - Remove all readers and writers. - - Should not remove reactor internal reactor connections (like a waker). - - @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers - which were removed. - """ - - def getReaders(): - """ - Return the list of file descriptors currently monitored for input - events by the reactor. - - @return: the list of file descriptors monitored for input events. - @rtype: C{list} of C{IReadDescriptor} - """ - - def getWriters(): - """ - Return the list file descriptors currently monitored for output events - by the reactor. - - @return: the list of file descriptors monitored for output events. - @rtype: C{list} of C{IWriteDescriptor} - """ - - -class IListeningPort(Interface): - """ - A listening port. - """ - - def startListening(): - """ - Start listening on this port. - - @raise CannotListenError: If it cannot listen on this port (e.g., it is - a TCP port and it cannot bind to the required - port number). - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return Deferred that fires - upon completion. - """ - - def getHost(): - """ - Get the host that this port is listening for. - - @return: An L{IAddress} provider. - """ - - -class ILoggingContext(Interface): - """ - Give context information that will be used to log events generated by - this item. - """ - - def logPrefix(): - """ - @return: Prefix used during log formatting to indicate context. - @rtype: C{str} - """ - - - -class IFileDescriptor(ILoggingContext): - """ - An interface representing a UNIX-style numeric file descriptor. - """ - - def fileno(): - """ - @raise: If the descriptor no longer has a valid file descriptor - number associated with it. - - @return: The platform-specified representation of a file descriptor - number. Or C{-1} if the descriptor no longer has a valid file - descriptor number associated with it. As long as the descriptor - is valid, calls to this method on a particular instance must - return the same value. - """ - - - def connectionLost(reason): - """ - Called when the connection was lost. - - This is called when the connection on a selectable object has been - lost. It will be called whether the connection was closed explicitly, - an exception occurred in an event handler, or the other end of the - connection closed it first. - - See also L{IHalfCloseableDescriptor} if your descriptor wants to be - notified separately of the two halves of the connection being closed. - - @param reason: A failure instance indicating the reason why the - connection was lost. L{error.ConnectionLost} and - L{error.ConnectionDone} are of special note, but the - failure may be of other classes as well. - """ - - - -class IReadDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can read. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doRead(): - """ - Some data is available for reading on your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IWriteDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can write. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doWrite(): - """ - Some data can be written to your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor): - """ - An L{IFileDescriptor} that can both read and write. - """ - - -class IHalfCloseableDescriptor(Interface): - """ - A descriptor that can be half-closed. - """ - - def writeConnectionLost(reason): - """ - Indicates write connection was lost. - """ - - def readConnectionLost(reason): - """ - Indicates read connection was lost. - """ - - -class ISystemHandle(Interface): - """ - An object that wraps a networking OS-specific handle. - """ - - def getHandle(): - """ - Return a system- and reactor-specific handle. - - This might be a socket.socket() object, or some other type of - object, depending on which reactor is being used. Use and - manipulate at your own risk. - - This might be used in cases where you want to set specific - options not exposed by the Twisted APIs. - """ - - -class IConsumer(Interface): - """ - A consumer consumes data from a producer. - """ - - def registerProducer(producer, streaming): - """ - Register to receive data from a producer. - - This sets self to be a consumer for a producer. When this object runs - out of data (as when a send(2) call on a socket succeeds in moving the - last data from a userspace buffer into a kernelspace buffer), it will - ask the producer to resumeProducing(). - - For L{IPullProducer} providers, C{resumeProducing} will be called once - each time data is required. - - For L{IPushProducer} providers, C{pauseProducing} will be called - whenever the write buffer fills up and C{resumeProducing} will only be - called when it empties. - - @type producer: L{IProducer} provider - - @type streaming: C{bool} - @param streaming: C{True} if C{producer} provides L{IPushProducer}, - C{False} if C{producer} provides L{IPullProducer}. - - @raise RuntimeError: If a producer is already registered. - - @return: C{None} - """ - - - def unregisterProducer(): - """ - Stop consuming data from a producer, without disconnecting. - """ - - - def write(data): - """ - The producer will write data by calling this method. - - The implementation must be non-blocking and perform whatever - buffering is necessary. If the producer has provided enough data - for now and it is a L{IPushProducer}, the consumer may call its - C{pauseProducing} method. - """ - - - -class IProducer(Interface): - """ - A producer produces data for a consumer. - - Typically producing is done by calling the write method of an class - implementing L{IConsumer}. - """ - - def stopProducing(): - """ - Stop producing data. - - This tells a producer that its consumer has died, so it must stop - producing data for good. - """ - - -class IPushProducer(IProducer): - """ - A push producer, also known as a streaming producer is expected to - produce (write to this consumer) data on a continuous basis, unless - it has been paused. A paused push producer will resume producing - after its resumeProducing() method is called. For a push producer - which is not pauseable, these functions may be noops. - """ - - def pauseProducing(): - """ - Pause producing data. - - Tells a producer that it has produced too much data to process for - the time being, and to stop until resumeProducing() is called. - """ - def resumeProducing(): - """ - Resume producing data. - - This tells a producer to re-add itself to the main loop and produce - more data for its consumer. - """ - -class IPullProducer(IProducer): - """ - A pull producer, also known as a non-streaming producer, is - expected to produce data each time resumeProducing() is called. - """ - - def resumeProducing(): - """ - Produce data for the consumer a single time. - - This tells a producer to produce data for the consumer once - (not repeatedly, once only). Typically this will be done - by calling the consumer's write() method a single time with - produced data. - """ - -class IProtocol(Interface): - - def dataReceived(data): - """ - Called whenever data is received. - - Use this method to translate to a higher-level message. Usually, some - callback will be made upon the receipt of each complete protocol - message. - - @param data: a string of indeterminate length. Please keep in mind - that you will probably need to buffer some data, as partial - (or multiple) protocol messages may be received! I recommend - that unit tests for protocols call through to this method with - differing chunk sizes, down to one byte at a time. - """ - - def connectionLost(reason): - """ - Called when the connection is shut down. - - Clear any circular references here, and any external references - to this Protocol. The connection has been closed. The C{reason} - Failure wraps a L{twisted.internet.error.ConnectionDone} or - L{twisted.internet.error.ConnectionLost} instance (or a subclass - of one of those). - - @type reason: L{twisted.python.failure.Failure} - """ - - def makeConnection(transport): - """ - Make a connection to a transport and a server. - """ - - def connectionMade(): - """ - Called when a connection is made. - - This may be considered the initializer of the protocol, because - it is called when the connection is completed. For clients, - this is called once the connection to the server has been - established; for servers, this is called after an accept() call - stops blocking and a socket has been received. If you need to - send any greeting or initial message, do it here. - """ - - -class IProcessProtocol(Interface): - """ - Interface for process-related event handlers. - """ - - def makeConnection(process): - """ - Called when the process has been created. - - @type process: L{IProcessTransport} provider - @param process: An object representing the process which has been - created and associated with this protocol. - """ - - - def childDataReceived(childFD, data): - """ - Called when data arrives from the child process. - - @type childFD: C{int} - @param childFD: The file descriptor from which the data was - received. - - @type data: C{str} - @param data: The data read from the child's file descriptor. - """ - - - def childConnectionLost(childFD): - """ - Called when a file descriptor associated with the child process is - closed. - - @type childFD: C{int} - @param childFD: The file descriptor which was closed. - """ - - - def processExited(reason): - """ - Called when the child process exits. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - - @since: 8.2 - """ - - - def processEnded(reason): - """ - Called when the child process exits and all file descriptors associated - with it have been closed. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - """ - - - -class IHalfCloseableProtocol(Interface): - """ - Implemented to indicate they want notification of half-closes. - - TCP supports the notion of half-closing the connection, e.g. - closing the write side but still not stopping reading. A protocol - that implements this interface will be notified of such events, - instead of having connectionLost called. - """ - - def readConnectionLost(): - """ - Notification of the read connection being closed. - - This indicates peer did half-close of write side. It is now - the responsibility of the this protocol to call - loseConnection(). In addition, the protocol MUST make sure a - reference to it still exists (i.e. by doing a callLater with - one of its methods, etc.) as the reactor will only have a - reference to it if it is writing. - - If the protocol does not do so, it might get garbage collected - without the connectionLost method ever being called. - """ - - def writeConnectionLost(): - """ - Notification of the write connection being closed. - - This will never be called for TCP connections as TCP does not - support notification of this type of half-close. - """ - - - -class IFileDescriptorReceiver(Interface): - """ - Protocols may implement L{IFileDescriptorReceiver} to receive file - descriptors sent to them. This is useful in conjunction with - L{IUNIXTransport}, which allows file descriptors to be sent between - processes on a single host. - """ - def fileDescriptorReceived(descriptor): - """ - Called when a file descriptor is received over the connection. - - @param descriptor: The descriptor which was received. - @type descriptor: C{int} - - @return: C{None} - """ - - - -class IProtocolFactory(Interface): - """ - Interface for protocol factories. - """ - - def buildProtocol(addr): - """ - Called when a connection has been established to addr. - - If None is returned, the connection is assumed to have been refused, - and the Port will close the connection. - - @type addr: (host, port) - @param addr: The address of the newly-established connection - - @return: None if the connection was refused, otherwise an object - providing L{IProtocol}. - """ - - def doStart(): - """ - Called every time this is connected to a Port or Connector. - """ - - def doStop(): - """ - Called every time this is unconnected from a Port or Connector. - """ - - -class ITransport(Interface): - """ - I am a transport for bytes. - - I represent (and wrap) the physical connection and synchronicity - of the framework which is talking to the network. I make no - representations about whether calls to me will happen immediately - or require returning to a control loop, or whether they will happen - in the same or another thread. Consider methods of this class - (aside from getPeer) to be 'thrown over the wall', to happen at some - indeterminate time. - """ - - def write(data): - """ - Write some data to the physical connection, in sequence, in a - non-blocking fashion. - - If possible, make sure that it is all written. No data will - ever be lost, although (obviously) the connection may be closed - before it all gets through. - """ - - def writeSequence(data): - """ - Write a list of strings to the physical connection. - - If possible, make sure that all of the data is written to - the socket at once, without first copying it all into a - single string. - """ - - def loseConnection(): - """ - Close my connection, after writing all pending data. - - Note that if there is a registered producer on a transport it - will not be closed until the producer has been unregistered. - """ - - def getPeer(): - """ - Get the remote address of this connection. - - Treat this method with caution. It is the unfortunate result of the - CGI and Jabber standards, but should not be considered reliable for - the usual host of reasons; port forwarding, proxying, firewalls, IP - masquerading, etc. - - @return: An L{IAddress} provider. - """ - - def getHost(): - """ - Similar to getPeer, but returns an address describing this side of the - connection. - - @return: An L{IAddress} provider. - """ - - -class ITCPTransport(ITransport): - """ - A TCP based transport. - """ - - def loseWriteConnection(): - """ - Half-close the write side of a TCP connection. - - If the protocol instance this is attached to provides - IHalfCloseableProtocol, it will get notified when the operation is - done. When closing write connection, as with loseConnection this will - only happen when buffer has emptied and there is no registered - producer. - """ - - - def abortConnection(): - """ - Close the connection abruptly. - - Discards any buffered data, stops any registered producer, - and, if possible, notifies the other end of the unclean - closure. - - @since: 11.1 - """ - - - def getTcpNoDelay(): - """ - Return if C{TCP_NODELAY} is enabled. - """ - - def setTcpNoDelay(enabled): - """ - Enable/disable C{TCP_NODELAY}. - - Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are - sent sooner, possibly at the expense of overall throughput. - """ - - def getTcpKeepAlive(): - """ - Return if C{SO_KEEPALIVE} is enabled. - """ - - def setTcpKeepAlive(enabled): - """ - Enable/disable C{SO_KEEPALIVE}. - - Enabling C{SO_KEEPALIVE} sends packets periodically when the connection - is otherwise idle, usually once every two hours. They are intended - to allow detection of lost peers in a non-infinite amount of time. - """ - - def getHost(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - def getPeer(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - - -class IUNIXTransport(ITransport): - """ - Transport for stream-oriented unix domain connections. - """ - def sendFileDescriptor(descriptor): - """ - Send a duplicate of this (file, socket, pipe, etc) descriptor to the - other end of this connection. - - The send is non-blocking and will be queued if it cannot be performed - immediately. The send will be processed in order with respect to other - C{sendFileDescriptor} calls on this transport, but not necessarily with - respect to C{write} calls on this transport. The send can only be - processed if there are also bytes in the normal connection-oriented send - buffer (ie, you must call C{write} at least as many times as you call - C{sendFileDescriptor}). - - @param descriptor: An C{int} giving a valid file descriptor in this - process. Note that a I{file descriptor} may actually refer to a - socket, a pipe, or anything else POSIX tries to treat in the same - way as a file. - - @return: C{None} - """ - - - -class ITLSTransport(ITCPTransport): - """ - A TCP transport that supports switching to TLS midstream. - - Once TLS mode is started the transport will implement L{ISSLTransport}. - """ - - def startTLS(contextFactory): - """ - Initiate TLS negotiation. - - @param contextFactory: A context factory (see L{ssl.py}) - """ - -class ISSLTransport(ITCPTransport): - """ - A SSL/TLS based transport. - """ - - def getPeerCertificate(): - """ - Return an object with the peer's certificate info. - """ - - -class IProcessTransport(ITransport): - """ - A process transport. - """ - - pid = Attribute( - "From before L{IProcessProtocol.makeConnection} is called to before " - "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} " - "giving the platform process ID of this process. C{pid} is L{None} " - "at all other times.") - - def closeStdin(): - """ - Close stdin after all data has been written out. - """ - - def closeStdout(): - """ - Close stdout. - """ - - def closeStderr(): - """ - Close stderr. - """ - - def closeChildFD(descriptor): - """ - Close a file descriptor which is connected to the child process, identified - by its FD in the child process. - """ - - def writeToChild(childFD, data): - """ - Similar to L{ITransport.write} but also allows the file descriptor in - the child process which will receive the bytes to be specified. - - @type childFD: C{int} - @param childFD: The file descriptor to which to write. - - @type data: C{str} - @param data: The bytes to write. - - @return: C{None} - - @raise KeyError: If C{childFD} is not a file descriptor that was mapped - in the child when L{IReactorProcess.spawnProcess} was used to create - it. - """ - - def loseConnection(): - """ - Close stdin, stderr and stdout. - """ - - def signalProcess(signalID): - """ - Send a signal to the process. - - @param signalID: can be - - one of C{"KILL"}, C{"TERM"}, or C{"INT"}. - These will be implemented in a - cross-platform manner, and so should be used - if possible. - - an integer, where it represents a POSIX - signal ID. - - @raise twisted.internet.error.ProcessExitedAlready: If the process has - already exited. - @raise OSError: If the C{os.kill} call fails with an errno different - from C{ESRCH}. - """ - - -class IServiceCollection(Interface): - """ - An object which provides access to a collection of services. - """ - - def getServiceNamed(serviceName): - """ - Retrieve the named service from this application. - - Raise a C{KeyError} if there is no such service name. - """ - - def addService(service): - """ - Add a service to this collection. - """ - - def removeService(service): - """ - Remove a service from this collection. - """ - - -class IUDPTransport(Interface): - """ - Transport for UDP DatagramProtocols. - """ - - def write(packet, addr=None): - """ - Write packet to given address. - - @param addr: a tuple of (ip, port). For connected transports must - be the address the transport is connected to, or None. - In non-connected mode this is mandatory. - - @raise twisted.internet.error.MessageLengthError: C{packet} was too - long. - """ - - def connect(host, port): - """ - Connect the transport to an address. - - This changes it to connected mode. Datagrams can only be sent to - this address, and will only be received from this address. In addition - the protocol's connectionRefused method might get called if destination - is not receiving datagrams. - - @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost') - @param port: port to connect to. - """ - - def getHost(): - """ - Returns L{IPv4Address}. - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return L{Deferred} that fires - upon completion. - """ - - - -class IUNIXDatagramTransport(Interface): - """ - Transport for UDP PacketProtocols. - """ - - def write(packet, address): - """ - Write packet to given address. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - -class IUNIXDatagramConnectedTransport(Interface): - """ - Transport for UDP ConnectedPacketProtocols. - """ - - def write(packet): - """ - Write packet to address we are connected to. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - def getPeer(): - """ - Returns L{UNIXAddress}. - """ - - -class IMulticastTransport(Interface): - """ - Additional functionality for multicast UDP. - """ - - def getOutgoingInterface(): - """ - Return interface of outgoing multicast packets. - """ - - def setOutgoingInterface(addr): - """ - Set interface for outgoing multicast packets. - - Returns Deferred of success. - """ - - def getLoopbackMode(): - """ - Return if loopback mode is enabled. - """ - - def setLoopbackMode(mode): - """ - Set if loopback mode is enabled. - """ - - def getTTL(): - """ - Get time to live for multicast packets. - """ - - def setTTL(ttl): - """ - Set time to live on multicast packets. - """ - - def joinGroup(addr, interface=""): - """ - Join a multicast group. Returns L{Deferred} of success or failure. - - If an error occurs, the returned L{Deferred} will fail with - L{error.MulticastJoinError}. - """ - - def leaveGroup(addr, interface=""): - """ - Leave multicast group, return L{Deferred} of success. - """ - - -class IStreamClientEndpoint(Interface): - """ - A stream client endpoint is a place that L{ClientFactory} can connect to. - For example, a remote TCP host/port pair would be a TCP client endpoint. - - @since: 10.1 - """ - - def connect(protocolFactory): - """ - Connect the C{protocolFactory} to the location specified by this - L{IStreamClientEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IProtocol} upon successful - connection otherwise a L{ConnectError} - """ - - - -class IStreamServerEndpoint(Interface): - """ - A stream server endpoint is a place that a L{Factory} can listen for - incoming connections. - - @since: 10.1 - """ - - def listen(protocolFactory): - """ - Listen with C{protocolFactory} at the location specified by this - L{IStreamServerEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IListeningPort} or an - L{CannotListenError} - """ - - - -class IStreamServerEndpointStringParser(Interface): - """ - An L{IStreamServerEndpointStringParser} is like an - L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s - instead of clients. It integrates with L{endpoints.serverFromString} in - much the same way. - """ - - prefix = Attribute( - """ - @see: L{IStreamClientEndpointStringParser.prefix} - """ - ) - - - def parseStreamServer(reactor, *args, **kwargs): - """ - Parse a stream server endpoint from a reactor and string-only arguments - and keyword arguments. - - @see: L{IStreamClientEndpointStringParser.parseStreamClient} - - @return: a stream server endpoint - @rtype: L{IStreamServerEndpoint} - """ - - - -class IStreamClientEndpointStringParser(Interface): - """ - An L{IStreamClientEndpointStringParser} is a parser which can convert - a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint} - provider. - - This interface is really only useful in the context of the plugin system - for L{endpoints.clientFromString}. See the document entitled "I{The - Twisted Plugin System}" for more details on how to write a plugin. - - If you place an L{IStreamClientEndpointStringParser} plugin in the - C{twisted.plugins} package, that plugin's C{parseStreamClient} method will - be used to produce endpoints for any description string that begins with - the result of that L{IStreamClientEndpointStringParser}'s prefix attribute. - """ - - prefix = Attribute( - """ - A C{str}, the description prefix to respond to. For example, an - L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its - C{prefix} attribute would be called for endpoint descriptions like - C{"foo:bar:baz"} or C{"foo:"}. - """ - ) - - - def parseStreamClient(*args, **kwargs): - """ - This method is invoked by L{endpoints.clientFromString}, if the type of - endpoint matches the return value from this - L{IStreamClientEndpointStringParser}'s C{prefix} method. - - @param args: The string arguments, minus the endpoint type, in the - endpoint description string, parsed according to the rules - described in L{endpoints.quoteStringArgument}. For example, if the - description were C{"my-type:foo:bar:baz=qux"}, C{args} would be - C{('foo','bar')} - - @param kwargs: The string arguments from the endpoint description - passed as keyword arguments. For example, if the description were - C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be - C{dict(baz='qux')}. - - @return: a client endpoint - @rtype: L{IStreamClientEndpoint} - """ -}}} ''' diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index 32c88ff2d..fd814dc22 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -10,580 +10,11 @@ Interface definitions for L{twisted.web}. body is not known in advance. """ -from zope.interface import Interface, Attribute - -#from twisted.internet.interfaces import IPushProducer - from twisted.web.iweb import ( IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, ) -''' {{{ -class IRequest(Interface): - """ - An HTTP request. - - @since: 9.0 - """ - - method = Attribute("A C{str} giving the HTTP method that was used.") - uri = Attribute( - "A C{str} giving the full encoded URI which was requested (including " - "query arguments).") - path = Attribute( - "A C{str} giving the encoded query path of the request URI.") - args = Attribute( - "A mapping of decoded query argument names as C{str} to " - "corresponding query argument values as C{list}s of C{str}. " - "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} " - "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], " - "'quux': ['spam']}}.") - - received_headers = Attribute( - "Backwards-compatibility access to C{requestHeaders}. Use " - "C{requestHeaders} instead. C{received_headers} behaves mostly " - "like a C{dict} and does not provide access to all header values.") - - requestHeaders = Attribute( - "A L{http_headers.Headers} instance giving all received HTTP request " - "headers.") - - content = Attribute( - "A file-like object giving the request body. This may be a file on " - "disk, a C{StringIO}, or some other type. The implementation is free " - "to decide on a per-request basis.") - - headers = Attribute( - "Backwards-compatibility access to C{responseHeaders}. Use" - "C{responseHeaders} instead. C{headers} behaves mostly like a " - "C{dict} and does not provide access to all header values nor " - "does it allow multiple values for one header to be set.") - - responseHeaders = Attribute( - "A L{http_headers.Headers} instance holding all HTTP response " - "headers to be sent.") - - def getHeader(key): - """ - Get an HTTP request header. - - @type key: C{str} - @param key: The name of the header to get the value of. - - @rtype: C{str} or C{NoneType} - @return: The value of the specified header, or C{None} if that header - was not present in the request. - """ - - - def getCookie(key): - """ - Get a cookie that was sent from the network. - """ - - - def getAllHeaders(): - """ - Return dictionary mapping the names of all received headers to the last - value received for each. - - Since this method does not return all header information, - C{requestHeaders.getAllRawHeaders()} may be preferred. - """ - - - def getRequestHostname(): - """ - Get the hostname that the user passed in to the request. - - This will either use the Host: header (if it is available) or the - host we are listening on if the header is unavailable. - - @returns: the requested hostname - @rtype: C{str} - """ - - - def getHost(): - """ - Get my originally requesting transport's host. - - @return: An L{IAddress}. - """ - - - def getClientIP(): - """ - Return the IP address of the client who submitted this request. - - @returns: the client IP address or C{None} if the request was submitted - over a transport where IP addresses do not make sense. - @rtype: L{str} or C{NoneType} - """ - - - def getClient(): - """ - Return the hostname of the IP address of the client who submitted this - request, if possible. - - This method is B{deprecated}. See L{getClientIP} instead. - - @rtype: C{NoneType} or L{str} - @return: The canonical hostname of the client, as determined by - performing a name lookup on the IP address of the client. - """ - - - def getUser(): - """ - Return the HTTP user sent with this request, if any. - - If no user was supplied, return the empty string. - - @returns: the HTTP user, if any - @rtype: C{str} - """ - - - def getPassword(): - """ - Return the HTTP password sent with this request, if any. - - If no password was supplied, return the empty string. - - @returns: the HTTP password, if any - @rtype: C{str} - """ - - - def isSecure(): - """ - Return True if this request is using a secure transport. - - Normally this method returns True if this request's HTTPChannel - instance is using a transport that implements ISSLTransport. - - This will also return True if setHost() has been called - with ssl=True. - - @returns: True if this request is secure - @rtype: C{bool} - """ - - - def getSession(sessionInterface=None): - """ - Look up the session associated with this request or create a new one if - there is not one. - - @return: The L{Session} instance identified by the session cookie in - the request, or the C{sessionInterface} component of that session - if C{sessionInterface} is specified. - """ - - - def URLPath(): - """ - @return: A L{URLPath} instance which identifies the URL for which this - request is. - """ - - - def prePathURL(): - """ - @return: At any time during resource traversal, a L{str} giving an - absolute URL to the most nested resource which has yet been - reached. - """ - - - def rememberRootURL(): - """ - Remember the currently-processed part of the URL for later - recalling. - """ - - - def getRootURL(): - """ - Get a previously-remembered URL. - """ - - - # Methods for outgoing response - def finish(): - """ - Indicate that the response to this request is complete. - """ - - - def write(data): - """ - Write some data to the body of the response to this request. Response - headers are written the first time this method is called, after which - new response headers may not be added. - """ - - - def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): - """ - Set an outgoing HTTP cookie. - - In general, you should consider using sessions instead of cookies, see - L{twisted.web.server.Request.getSession} and the - L{twisted.web.server.Session} class for details. - """ - - - def setResponseCode(code, message=None): - """ - Set the HTTP response code. - """ - - - def setHeader(k, v): - """ - Set an HTTP response header. Overrides any previously set values for - this header. - - @type name: C{str} - @param name: The name of the header for which to set the value. - - @type value: C{str} - @param value: The value to set for the named header. - """ - - - def redirect(url): - """ - Utility function that does a redirect. - - The request should have finish() called after this. - """ - - - def setLastModified(when): - """ - Set the C{Last-Modified} time for the response to this request. - - If I am called more than once, I ignore attempts to set Last-Modified - earlier, only replacing the Last-Modified time if it is to a later - value. - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} if appropriate for the time given. - - @param when: The last time the resource being returned was modified, in - seconds since the epoch. - @type when: L{int}, L{long} or L{float} - - @return: If I am a C{If-Modified-Since} conditional request and the time - given is not newer than the condition, I return - L{CACHED} to indicate that you should write no body. - Otherwise, I return a false value. - """ - - - def setETag(etag): - """ - Set an C{entity tag} for the outgoing response. - - That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for - comparing two or more entities from the same requested resource." - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} or - L{PRECONDITION_FAILED}, if appropriate for the - tag given. - - @param etag: The entity tag for the resource being returned. - @type etag: C{str} - - @return: If I am a C{If-None-Match} conditional request and the tag - matches one in the request, I return L{CACHED} to - indicate that you should write no body. Otherwise, I return a - false value. - """ - - - def setHost(host, port, ssl=0): - """ - Change the host and port the request thinks it's using. - - This method is useful for working with reverse HTTP proxies (e.g. both - Squid and Apache's mod_proxy can do this), when the address the HTTP - client is using is different than the one we're listening on. - - For example, Apache may be listening on https://www.example.com, and - then forwarding requests to http://localhost:8080, but we don't want - HTML produced by Twisted to say 'http://localhost:8080', they should - say 'https://www.example.com', so we do:: - - request.setHost('www.example.com', 443, ssl=1) - """ - - - -class ICredentialFactory(Interface): - """ - A credential factory defines a way to generate a particular kind of - authentication challenge and a way to interpret the responses to these - challenges. It creates - L{ICredentials} providers from - responses. These objects will be used with L{twisted.cred} to authenticate - an authorize requests. - """ - scheme = Attribute( - "A C{str} giving the name of the authentication scheme with which " - "this factory is associated. For example, C{'basic'} or C{'digest'}.") - - - def getChallenge(request): - """ - Generate a new challenge to be sent to a client. - - @type peer: L{twisted.web.http.Request} - @param peer: The request the response to which this challenge will be - included. - - @rtype: C{dict} - @return: A mapping from C{str} challenge fields to associated C{str} - values. - """ - - - def decode(response, request): - """ - Create a credentials object from the given response. - - @type response: C{str} - @param response: scheme specific response string - - @type request: L{twisted.web.http.Request} - @param request: The request being processed (from which the response - was taken). - - @raise twisted.cred.error.LoginFailed: If the response is invalid. - - @rtype: L{twisted.cred.credentials.ICredentials} provider - @return: The credentials represented by the given response. - """ - - - -class IBodyProducer(IPushProducer): - """ - Objects which provide L{IBodyProducer} write bytes to an object which - provides L{IConsumer} by calling its - C{write} method repeatedly. - - L{IBodyProducer} providers may start producing as soon as they have an - L{IConsumer} provider. That is, they - should not wait for a C{resumeProducing} call to begin writing data. - - L{IConsumer.unregisterProducer} - must not be called. Instead, the - L{Deferred} returned from C{startProducing} - must be fired when all bytes have been written. - - L{IConsumer.write} may - synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or - C{stopProducing}. These methods must be implemented with this in mind. - - @since: 9.0 - """ - - # Despite the restrictions above and the additional requirements of - # stopProducing documented below, this interface still needs to be an - # IPushProducer subclass. Providers of it will be passed to IConsumer - # providers which only know about IPushProducer and IPullProducer, not - # about this interface. This interface needs to remain close enough to one - # of those interfaces for consumers to work with it. - - length = Attribute( - """ - C{length} is a C{int} indicating how many bytes in total this - L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH} - if this is not known in advance. - """) - - def startProducing(consumer): - """ - Start producing to the given - L{IConsumer} provider. - - @return: A L{Deferred} which fires with - C{None} when all bytes have been produced or with a - L{Failure} if there is any problem - before all bytes have been produced. - """ - - - def stopProducing(): - """ - In addition to the standard behavior of - L{IProducer.stopProducing} - (stop producing data), make sure the - L{Deferred} returned by - C{startProducing} is never fired. - """ - - - -class IRenderable(Interface): - """ - An L{IRenderable} is an object that may be rendered by the - L{twisted.web.template} templating system. - """ - - def lookupRenderMethod(name): - """ - Look up and return the render method associated with the given name. - - @type name: C{str} - @param name: The value of a render directive encountered in the - document returned by a call to L{IRenderable.render}. - - @return: A two-argument callable which will be invoked with the request - being responded to and the tag object on which the render directive - was encountered. - """ - - - def render(request): - """ - Get the document for this L{IRenderable}. - - @type request: L{IRequest} provider or C{NoneType} - @param request: The request in response to which this method is being - invoked. - - @return: An object which can be flattened. - """ - - - -class ITemplateLoader(Interface): - """ - A loader for templates; something usable as a value for - L{twisted.web.template.Element}'s C{loader} attribute. - """ - - def load(): - """ - Load a template suitable for rendering. - - @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and - other L{IRenderable} providers. - """ - - - -class IResponse(Interface): - """ - An object representing an HTTP response received from an HTTP server. - - @since: 11.1 - """ - - version = Attribute( - "A three-tuple describing the protocol and protocol version " - "of the response. The first element is of type C{str}, the second " - "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.") - - - code = Attribute("The HTTP status code of this response, as a C{int}.") - - - phrase = Attribute( - "The HTTP reason phrase of this response, as a C{str}.") - - - headers = Attribute("The HTTP response L{Headers} of this response.") - - - length = Attribute( - "The C{int} number of bytes expected to be in the body of this " - "response or L{UNKNOWN_LENGTH} if the server did not indicate how " - "many bytes to expect. For I{HEAD} responses, this will be 0; if " - "the response includes a I{Content-Length} header, it will be " - "available in C{headers}.") - - - def deliverBody(protocol): - """ - Register an L{IProtocol} provider - to receive the response body. - - The protocol will be connected to a transport which provides - L{IPushProducer}. The protocol's C{connectionLost} method will be - called with: - - - ResponseDone, which indicates that all bytes from the response - have been successfully delivered. - - - PotentialDataLoss, which indicates that it cannot be determined - if the entire response body has been delivered. This only occurs - when making requests to HTTP servers which do not set - I{Content-Length} or a I{Transfer-Encoding} in the response. - - - ResponseFailed, which indicates that some bytes from the response - were lost. The C{reasons} attribute of the exception may provide - more specific indications as to why. - """ - - - -class _IRequestEncoder(Interface): - """ - An object encoding data passed to L{IRequest.write}, for example for - compression purpose. - - @since: 12.3 - """ - - def encode(data): - """ - Encode the data given and return the result. - - @param data: The content to encode. - @type data: C{str} - - @return: The encoded data. - @rtype: C{str} - """ - - - def finish(): - """ - Callback called when the request is closing. - - @return: If necessary, the pending data accumulated from previous - C{encode} calls. - @rtype: C{str} - """ - - - -class _IRequestEncoderFactory(Interface): - """ - A factory for returing L{_IRequestEncoder} instances. - - @since: 12.3 - """ - - def encoderForRequest(request): - """ - If applicable, returns a L{_IRequestEncoder} instance which will encode - the request. - """ - - - -UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" -}}} ''' __all__ = [ "ICredentialFactory", "IRequest", "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", From 534772f6ea8389ec4f51d5ea4c4572080a3d3d7b Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 2 Dec 2016 01:15:37 +0000 Subject: [PATCH 24/62] Import xlib.tx code from twisted proper --- scrapy/core/downloader/handlers/http11.py | 5 +++-- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- tests/test_downloadermiddleware_retry.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 404e9160b..54aa359fb 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -13,8 +13,9 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import PotentialDataLoss -from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, TCP4ClientEndpoint +from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ + HTTPConnectionPool +from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 521327bfe..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -3,10 +3,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy import signals from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -from scrapy.xlib.tx import ResponseFailed class HttpCacheMiddleware(object): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 74938067f..c9c512be8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -17,10 +17,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message -from scrapy.xlib.tx import ResponseFailed from scrapy.core.downloader.handlers.http11 import TunnelError logger = logging.getLogger(__name__) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 3de9399cf..eb17974bf 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -3,10 +3,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.xlib.tx import ResponseFailed from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler From 67cf64edbee99287bc8f663be033d1343e3a77ae Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 2 Dec 2016 20:53:06 +0000 Subject: [PATCH 25/62] Deprecate scrapy.xlib.tx --- scrapy/xlib/tx/__init__.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index 1c9bf09e5..0d94307b7 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -1,19 +1,10 @@ -from scrapy import twisted_version -if twisted_version > (13, 0, 0): - from twisted.web import client - from twisted.internet import endpoints -if twisted_version >= (11, 1, 0): - from . import client, endpoints -else: - from scrapy.exceptions import NotSupported - class _Mocked(object): - def __init__(self, *args, **kw): - raise NotSupported('HTTP1.1 not supported') - class _Mock(object): - def __getattr__(self, name): - return _Mocked - client = endpoints = _Mock() +from __future__ import absolute_import +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning + +from twisted.web import client +from twisted.internet import endpoints Agent = client.Agent # since < 11.1 ProxyAgent = client.ProxyAgent # since 11.1 @@ -21,3 +12,8 @@ ResponseDone = client.ResponseDone # since 11.1 ResponseFailed = client.ResponseFailed # since 11.1 HTTPConnectionPool = client.HTTPConnectionPool # since 12.1 TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1 + +warnings.warn("Importing from scrapy.xlib.tx is deprecated and will" + " no longer be supported in future Scrapy versions." + " Update your code to import from twisted proper.", + ScrapyDeprecationWarning, stacklevel=2) From 67bc2e0b18d990f08bfeb913be0c5bd7f299015a Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 2 Dec 2016 21:00:39 +0000 Subject: [PATCH 26/62] Wipe scrapy.xlib.tx --- scrapy/xlib/{tx/__init__.py => tx.py} | 0 scrapy/xlib/tx/LICENSE | 57 -------------------------- scrapy/xlib/tx/README | 2 - scrapy/xlib/tx/_newclient.py | 59 --------------------------- scrapy/xlib/tx/client.py | 58 -------------------------- scrapy/xlib/tx/endpoints.py | 26 ------------ scrapy/xlib/tx/interfaces.py | 28 ------------- scrapy/xlib/tx/iweb.py | 23 ----------- 8 files changed, 253 deletions(-) rename scrapy/xlib/{tx/__init__.py => tx.py} (100%) delete mode 100644 scrapy/xlib/tx/LICENSE delete mode 100644 scrapy/xlib/tx/README delete mode 100644 scrapy/xlib/tx/_newclient.py delete mode 100644 scrapy/xlib/tx/client.py delete mode 100644 scrapy/xlib/tx/endpoints.py delete mode 100644 scrapy/xlib/tx/interfaces.py delete mode 100644 scrapy/xlib/tx/iweb.py diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx.py similarity index 100% rename from scrapy/xlib/tx/__init__.py rename to scrapy/xlib/tx.py diff --git a/scrapy/xlib/tx/LICENSE b/scrapy/xlib/tx/LICENSE deleted file mode 100644 index 8529f6edf..000000000 --- a/scrapy/xlib/tx/LICENSE +++ /dev/null @@ -1,57 +0,0 @@ -Copyright (c) 2001-2013 -Allen Short -Andy Gayton -Andrew Bennetts -Antoine Pitrou -Apple Computer, Inc. -Benjamin Bruheim -Bob Ippolito -Canonical Limited -Christopher Armstrong -David Reid -Donovan Preston -Eric Mangold -Eyal Lotem -Itamar Turner-Trauring -James Knight -Jason A. Mobarak -Jean-Paul Calderone -Jessica McKellar -Jonathan Jacobs -Jonathan Lange -Jonathan D. Simms -Jürgen Hermann -Kevin Horn -Kevin Turner -Mary Gardiner -Matthew Lefkowitz -Massachusetts Institute of Technology -Moshe Zadka -Paul Swartz -Pavel Pergamenshchik -Ralph Meijer -Sean Riley -Software Freedom Conservancy -Travis B. Hartwell -Thijs Triemstra -Thomas Herve -Timothy Allen - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scrapy/xlib/tx/README b/scrapy/xlib/tx/README deleted file mode 100644 index 75ef485ce..000000000 --- a/scrapy/xlib/tx/README +++ /dev/null @@ -1,2 +0,0 @@ -This source files are adapted copies from Twisted trunk to support HTTP1.1 -handler under Twisted >= 11.1 and Twisted <= 13.0.0 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py deleted file mode 100644 index 39cd20f95..000000000 --- a/scrapy/xlib/tx/_newclient.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- test-case-name: twisted.web.test.test_newclient -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -An U{HTTP 1.1} client. - -The way to use the functionality provided by this module is to: - - - Connect a L{HTTP11ClientProtocol} to an HTTP server - - Create a L{Request} with the appropriate data - - Pass the request to L{HTTP11ClientProtocol.request} - - The returned Deferred will fire with a L{Response} object - - Create a L{IProtocol} provider which can handle the response body - - Connect it to the response with L{Response.deliverBody} - - When the protocol's C{connectionLost} method is called, the response is - complete. See L{Response.deliverBody} for details. - -Various other classes in this module support this usage: - - - HTTPParser is the basic HTTP parser. It can handle the parts of HTTP which - are symmetric between requests and responses. - - - HTTPClientParser extends HTTPParser to handle response-specific parts of - HTTP. One instance is created for each request to parse the corresponding - response. -""" - -__metaclass__ = type - -from zope.interface import implements - -from twisted.python import log -from twisted.python.reflect import fullyQualifiedName -from twisted.python.failure import Failure -from twisted.internet.interfaces import IConsumer, IPushProducer -from twisted.internet.error import ConnectionDone -from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred -from twisted.internet.defer import CancelledError -from twisted.internet.protocol import Protocol -from twisted.web.iweb import UNKNOWN_LENGTH, IResponse -from twisted.web.http import NO_CONTENT, NOT_MODIFIED -from twisted.web.http import _DataLoss, PotentialDataLoss -from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder - -from twisted.web._newclient import ( - BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, - RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted, - WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent, - ResponseNeverReceived, HTTPParser, HTTPClientParser, Request, - LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder, - TransportProxyProducer, HTTP11ClientProtocol -) - -# States HTTPParser can be in -STATUS = 'STATUS' -HEADER = 'HEADER' -BODY = 'BODY' -DONE = 'DONE' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py deleted file mode 100644 index c2d50648a..000000000 --- a/scrapy/xlib/tx/client.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -HTTP client. -""" - -from __future__ import division, absolute_import - -import os - -try: - from urlparse import urlunparse - from urllib import splithost, splittype -except ImportError: - from urllib.parse import splithost, splittype - from urllib.parse import urlunparse as _urlunparse - - def urlunparse(parts): - result = _urlunparse(tuple([p.decode("charmap") for p in parts])) - return result.encode("charmap") -import zlib - -from zope.interface import implementer - -from twisted.python import log -from twisted.python.failure import Failure -from twisted.web import http -from twisted.internet import defer, protocol, task, reactor -from twisted.internet.interfaces import IProtocol -from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from twisted.python import failure -from twisted.python.components import proxyForInterface -from twisted.web import error -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse -from twisted.web.http_headers import Headers - -from twisted.web.client import ( - PartialDownloadError, FileBodyProducer, - CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, - Agent, ProxyAgent, HTTPConnectionPool, readBody, -) - - -# The code which follows is based on the new HTTP client implementation. It -# should be significantly better than anything above, though it is not yet -# feature equivalent. - -from twisted.web._newclient import Response -from twisted.web._newclient import ResponseDone, ResponseFailed - - -__all__ = [ - 'PartialDownloadError', - 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', - 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', - 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py deleted file mode 100644 index 197e43ed3..000000000 --- a/scrapy/xlib/tx/endpoints.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- test-case-name: twisted.internet.test.test_endpoints -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that -wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces. - -This also implements an extensible mini-language for describing endpoints, -parsed by the L{clientFromString} and L{serverFromString} functions. - -@since: 10.1 -""" - -from __future__ import division, absolute_import - -from twisted.internet.endpoints import ( - clientFromString, serverFromString, quoteStringArgument, - TCP4ServerEndpoint, TCP6ServerEndpoint, - TCP4ClientEndpoint, TCP6ClientEndpoint, - UNIXServerEndpoint, UNIXClientEndpoint, - SSL4ServerEndpoint, SSL4ClientEndpoint, - AdoptedStreamServerEndpoint, connectProtocol, -) - -__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py deleted file mode 100644 index fdcbf3977..000000000 --- a/scrapy/xlib/tx/interfaces.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Interface documentation. - -Maintainer: Itamar Shtull-Trauring -""" - -from __future__ import division, absolute_import - -from twisted.internet.interfaces import ( - IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, - IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, - IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, - IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet, - IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, - IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, - ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, - IProtocol, IProcessProtocol, IHalfCloseableProtocol, - IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport, - IUNIXTransport, - ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection, - IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, - IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint, - IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, - IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver -) diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py deleted file mode 100644 index fd814dc22..000000000 --- a/scrapy/xlib/tx/iweb.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- test-case-name: twisted.web.test -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Interface definitions for L{twisted.web}. - -@var UNKNOWN_LENGTH: An opaque object which may be used as the value of - L{IBodyProducer.length} to indicate that the length of the entity - body is not known in advance. -""" - -from twisted.web.iweb import ( - IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, - IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, -) - -__all__ = [ - "ICredentialFactory", "IRequest", - "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", - "_IRequestEncoderFactory", - - "UNKNOWN_LENGTH"] From c58ea021b816f20fda1b522f80d8b6a1c22f2454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Sun, 4 Dec 2016 11:56:14 -0300 Subject: [PATCH 27/62] fixes docs --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 18d2f8084..2380a340c 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -87,13 +87,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: list + :type to: str or iterable :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: list + :type cc: str or iterable :param body: the e-mail body :type body: str From a4178f99daf69e930fcc635421cc3a9ba519e36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Mon, 5 Dec 2016 15:24:26 -0300 Subject: [PATCH 28/62] fixes params types in docs. --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 2380a340c..f68448276 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -87,13 +87,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: str or iterable + :type to: str or list of str :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: str or iterable + :type cc: str or list of str :param body: the e-mail body :type body: str From c08d278c0c7549b3377df995cdf724e3fbb8c02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Mon, 5 Dec 2016 16:47:24 -0300 Subject: [PATCH 29/62] removes note from docs. --- docs/topics/email.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index f68448276..aac93a91a 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -35,12 +35,6 @@ And here is how to use it to send an e-mail (without attachments):: mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"]) -.. note:: - As shown in the example above, ``to`` and ``cc`` need to be lists - of email addresses, not single addresses, and even for one recipient, - i.e. ``to="someone@example.com"`` will not work. - - MailSender class reference ========================== From 89d5f5acd3627cef040c421fce516bb87cfd4b5e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Dec 2016 22:42:16 +0100 Subject: [PATCH 30/62] Update changelog for upcoming 1.2.2 release --- docs/news.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c302d2e17..db5856d31 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,41 @@ Release notes ============= +Scrapy 1.2.2 (2016-12-XX) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Fix a cryptic traceback when a pipeline fails on ``open_spider()`` (:issue:`2011`) +- Fix embedded IPython shell variables (fixing :issue:`396` that re-appeared + in 1.2.0, fixed in :issue:`2418`) +- A couple of patches when dealing with robots.txt: + + - handle (non-standard) relative sitemap URLs (:issue:`2390`) + - handle non-ASCII URLs and User-Agents in Python 2 (:issue:`2373`) + +Documentation +~~~~~~~~~~~~~ + +- Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`) +- Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`) +- A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`) + and clarifications (:issue:`2354`, :issue:`2325`) + +Other changes +~~~~~~~~~~~~~ + +- Advertize `conda-forge`_ as Scrapy's official conda channel (:issue:`2387`) +- More helpful error messages when trying to use ``.css()`` or ``.xpath()`` + on non-Text Responses (:issue:`2264`) +- ``startproject`` command now generates a sample ``middlewares.py`` file (:issue:`2335`) +- Add more dependencies' version info in ``scrapy version`` verbose output (:issue:`2404`) +- Remove all ``*.pyc`` files from source distribution (:issue:`2386`) + +.. _conda-forge: https://anaconda.org/conda-forge/scrapy + + Scrapy 1.2.1 (2016-10-21) ------------------------- From aa2e1b030d717951475c3e4d06fcb447ab1e1181 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 14:44:19 +0100 Subject: [PATCH 31/62] Add reference to fixed scheduler settings documentation --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index db5856d31..3c75a0228 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -23,7 +23,7 @@ Documentation - Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`) - Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`) - A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`) - and clarifications (:issue:`2354`, :issue:`2325`) + and clarifications (:issue:`2354`, :issue:`2325`, :issue:`2414`) Other changes ~~~~~~~~~~~~~ From 09e310d0b7db40a244cf3508c722ab251ac452eb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 15:03:38 +0100 Subject: [PATCH 32/62] Set release date for 1.2.2 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 3c75a0228..5e28fb130 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.2.2 (2016-12-XX) +Scrapy 1.2.2 (2016-12-06) ------------------------- Bug fixes From f3d599532943ebf6a4639e9e99781c2f51cd24c5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 15:21:00 +0100 Subject: [PATCH 33/62] =?UTF-8?q?Bump=20version:=201.2.1=20=E2=86=92=201.2?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ffc933d13..ee039790d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.1 +current_version = 1.2.2 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 6085e9465..23aa83906 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.2.1 +1.2.2 From 5efd65255c88ebb156b07f5f09a0a0fdb66f8e7e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 18:49:53 +0100 Subject: [PATCH 34/62] TST: Randomize IMAGES_EXPIRES above 90 days --- tests/test_pipeline_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6c1976b63..342f25ea9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -244,7 +244,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { - "IMAGES_EXPIRES": random.randint(1, 1000), + "IMAGES_EXPIRES": random.randint(100, 1000), "IMAGES_STORE": self.tempdir, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), From 778bed07bf771dd3942ea8cd51b7944065f4e2cd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 17:56:13 +0100 Subject: [PATCH 35/62] Let framework handle only HTTP redirects by default for fetch and shell commands --- scrapy/commands/fetch.py | 11 ++++++++--- scrapy/commands/shell.py | 6 +++--- scrapy/shell.py | 12 ++++++++---- scrapy/utils/datatypes.py | 10 ++++++++++ tests/test_command_fetch.py | 2 +- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a157b19f8..6fe6d73b9 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -27,8 +28,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") - parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ - default=False, help="do not handle status codes like redirects and print response as-is") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -52,7 +53,11 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - if opts.no_status_aware: + # by default, let the framework handle redirects, + # i.e. command handles all codes expect 3xx + if not opts.no_redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400)) + else: request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index bc0203d89..40a58d94a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,8 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ - default=False, help="do not transparently handle status codes like redirects") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -70,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url, handle_statuses=opts.no_status_aware) + shell.start(url=url, redirect=not opts.no_redirect) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 966003f17..6c78722be 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,6 +7,7 @@ from __future__ import print_function import os import signal +from six.moves import range import warnings from twisted.internet import reactor, threads, defer @@ -20,6 +21,7 @@ from scrapy.item import BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.console import start_python_console +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.conf import get_config @@ -40,11 +42,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): + def start(self, url=None, request=None, response=None, spider=None, redirect=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider, handle_statuses=handle_statuses) + self.fetch(url, spider, redirect=redirect) elif request: self.fetch(request, spider) elif response: @@ -98,13 +100,15 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): + def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): if isinstance(request_or_url, Request): request = request_or_url else: url = any_to_uri(request_or_url) request = Request(url, dont_filter=True, **kwargs) - if handle_statuses: + if redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: request.meta['handle_httpstatus_all'] = True response = None try: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d04b43176..e516185bd 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -304,3 +304,13 @@ class LocalCache(OrderedDict): while len(self) >= self.limit: self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) + + +class SequenceExclude(object): + """Object to test if an item is NOT within some sequence.""" + + def __init__(self, seq): + self.seq = seq + + def __contains__(self, item): + return item not in self.seq diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 45d03a129..3fa3ed930 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -21,7 +21,7 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_redirect_disabled(self): - _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')]) + _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')]) err = err.strip() self.assertIn(b'downloader/response_status_count/302', err, err) self.assertNotIn(b'downloader/response_status_count/200', err, err) From 7e54de24550df658690277839cbee99c9afe4bc8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 18:41:24 +0100 Subject: [PATCH 36/62] Add tests for shell command with and without --no-redirect --- tests/test_command_shell.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7bb7439d6..ee6e8ad8e 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -49,6 +49,16 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) assert out.strip().endswith(b'/redirected') + @defer.inlineCallbacks + def test_redirect_follow_302(self): + _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'200') + + @defer.inlineCallbacks + def test_redirect_not_follow_302(self): + _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'302') + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') From 2cd579a7748d0c37eac557216b857c38fbcf80df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 19:07:32 +0100 Subject: [PATCH 37/62] Add test for fetch(url) within shell with and without redirect --- tests/test_command_shell.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index ee6e8ad8e..3e27d6abd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -59,6 +59,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) assert out.strip().endswith(b'302') + @defer.inlineCallbacks + def test_fetch_redirect_follow_302(self): + """Test that calling `fetch(url)` follows HTTP redirects by default.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}')" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Redirecting (302)' in errout + assert b'Crawled (200)' in errout + + @defer.inlineCallbacks + def test_fetch_redirect_not_follow_302(self): + """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Crawled (302)' in errout + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') From 948e3cd00328f9a23410b3a7197975f56efadd16 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 8 Dec 2016 12:50:26 +0100 Subject: [PATCH 38/62] Warn user instead of failing for wrong SPIDER_MODULES setting --- scrapy/spiderloader.py | 13 ++++++++++--- tests/test_spiderloader/__init__.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index fbf68cec4..265182329 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import warnings from zope.interface import implementer @@ -18,15 +19,21 @@ class SpiderLoader(object): self.spider_modules = settings.getlist('SPIDER_MODULES') self._spiders = {} self._load_all_spiders() - + def _load_spiders(self, module): for spcls in iter_spider_classes(module): self._spiders[spcls.name] = spcls def _load_all_spiders(self): for name in self.spider_modules: - for module in walk_modules(name): - self._load_spiders(module) + try: + for module in walk_modules(name): + self._load_spiders(module) + except ImportError as e: + msg = ("Could not load spiders from module '{}'; " + "Check SPIDER_MODULES setting " + "(exception: {})".format(name, str(e))) + warnings.warn(msg, RuntimeWarning) @classmethod def from_settings(cls, settings): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index fbd2c1669..b2ad93b3f 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,6 +1,7 @@ import sys import os import shutil +import warnings from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase): crawler = runner.create_crawler('spider1') self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) self.assertEqual(crawler.spidercls.name, 'spider1') + + def test_bad_spider_modules_warning(self): + + with warnings.catch_warnings(record=True) as w: + module = 'tests.test_spiderloader.test_spiders.doesnotexist' + settings = Settings({'SPIDER_MODULES': [module]}) + spider_loader = SpiderLoader.from_settings(settings) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) From 7d1783603251923bb549d34a64d43952fe03b3bc Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 8 Dec 2016 17:27:25 +0100 Subject: [PATCH 39/62] Update documentation about --no-redirect option --- docs/topics/commands.rst | 28 ++++++++++++++++++ docs/topics/shell.rst | 62 ++++++++++++++++++++++++++-------------- scrapy/shell.py | 9 ++++-- 3 files changed, 75 insertions(+), 24 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 32669104c..3a26b19ae 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page. If used outside a project, no particular per-spider behaviour would be applied and it will just use the default Scrapy downloader settings. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--headers``: print the response's HTTP headers instead of the response's body + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html @@ -368,11 +376,31 @@ given. Also supports UNIX-style local file paths, either relative with ``./`` or ``../`` prefixes or absolute file paths. See :ref:`topics-shell` for more info. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``-c code``: evaluate the code in the shell, print the result and exit + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy shell http://www.example.com/some/page.html [ ... scrapy shell starts ... ] + $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)' + (200, 'http://www.example.com/') + + # shell follows HTTP redirects by default + $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (200, 'http://example.com/') + + # you can disable this with --no-redirect + $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') + + .. command:: parse parse diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..6eb81a71f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -97,8 +97,12 @@ Available Shortcuts * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``fetch(request_or_url)`` - fetch a new response from the given request or - URL and update all related objects accordingly. + * ``fetch(url[, redirect=True])`` - fetch a new response from the given + URL and update all related objects accordingly. You can optionaly ask for + HTTP 3xx redirections to not be followed by passing ``redirect=False`` + + * ``fetch(request)`` - fetch a new response from the given request and + update all related objects accordingly. * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order @@ -157,36 +161,28 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available Scrapy objects: - [s] crawler + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler [s] item {} [s] request - [s] response <200 http://scrapy.org> - [s] settings - [s] spider + [s] response <200 https://scrapy.org/> + [s] settings + [s] spider [s] Useful shortcuts: + [s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed) + [s] fetch(req) Fetch a scrapy.Request and update local objects [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> + After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") - [s] Available Scrapy objects: - [s] crawler - [s] item {} - [s] request - [s] response <200 https://www.reddit.com/> - [s] settings - [s] spider - [s] Useful shortcuts: - [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects - [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() [u'reddit: the front page of the internet'] @@ -194,12 +190,36 @@ After that, we can start playing with the objects:: >>> request = request.replace(method="POST") >>> fetch(request) - [s] Available Scrapy objects: - [s] crawler - ... + >>> response.status + 404 + + >>> from pprint import pprint + + >>> pprint(response.headers) + {'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} >>> + .. _topics-shell-inspect-response: Invoking the shell from spiders to inspect responses diff --git a/scrapy/shell.py b/scrapy/shell.py index 6c78722be..babc267c7 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -148,10 +148,13 @@ class Shell(object): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) b.append("Useful shortcuts:") - b.append(" shelp() Shell help (print this help)") if self.inthread: - b.append(" fetch(req_or_url) Fetch request (or URL) and " - "update local objects") + b.append(" fetch(url[, redirect=True]) " + "Fetch URL and update local objects " + "(by default, redirects are followed)") + b.append(" fetch(req) " + "Fetch a scrapy.Request and update local objects ") + b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") return "\n".join("[s] %s" % l for l in b) From a75ad2bbc63e2fd351c43069a034461c1ab673cf Mon Sep 17 00:00:00 2001 From: Akhil Lb Date: Wed, 4 Nov 2015 01:59:57 +0530 Subject: [PATCH 40/62] LOG_SHORT_NAMES option --- docs/topics/logging.rst | 5 +++++ docs/topics/settings.rst | 10 ++++++++++ scrapy/settings/default_settings.py | 1 + scrapy/utils/log.py | 3 ++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index b7aa6d985..231f5186b 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -150,6 +150,7 @@ These settings can be used to configure the logging: * :setting:`LOG_FORMAT` * :setting:`LOG_DATEFORMAT` * :setting:`LOG_STDOUT` +* :setting:`LOG_SHORT_NAMES` The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be @@ -170,6 +171,10 @@ listed in `logging's logrecord attributes docs `_ respectively. +If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy +component that prints the log. It is unset by default, hence logs contain the +scrapy component responsible for that log output. + Command-line options -------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a17472564..c528987ec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -788,6 +788,16 @@ If ``True``, all standard output (and error) of your process will be redirected to the log. For example if you ``print 'hello'`` it will appear in the Scrapy log. +.. setting:: LOG_SHORT_NAMES + +LOG_SHORT_NAMES +____________ + +Default: ``False`` + +If ``True``, the logs will just contain the root path. If it is set to ``False`` +then it displays the component responsible for the log output + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 61f4bd567..24714a7a8 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -191,6 +191,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51f303216..f33ce7017 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -118,7 +118,8 @@ def _get_handler(settings): ) handler.setFormatter(formatter) handler.setLevel(settings.get('LOG_LEVEL')) - handler.addFilter(TopLevelFormatter(['scrapy'])) + if settings.getbool('LOG_SHORT_NAMES'): + handler.addFilter(TopLevelFormatter(['scrapy'])) return handler From 05cec0f2f348345e3d32242968450fb523d25dff Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jan 2016 15:21:05 +0500 Subject: [PATCH 41/62] fixed ReST syntax --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c528987ec..503f4afb1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -791,7 +791,7 @@ log. .. setting:: LOG_SHORT_NAMES LOG_SHORT_NAMES -____________ +--------------- Default: ``False`` From 6eab59cbac6a35e7d92a7391541ad2f16493338b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Dec 2016 02:14:12 +0500 Subject: [PATCH 42/62] TST cleanup runspider tests --- tests/test_commands.py | 47 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index b507c46bc..bcd7215a0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -38,11 +38,11 @@ class ProjectTest(unittest.TestCase): return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs) - def proc(self, *new_args, **kwargs): + def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args p = subprocess.Popen(args, cwd=self.cwd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs) + **popen_kwargs) waited = 0 interval = 0.2 @@ -182,6 +182,17 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug("It Works!") + return [] +""" + @contextmanager def _create_file(self, content, name): tmpdir = self.mktemp() @@ -194,32 +205,23 @@ class RunSpiderCommandTest(CommandTest): finally: rmtree(tmpdir) - def runspider(self, code, name='myspider.py'): + def runspider(self, code, name='myspider.py', args=()): with self._create_file(code, name) as fname: - return self.proc('runspider', fname) + return self.proc('runspider', fname, *args) + + def get_log(self, code, name='myspider.py', args=()): + p = self.runspider(code, name=name, args=args) + return to_native_str(p.stderr.read()) def test_runspider(self): - spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug("It Works!") - return [] -""" - p = self.runspider(spider) - log = to_native_str(p.stderr.read()) - + log = self.get_log(self.debug_log_spider) self.assertIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) def test_runspider_no_spider_found(self): - p = self.runspider("from scrapy.spiders import Spider\n") - log = to_native_str(p.stderr.read()) + log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): @@ -228,12 +230,11 @@ class MySpider(scrapy.Spider): self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): - p = self.runspider('', 'myspider.txt') - log = to_native_str(p.stderr.read()) + log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) def test_start_requests_errors(self): - p = self.runspider(""" + log = self.get_log(""" import scrapy class BadSpider(scrapy.Spider): @@ -241,11 +242,11 @@ class BadSpider(scrapy.Spider): def start_requests(self): raise Exception("oops!") """, name="badspider.py") - log = to_native_str(p.stderr.read()) print(log) self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + class BenchCommandTest(CommandTest): def test_run(self): From e46572d6f2de1533b1df2ab206971351c22bbbbe Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Dec 2016 02:19:33 +0500 Subject: [PATCH 43/62] TST end-to-end test for LOG_LEVEL option there were no end-to-end tests for this option --- tests/test_commands.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index bcd7215a0..1dd88f342 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -220,6 +220,12 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_runspider_log_level(self): + log = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_LEVEL=INFO')) + self.assertNotIn("DEBUG: It Works!", log) + self.assertIn("INFO: Spider opened", log) + def test_runspider_no_spider_found(self): log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) From 05b4555f3932afb04ab3b15893a5858fca9dec04 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Dec 2016 02:19:51 +0500 Subject: [PATCH 44/62] TST tests for LOG_SHORT_NAMES --- tests/test_commands.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1dd88f342..922098668 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -226,6 +226,21 @@ class MySpider(scrapy.Spider): self.assertNotIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) + def test_runspider_log_short_names(self): + log1 = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_SHORT_NAMES=1')) + print(log1) + self.assertIn("[myspider] DEBUG: It Works!", log1) + self.assertIn("[scrapy]", log1) + self.assertNotIn("[scrapy.core.engine]", log1) + + log2 = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_SHORT_NAMES=0')) + print(log2) + self.assertIn("[myspider] DEBUG: It Works!", log2) + self.assertNotIn("[scrapy]", log2) + self.assertIn("[scrapy.core.engine]", log2) + def test_runspider_no_spider_found(self): log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) From f457379a54cdbad60bcf24947e053340abc7e16b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 9 Dec 2016 16:56:26 +0100 Subject: [PATCH 45/62] Add stacktrace in warning message --- scrapy/spiderloader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 265182329..d4f0f663f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import traceback import warnings from zope.interface import implementer @@ -30,9 +31,9 @@ class SpiderLoader(object): for module in walk_modules(name): self._load_spiders(module) except ImportError as e: - msg = ("Could not load spiders from module '{}'; " - "Check SPIDER_MODULES setting " - "(exception: {})".format(name, str(e))) + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "Check SPIDER_MODULES setting".format( + modname=name, tb=traceback.format_exc())) warnings.warn(msg, RuntimeWarning) @classmethod From f7e4081414d318ddb5297fb98a120e57044dc622 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Dec 2016 22:37:53 +0100 Subject: [PATCH 46/62] Add tests for SequenceExclude container --- tests/test_utils_datatypes.py | 63 ++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b31d2179c..80f797227 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,7 @@ import copy import unittest -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaselessDict, SequenceExclude __doctests__ = ['scrapy.utils.datatypes'] @@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase): assert isinstance(h2, CaselessDict) +class SequenceExcludeTest(unittest.TestCase): + + def test_list(self): + seq = [1, 2, 3] + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn(4, d) + self.assertNotIn(2, d) + + def test_range(self): + seq = range(10, 20) + d = SequenceExclude(seq) + self.assertIn(5, d) + self.assertIn(20, d) + self.assertNotIn(15, d) + + def test_six_range(self): + import six.moves + seq = six.moves.range(10**3, 10**6) + d = SequenceExclude(seq) + self.assertIn(10**2, d) + self.assertIn(10**7, d) + self.assertNotIn(10**4, d) + + def test_range_step(self): + seq = range(10, 20, 3) + d = SequenceExclude(seq) + are_not_in = [v for v in range(10, 20, 3) if v in d] + self.assertEquals([], are_not_in) + + are_not_in = [v for v in range(10, 20) if v in d] + self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + + def test_string_seq(self): + seq = "cde" + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_stringset_seq(self): + seq = set("cde") + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_set(self): + """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" + seq = set([-3, "test", 1.1]) + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn("foo", d) + self.assertIn(3.14, d) + self.assertIn(set("bar"), d) + + # supplied sequence is a set, so checking for list (non)inclusion fails + self.assertRaises(TypeError, (0, 1, 2) in d) + self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c']) + + for v in [-3, "test", 1.1]: + self.assertNotIn(v, d) + if __name__ == "__main__": unittest.main() From 70a69d2199c3c08a7e16f33ea5d35fd4066eb14b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Dec 2016 22:40:48 +0100 Subject: [PATCH 47/62] Use built-in range() --- scrapy/commands/fetch.py | 2 +- scrapy/shell.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 6fe6d73b9..7d4840529 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -56,7 +56,7 @@ class Command(ScrapyCommand): # by default, let the framework handle redirects, # i.e. command handles all codes expect 3xx if not opts.no_redirect: - request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400)) + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) else: request.meta['handle_httpstatus_all'] = True diff --git a/scrapy/shell.py b/scrapy/shell.py index babc267c7..6f94635a1 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,7 +7,6 @@ from __future__ import print_function import os import signal -from six.moves import range import warnings from twisted.internet import reactor, threads, defer From 0fc73a9d558158f1686f9cc9c289fe364b5df536 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 16 Dec 2016 21:47:58 +0500 Subject: [PATCH 48/62] DOC update examples with long longger names --- docs/intro/tutorial.rst | 24 +++---- docs/topics/benchmarking.rst | 92 +++++++++++++++++---------- docs/topics/downloader-middleware.rst | 8 +-- docs/topics/settings.rst | 2 +- docs/topics/shell.rst | 10 +-- 5 files changed, 81 insertions(+), 55 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 0941eb1e5..8e14d1b7c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -130,15 +130,15 @@ will send some requests for the ``quotes.toscrape.com`` domain. You will get an similar to this:: ... (omitted for brevity) - 2016-09-20 14:48:00 [scrapy] INFO: Spider opened - 2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) - 2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) (referer: None) - 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) (referer: None) - 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html - 2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) (referer: None) - 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html - 2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished) + 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened + 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html + 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html + 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) ... Now, check the files in the current directory. You should notice that two new @@ -212,7 +212,7 @@ using the shell :ref:`Scrapy shell `. Run:: You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler @@ -429,9 +429,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} - 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 632190067..99469ebf1 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -18,40 +18,66 @@ To run it use:: You should see an output like this:: - 2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot) - 2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened - 2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats: - {'downloader/request_bytes': 231508, - 'downloader/request_count': 682, - 'downloader/request_method_count/GET': 682, - 'downloader/response_bytes': 1172802, - 'downloader/response_count': 682, - 'downloader/response_status_count/200': 682, - 'finish_reason': 'closespider_timeout', - 'finish_time': datetime.datetime(2013, 5, 16, 16, 8, 57, 985539), - 'log_count/INFO': 14, - 'request_depth_max': 34, - 'response_received_count': 682, - 'scheduler/dequeued': 682, - 'scheduler/dequeued/memory': 682, - 'scheduler/enqueued': 12767, - 'scheduler/enqueued/memory': 12767, - 'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)} - 2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout) + 2016-12-16 21:18:48 [scrapy.utils.log] INFO: Scrapy 1.2.2 started (bot: quotesbot) + 2016-12-16 21:18:48 [scrapy.utils.log] INFO: Overridden settings: {'CLOSESPIDER_TIMEOUT': 10, 'ROBOTSTXT_OBEY': True, 'SPIDER_MODULES': ['quotesbot.spiders'], 'LOGSTATS_INTERVAL': 1, 'BOT_NAME': 'quotesbot', 'LOG_LEVEL': 'INFO', 'NEWSPIDER_MODULE': 'quotesbot.spiders'} + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled extensions: + ['scrapy.extensions.closespider.CloseSpider', + 'scrapy.extensions.logstats.LogStats', + 'scrapy.extensions.telnet.TelnetConsole', + 'scrapy.extensions.corestats.CoreStats'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: + ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', + 'scrapy.downloadermiddlewares.retry.RetryMiddleware', + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', + 'scrapy.downloadermiddlewares.stats.DownloaderStats'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: + ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.spidermiddlewares.referer.RefererMiddleware', + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', + 'scrapy.spidermiddlewares.depth.DepthMiddleware'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled item pipelines: + [] + 2016-12-16 21:18:49 [scrapy.core.engine] INFO: Spider opened + 2016-12-16 21:18:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:50 [scrapy.extensions.logstats] INFO: Crawled 70 pages (at 4200 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:51 [scrapy.extensions.logstats] INFO: Crawled 134 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:52 [scrapy.extensions.logstats] INFO: Crawled 198 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:53 [scrapy.extensions.logstats] INFO: Crawled 254 pages (at 3360 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:54 [scrapy.extensions.logstats] INFO: Crawled 302 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:55 [scrapy.extensions.logstats] INFO: Crawled 358 pages (at 3360 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:56 [scrapy.extensions.logstats] INFO: Crawled 406 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:57 [scrapy.extensions.logstats] INFO: Crawled 438 pages (at 1920 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:58 [scrapy.extensions.logstats] INFO: Crawled 470 pages (at 1920 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:59 [scrapy.core.engine] INFO: Closing spider (closespider_timeout) + 2016-12-16 21:18:59 [scrapy.extensions.logstats] INFO: Crawled 518 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:19:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats: + {'downloader/request_bytes': 229995, + 'downloader/request_count': 534, + 'downloader/request_method_count/GET': 534, + 'downloader/response_bytes': 1565504, + 'downloader/response_count': 534, + 'downloader/response_status_count/200': 534, + 'finish_reason': 'closespider_timeout', + 'finish_time': datetime.datetime(2016, 12, 16, 16, 19, 0, 647725), + 'log_count/INFO': 17, + 'request_depth_max': 19, + 'response_received_count': 534, + 'scheduler/dequeued': 533, + 'scheduler/dequeued/memory': 533, + 'scheduler/enqueued': 10661, + 'scheduler/enqueued/memory': 10661, + 'start_time': datetime.datetime(2016, 12, 16, 16, 18, 49, 799869)} + 2016-12-16 21:19:00 [scrapy.core.engine] INFO: Spider closed (closespider_timeout) -That tells you that Scrapy is able to crawl about 3900 pages per minute in the +That tells you that Scrapy is able to crawl about 3000 pages per minute in the hardware where you run it. Note that this is a very simple spider intended to follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 29d9b0298..3b9a5335a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -238,14 +238,14 @@ header) and all cookies received in responses (ie. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: - 2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ Set-Cookie: ip_isocode=US Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) (referer: None) + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [...] diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 503f4afb1..0515a9e0d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1037,7 +1037,7 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap Example entry in logs:: - 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + 1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request: - reason: cannot serialize (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..da91108b2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -173,7 +173,7 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() - u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' + 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") [s] Available Scrapy objects: @@ -189,7 +189,7 @@ After that, we can start playing with the objects:: [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() - [u'reddit: the front page of the internet'] + ['reddit: the front page of the internet'] >>> request = request.replace(method="POST") @@ -234,8 +234,8 @@ Here's an example of how you would call it from your spider:: When you run the spider, you will get something similar to this:: - 2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] crawler ... @@ -258,7 +258,7 @@ Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the crawling:: >>> ^D - 2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is From da19f0b7b73ca4fd78d828e710e111c60bc658e3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 16 Dec 2016 22:14:54 +0500 Subject: [PATCH 49/62] DOC how to override log level for a specific Scrapy component --- docs/topics/logging.rst | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 231f5186b..ac3b614fc 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -10,7 +10,7 @@ Logging about the new logging system. Scrapy uses `Python's builtin logging system -`_ for event logging. We'll +`_ for event logging. We'll provide some simple examples to get you started, but for more advanced use-cases it's strongly suggested to read thoroughly its documentation. @@ -193,6 +193,43 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +Advanced customization +---------------------- + +Because Scrapy uses stdlib logging module, you can customize logging using +all features of stdlib logging. + +For example, let's say you're scraping a website which returns many +HTTP 404 and 500 responses, and you want to hide all messages like this:: + + 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring + response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + is not handled or not allowed + +The first thing to note is a logger name - it is in brackets: +``[scrapy.spidermiddlewares.httperror]``. If you get just ``[scrapy]`` then +:setting:`LOG_SHORT_NAMES` is likely set to True; set it to False and re-run +the crawl. + +Next, we can see that the message has INFO level. To hide it +we should set logging level for ``scrapy.spidermiddlewares.httperror`` +higher than INFO; next level after INFO is WARNING. It could be done +e.g. in the spider's ``__init__`` method:: + + import logging + import scrapy + + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('scrapy.spidermiddlewares.httperror') + logger.setLevel(logging.WARNING) + super().__init__(*args, **kwargs) + +If you run this spider again then INFO messages from +``scrapy.spidermiddlewares.httperror`` logger will be gone. + scrapy.utils.log module ======================= From 2b3abdb7006a2875744a89d45f46e858c981d751 Mon Sep 17 00:00:00 2001 From: zhouyc Date: Mon, 19 Dec 2016 11:27:57 +0800 Subject: [PATCH 50/62] update parsel version which will cause attribute error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4eb8d2318..3ae7915b4 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.3', + 'parsel>=0.9.5', 'PyDispatcher>=2.0.5', 'service_identity', ], From 140a57d7b00f0b044598715ab640bd7e96e32318 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 19 Dec 2016 17:51:30 +0100 Subject: [PATCH 51/62] Amend note on --no-redirect option for shell tool --- docs/topics/commands.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3a26b19ae..6636c30cb 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -382,7 +382,9 @@ Supported options: * ``-c code``: evaluate the code in the shell, print the result and exit -* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them); + this only affects the URL you may pass as argument on the command line; + once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default. Usage example:: @@ -397,6 +399,7 @@ Usage example:: (200, 'http://example.com/') # you can disable this with --no-redirect + # (only for the URL passed as command line argument) $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') From 6dec4a3ccb455511decc4c1414b9042c57a205ea Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 20 Dec 2016 20:02:31 -0400 Subject: [PATCH 52/62] ENH Pass arguments to logger rather than formatted message. This not only use the standard form but helps error aggregation libraries (i.e.: Sentry) to avoid duplicating the message. --- scrapy/core/downloader/handlers/http11.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 54aa359fb..ecd7f90d3 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,14 +319,13 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ("Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize})." - ).format(url=request.url, size=expected_size, maxsize=maxsize) + error_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} - logger.error(error_message) + logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_message) + raise defer.CancelledError(error_msg % error_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From e9b3cf01f4393c50b4a10f2032db0fecae5aa5f4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 9 Dec 2016 17:23:59 +0100 Subject: [PATCH 53/62] Update changelog for upcoming 1.3.0 release --- docs/news.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5e28fb130..5f759f0fc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,26 @@ Release notes ============= +Scrapy 1.3.0 (2016-12-XX) +------------------------- + +New Features +~~~~~~~~~~~~ + +- ``MailSender`` now accepts single strings as values for ``to`` and ``cc`` + arguments (:issue:`2272`) + +Dependencies & Cleanups +~~~~~~~~~~~~~~~~~~~~~~~ + +- Scrapy now requires Twisted >= 13.1 which is the case for many Linux + distributions already. +- As a consequence, we got rid of ``scrapy.xlib.tx.*`` modules, which + copied some of Twisted code for users stuck with an "old" Twisted version +- ``ChunkedTransferMiddleware`` is deprecated and removed from the default + downloader middlewares. + + Scrapy 1.2.2 (2016-12-06) ------------------------- From 9d5afd8c35410e2874a97a17b1a2b5f4deef2b38 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 16 Dec 2016 18:46:49 +0100 Subject: [PATCH 54/62] Add note on HttpErrorMiddleware new logging level --- docs/news.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5f759f0fc..a7cc30b0b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -22,6 +22,12 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. +Logging +~~~~~~~ + +- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; + this is technically **backwards incompatible** so please check your log parsers. + Scrapy 1.2.2 (2016-12-06) ------------------------- From 9098001888a57ac4db030fc6563d541174f47d47 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 19 Dec 2016 17:09:47 +0100 Subject: [PATCH 55/62] Add note on LOG_SHORT_NAMES --- docs/news.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index a7cc30b0b..02227aa8e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -27,6 +27,11 @@ Logging - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; this is technically **backwards incompatible** so please check your log parsers. +- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, + instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); + this is **backwards incompatible** if you have log parsers expecting the short + logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` + set to ``True``. Scrapy 1.2.2 (2016-12-06) From 49a84c2d414386b27a3f9e4439c5a967ce6cb9bc Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 14:32:21 +0100 Subject: [PATCH 56/62] Add note on HTTP redirects with shell and fetch --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 02227aa8e..02976fedf 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,6 +11,9 @@ New Features - ``MailSender`` now accepts single strings as values for ``to`` and ``cc`` arguments (:issue:`2272`) +- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside + scrapy shell now follow HTTP redirections by default (:issue:`2290`); + See :command:`fetch` and :command:`shell` for details. Dependencies & Cleanups ~~~~~~~~~~~~~~~~~~~~~~~ From 4eeec3e42cf4a48dd3481e43ca064576a4a825d7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 15:59:18 +0100 Subject: [PATCH 57/62] Add preamble on why 1.3.0 comes so soon after 1.2.2 --- docs/news.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 02976fedf..09c47dc30 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,14 @@ Release notes Scrapy 1.3.0 (2016-12-XX) ------------------------- +This release comes rather soon after 1.2.2 for one main reason: +it was found out that releases since 0.18 up to 1.2.2 (included) use +some backported code from Twisted, even if newer Twisted modules are available. +Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly. + +As it is a major change, we wanted to get the bug fix out quickly +while not breaking any projects using the 1.2 series. + New Features ~~~~~~~~~~~~ From d60a6899dfe0fa5ac08298fa466b71e4f221b93e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:01:36 +0100 Subject: [PATCH 58/62] Merge logging changes into "New features" section --- docs/news.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 09c47dc30..f42edba30 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -22,6 +22,13 @@ New Features - ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside scrapy shell now follow HTTP redirections by default (:issue:`2290`); See :command:`fetch` and :command:`shell` for details. +- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; + this is technically **backwards incompatible** so please check your log parsers. +- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, + instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); + this is **backwards incompatible** if you have log parsers expecting the short + logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` + set to ``True``. Dependencies & Cleanups ~~~~~~~~~~~~~~~~~~~~~~~ @@ -33,17 +40,6 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. -Logging -~~~~~~~ - -- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; - this is technically **backwards incompatible** so please check your log parsers. -- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, - instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); - this is **backwards incompatible** if you have log parsers expecting the short - logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` - set to ``True``. - Scrapy 1.2.2 (2016-12-06) ------------------------- From b9e7ca044cfb82fc9592cef7e2fdcfa422fe8d44 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:11:19 +0100 Subject: [PATCH 59/62] Reword things a tiny bit --- docs/news.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index f42edba30..6690ca558 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,8 +8,10 @@ Scrapy 1.3.0 (2016-12-XX) This release comes rather soon after 1.2.2 for one main reason: it was found out that releases since 0.18 up to 1.2.2 (included) use -some backported code from Twisted, even if newer Twisted modules are available. +some backported code from Twisted (``scrapy.xlib.tx.*``), +even if newer Twisted modules are available. Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly. +(See also cleanups below.) As it is a major change, we wanted to get the bug fix out quickly while not breaking any projects using the 1.2 series. From f8793e2460977a0c218900687cd7c6d592a5c7d5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:27:01 +0100 Subject: [PATCH 60/62] Set release date for 1.3.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 6690ca558..cce46599b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.3.0 (2016-12-XX) +Scrapy 1.3.0 (2016-12-21) ------------------------- This release comes rather soon after 1.2.2 for one main reason: From ac74d5a467908a8c3db05f80e9d533be2c9deb64 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:28:44 +0100 Subject: [PATCH 61/62] =?UTF-8?q?Bump=20version:=201.2.2=20=E2=86=92=201.3?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ee039790d..57ff603fa 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.2 +current_version = 1.3.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 23aa83906..f0bb29e76 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.2.2 +1.3.0 From 07f9985a941d2fce4d7115a35dc983bc21ded4be Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 17:03:11 +0100 Subject: [PATCH 62/62] TST: Randomize FILES_EXPIRES above 90 days --- tests/test_pipeline_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 157c21a89..e3ec04b8d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -208,7 +208,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { - "FILES_EXPIRES": random.randint(1, 1000), + "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), "FILES_STORE": self.tempdir