From 2a540206a74af8d38a01aaa5a37adc1008cad6ca Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Tue, 19 Aug 2014 13:57:00 -0300 Subject: [PATCH 0001/3444] fix xmliter namespace on selected node --- scrapy/utils/iterators.py | 22 +++++++++++++++------- tests/test_utils_iterators.py | 6 +++++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 150b077ae..11b873f2e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -20,19 +20,27 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) + DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S) HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S) + NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S) text = _body_or_str(obj) - header_start = re.search(HEADER_START_RE, text) - header_start = header_start.group(1).strip() if header_start else '' - header_end = re_rsearch(HEADER_END_RE, text) - header_end = text[header_end[1]:].strip() if header_end else '' + document_header = re.search(DOCUMENT_HEADER_RE, text) + document_header = document_header.group().strip() if document_header else '' + header_end_idx = re_rsearch(HEADER_END_RE, text) + header_end = text[header_end_idx[1]:].strip() if header_end_idx else '' + namespaces = {} + if header_end: + for tagname in reversed(re.findall(END_TAG_RE, header_end)): + tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + if tag: + namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): - nodetext = header_start + match.group() + header_end - yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] + nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + yield Selector(text=nodetext, type='xml') def csviter(obj, delimiter=None, headers=None, encoding=None): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fe53f831f..8b5941605 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -61,7 +61,6 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'item') - node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) @@ -74,6 +73,11 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').extract(), []) self.assertEqual(node.xpath('price/text()').extract(), []) + my_iter = self.xmliter(response, 'g:image_link') + node = next(my_iter) + node.register_namespace('g', 'http://base.google.com/ns/1.0') + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_exception(self): body = u"""onetwo""" From f4dd8bcdc29cb5411d0fe2ed8f53e75883402ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Tue, 16 Jun 2015 17:31:37 +0300 Subject: [PATCH 0002/3444] Disable dupefilter in shell --- scrapy/commands/shell.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 92ebbe605..77ae1358b 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -16,7 +16,11 @@ from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): requires_project = False - default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0} + default_settings = { + 'KEEP_ALIVE': True, + 'LOGSTATS_INTERVAL': 0, + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', + } def syntax(self): return "[url|file]" From 1b4fd3a8dffbbfc0207d707cd11ca84d11e76064 Mon Sep 17 00:00:00 2001 From: nyov Date: Sun, 12 Jul 2015 18:27:41 +0000 Subject: [PATCH 0003/3444] Support anonymous connections in S3DownloadHandler Also consider any unknown keyword args for S3DownloadHandler as arguments to pass on to S3Connection (e.g. proxy settings). --- scrapy/core/downloader/handlers/s3.py | 11 +++++++++-- tests/test_downloader_handlers.py | 21 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 311815b70..38cfd1e10 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -35,7 +35,7 @@ def get_s3_connection(): class S3DownloadHandler(object): def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ - httpdownloadhandler=HTTPDownloadHandler): + httpdownloadhandler=HTTPDownloadHandler, **kw): _S3Connection = get_s3_connection() if _S3Connection is None: @@ -46,8 +46,15 @@ class S3DownloadHandler(object): if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + # If no credentials could be found anywhere, + # consider this an anonymous connection request by default; + # unless 'anon' was set explicitly (True/False). + anon = kw.get('anon', None) + if anon is None and not aws_access_key_id and not aws_secret_access_key: + kw['anon'] = True + try: - self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key) + self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw) except Exception as ex: raise NotConfigured(str(ex)) self._download_http = httpdownloadhandler(settings).download_request diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e4d957d8e..8280b21aa 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -403,6 +403,23 @@ class HttpDownloadHandlerMock(object): def download_request(self, request, spider): return request +class S3AnonTestCase(unittest.TestCase): + skip = 'boto' not in optional_features and 'missing boto library' + + def setUp(self): + self.s3reqh = S3DownloadHandler(Settings(), + httpdownloadhandler=HttpDownloadHandlerMock, + #anon=True, # is implicit + ) + self.download_request = self.s3reqh.download_request + self.spider = Spider('foo') + + def test_anon_request(self): + req = Request('s3://aws-publicdatasets/') + httpreq = self.download_request(req, self.spider) + self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) + self.assertEqual(self.s3reqh.conn.anon, True) + class S3TestCase(unittest.TestCase): download_handler_cls = S3DownloadHandler try: @@ -420,8 +437,8 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' def setUp(self): - s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \ - self.AWS_SECRET_ACCESS_KEY, \ + s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, + self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request self.spider = Spider('foo') From ecbfe4bd6661acd165407e7f1e5abf1f0a2fa31c Mon Sep 17 00:00:00 2001 From: nyov Date: Sun, 12 Jul 2015 16:53:54 +0000 Subject: [PATCH 0004/3444] drop deprecated "optional_features" set --- scrapy/__init__.py | 9 +-------- scrapy/core/downloader/handlers/http.py | 4 ++-- scrapy/utils/log.py | 3 --- tests/test_downloader_handlers.py | 7 +++---- tests/test_downloadermiddleware_retry.py | 4 ++-- tests/test_toplevel.py | 4 ---- 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c0477f509..03ec6c667 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,7 +2,7 @@ Scrapy - a web crawling and web scraping framework written for Python """ -__all__ = ['__version__', 'version_info', 'optional_features', 'twisted_version', +__all__ = ['__version__', 'version_info', 'twisted_version', 'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field'] # Scrapy version @@ -27,15 +27,8 @@ del warnings from . import _monkeypatches del _monkeypatches -# WARNING: optional_features set is deprecated and will be removed soon. Do not use. -optional_features = set() -# TODO: backwards compatibility, remove for Scrapy 0.20 -optional_features.add('ssl') - from twisted import version as _txv twisted_version = (_txv.major, _txv.minor, _txv.micro) -if twisted_version >= (11, 1, 0): - optional_features.add('http11') # Declare top-level shortcuts from scrapy.spiders import Spider diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 1efebb939..81da2615a 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,7 +1,7 @@ -from scrapy import optional_features +from scrapy import twisted_version from .http10 import HTTP10DownloadHandler -if 'http11' in optional_features: +if twisted_version >= (11, 1, 0): from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler else: HTTPDownloadHandler = HTTP10DownloadHandler diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index d40202953..cc2f0b164 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -126,9 +126,6 @@ def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Optional features available: %(features)s", - {'features': ", ".join(scrapy.optional_features)}) - d = dict(overridden_settings(settings)) logger.info("Overridden settings: %(settings)r", {'settings': d}) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e4d957d8e..d2a349b40 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -27,7 +27,6 @@ from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.settings import Settings -from scrapy import optional_features from scrapy.utils.test import get_crawler from scrapy.exceptions import NotConfigured @@ -220,7 +219,7 @@ class Http10TestCase(HttpTestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler - if 'http11' not in optional_features: + if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' def test_download_without_maxsize_limit(self): @@ -267,7 +266,7 @@ class Http11TestCase(HttpTestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - if 'http11' not in optional_features: + if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' def setUp(self): @@ -392,7 +391,7 @@ class Http10ProxyTestCase(HttpProxyTestCase): class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler - if 'http11' not in optional_features: + if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c0381e144..20561e771 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -4,7 +4,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError -from scrapy import optional_features +from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.xlib.tx import ResponseFailed from scrapy.spiders import Spider @@ -75,7 +75,7 @@ class RetryTest(unittest.TestCase): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost] - if 'http11' in optional_features: + if twisted_version >= (11, 1, 0): # http11 available exceptions.append(ResponseFailed) for exc in exceptions: diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index e9f220092..91bbe43bc 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -11,10 +11,6 @@ class ToplevelTestCase(TestCase): def test_version_info(self): self.assertIs(type(scrapy.version_info), tuple) - def test_optional_features(self): - self.assertIs(type(scrapy.optional_features), set) - self.assertIn('ssl', scrapy.optional_features) - def test_request_shortcut(self): from scrapy.http import Request, FormRequest self.assertIs(scrapy.Request, Request) From 97a52665a0b96c8e2b93303dd5b10276ffac1ed2 Mon Sep 17 00:00:00 2001 From: Demelziraptor Date: Fri, 18 Sep 2015 17:16:43 +0900 Subject: [PATCH 0005/3444] interpreting json-amazonui-streaming as TextResponse --- scrapy/responsetypes.py | 1 + tests/test_responsetypes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 4880cc7b9..319656648 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -25,6 +25,7 @@ class ResponseTypes(object): 'application/xml': 'scrapy.http.XmlResponse', 'application/json': 'scrapy.http.TextResponse', 'application/x-json': 'scrapy.http.TextResponse', + 'application/json-amazonui-streaming': 'scrapy.http.TextResponse', 'application/javascript': 'scrapy.http.TextResponse', 'application/x-javascript': 'scrapy.http.TextResponse', 'text/xml': 'scrapy.http.XmlResponse', diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 2374d518f..b34147b74 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -36,6 +36,7 @@ class ResponseTypesTest(unittest.TestCase): ('application/xml; charset=UTF-8', XmlResponse), ('application/octet-stream', Response), ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse), + ('application/json-amazonui-streaming;charset=UTF-8', TextResponse), ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) From e379f58cad0289d725a5241606542d0e20ecd73d Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 29 Oct 2015 14:52:31 +0800 Subject: [PATCH 0006/3444] fixed: Issue #1564 (Incorrectly picked URL in `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `` tag. ) --- scrapy/linkextractors/regex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index b6f8d5d30..0fc7b079f 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,7 +1,7 @@ import re from six.moves.urllib.parse import urljoin -from w3lib.html import remove_tags, replace_entities, replace_escape_chars +from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link from .sgml import SgmlLinkExtractor @@ -31,7 +31,7 @@ class RegexLinkExtractor(SgmlLinkExtractor): return clean_url if base_url is None: - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url + base_url = get_base_url(response_text, response_url, response_encoding) links_text = linkre.findall(response_text) return [Link(clean_url(url).encode(response_encoding), From 98a2e77a75f0b203ce9841801e342a11f15ad419 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Mon, 2 Nov 2015 15:30:49 +0300 Subject: [PATCH 0007/3444] issue GH #1550 - fixed error: shell command wasn't accepting files URIs --- scrapy/commands/shell.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 92ebbe605..f10da4370 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -5,11 +5,11 @@ See documentation in docs/topics/shell.rst """ from threading import Thread +from w3lib.url import any_to_uri from scrapy.commands import ScrapyCommand from scrapy.shell import Shell from scrapy.http import Request -from scrapy.utils.url import add_http_if_no_scheme from scrapy.utils.spider import spidercls_for_request, DefaultSpider @@ -43,7 +43,8 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None if url: - url = add_http_if_no_scheme(url) + url = any_to_uri(url) + spider_loader = self.crawler_process.spider_loader spidercls = DefaultSpider From a41c64bfb97a7bba8ec138345d48b94d8734d27e Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Mon, 2 Nov 2015 16:06:21 +0300 Subject: [PATCH 0008/3444] issue GH #1550 - fixed bugs in scrapy.utils.url.add_http_if_no_scheme(): when given URI where scheme is present, but not 'http' the function gave bad result --- scrapy/utils/url.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 398407a64..3eac5fb3d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import posixpath +from urlparse import urlsplit, urlunsplit from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag, urlparse, parse_qsl, urlencode, unquote) @@ -114,10 +115,16 @@ def escape_ajax(url): def add_http_if_no_scheme(url): """Add http as the default scheme if it is missing from the url.""" - if url.startswith('//'): - url = 'http:' + url - return url - parser = parse_url(url) - if not parser.scheme or not parser.netloc: - url = 'http://' + url - return url + parts = urlsplit(url) + scheme = parts.scheme or "http" + if parts.netloc: + netloc = parts.netloc + path = parts.path + else: + path_parts = url.split("/", 1) + netloc = path_parts[0] + path = path_parts[1] if len(path_parts) > 1 else "/" + + return urlunsplit(( + scheme, netloc, path, parts.query, parts.fragment + )) From bc9db65358bcae3fc228d4b8099d319cba479ff2 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Mon, 2 Nov 2015 16:08:19 +0300 Subject: [PATCH 0009/3444] issue GH #1550 - scrapy shell argument fixes: "example.com" requests "http://example.com"; "example" requests "file://example"; "./example.com" requests "file://example.com" --- scrapy/commands/shell.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f10da4370..cb441bc9d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -5,11 +5,13 @@ See documentation in docs/topics/shell.rst """ from threading import Thread +import urlparse from w3lib.url import any_to_uri from scrapy.commands import ScrapyCommand from scrapy.shell import Shell from scrapy.http import Request +from scrapy.utils.url import add_http_if_no_scheme from scrapy.utils.spider import spidercls_for_request, DefaultSpider @@ -43,7 +45,16 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None if url: - url = any_to_uri(url) + parts = urlparse.urlsplit(url) + if not parts.scheme: + if "." not in parts.path.split("/", 1)[0]: + url = any_to_uri(url) + + for pattern in ["/", "./", "../"]: + if url.startswith(pattern): + url = any_to_uri(url) + break + url = add_http_if_no_scheme(url) spider_loader = self.crawler_process.spider_loader From 94486bb294a6e765efe14affddb1df355a6c298b Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Mon, 2 Nov 2015 23:00:42 +0800 Subject: [PATCH 0010/3444] added: Test case for the fix. --- tests/test_linkextractors_deprecated.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index e3664f8d8..89dcb75c2 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -190,3 +190,20 @@ class RegexLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + + def test_html_base_href(self): + html = """ + + + + + + + + + """ + response = HtmlResponse("http://a.com/", body=html) + lx = RegexLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://b.com/test.html', text=u'', nofollow=False), + ]) From dd45b31fe42635ff43df62c667f1ace5f9169737 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Tue, 3 Nov 2015 14:32:30 +0300 Subject: [PATCH 0011/3444] issue GH #1550 - rewritten add_http_if_no_scheme() --- scrapy/utils/url.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 3eac5fb3d..0e36003ad 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,7 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import posixpath -from urlparse import urlsplit, urlunsplit +import re from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag, urlparse, parse_qsl, urlencode, unquote) @@ -115,16 +115,10 @@ def escape_ajax(url): def add_http_if_no_scheme(url): """Add http as the default scheme if it is missing from the url.""" - parts = urlsplit(url) - scheme = parts.scheme or "http" - if parts.netloc: - netloc = parts.netloc - path = parts.path - else: - path_parts = url.split("/", 1) - netloc = path_parts[0] - path = path_parts[1] if len(path_parts) > 1 else "/" + match = re.match(r"^\w+://", url, flags=re.I) + parts = urlparse(url) + if not match: + scheme = "http:" if parts.netloc else "http://" + url = scheme + url - return urlunsplit(( - scheme, netloc, path, parts.query, parts.fragment - )) + return url From 97b51ea33b6b96241ef87861619e6820ee9e765c Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Tue, 3 Nov 2015 14:57:37 +0300 Subject: [PATCH 0012/3444] issue GH #1550 - six library is used instead of urlparse for python3 compatibility --- scrapy/commands/shell.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index cb441bc9d..6f58a2300 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,8 +4,9 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ +import re +from six.moves.urllib.parse import urlparse, urlunparse from threading import Thread -import urlparse from w3lib.url import any_to_uri from scrapy.commands import ScrapyCommand @@ -45,7 +46,7 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None if url: - parts = urlparse.urlsplit(url) + parts = urlparse(url) if not parts.scheme: if "." not in parts.path.split("/", 1)[0]: url = any_to_uri(url) From 881bf19e6804c3b82543f4f23d6ed60e1a4bb651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 17:08:13 +0200 Subject: [PATCH 0013/3444] FormRequest: test case-insensitive type attribute --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 60fd855dd..593698dd2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -305,6 +305,19 @@ class FormRequestTest(RequestTest): request = FormRequest.from_response(response, url='/relative') self.assertEqual(request.url, 'http://example.com/relative') + def test_from_response_case_insensitive(self): + response = _buildresponse( + """
+ + + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs['clickable1'], ['clicked1']) + self.assertFalse('i1' in fs, fs) # xpath in _get_inputs() + self.assertFalse('clickable2' in fs, fs) # xpath in _get_clickable() + def test_from_response_submit_first_clickable(self): response = _buildresponse( """
From 650acad2b73c1abafe8ff95808b4d77b65ef67cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 17:57:46 +0200 Subject: [PATCH 0014/3444] FormRequest: fix case-insensitive type attributes --- scrapy/http/request/form.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4a9bd732e..ea4bfd564 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -107,8 +107,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type!="submit" and @type!="image" and @type!="reset"' - 'and ((@type!="checkbox" and @type!="radio") or @checked)]') + '|descendant::input[@type[' + ' translate(., "SUBMIT", "submit") != "submit"' + ' and translate(., "IMAGE", "image") !="image"' + ' and translate(., "RESET", "reset") != "reset"' + ' and (../@checked or (' + ' translate(., "CHECKBOX", "checkbox") != "checkbox"' + ' and translate(., "RADIO", "radio") != "radio"))]]') values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -151,9 +156,11 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [el for el in form.xpath('descendant::input[@type="submit"]' - '|descendant::button[@type="submit"]' - '|descendant::button[not(@type)]')] + clickables = [ + el for el in form.xpath( + 'descendant::*[(self::input or self::button)' + ' and translate(@type, "SUBMIT", "submit") = "submit"]' + '|descendant::button[not(@type)]')] if not clickables: return From 7a438a51b783f099894212abdefc95ad591fce3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:03:54 +0200 Subject: [PATCH 0015/3444] FormRequest: test default type (is text) --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 593698dd2..7964a6591 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -675,10 +675,11 @@ class FormRequestTest(RequestTest): + ''') req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']}) + self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']}) def test_from_response_input_hidden(self): res = _buildresponse( From 2d25eab0df4c5c8bb1894cbc45cc15059d726c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:05:07 +0200 Subject: [PATCH 0016/3444] FormRequest: 's default type must be text --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea4bfd564..8ff394602 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -107,7 +107,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type[' + '|descendant::input[not(@type) or @type[' ' translate(., "SUBMIT", "submit") != "submit"' ' and translate(., "IMAGE", "image") !="image"' ' and translate(., "RESET", "reset") != "reset"' From 395ef805eb74c1c575d32ab0c52d406f55c7d276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:54:02 +0200 Subject: [PATCH 0017/3444] FormRequest: test unicode xpath expr & exception --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7964a6591..f82b2de8d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,6 @@ import cgi import unittest +import re import six from six.moves import xmlrpc_client as xmlrpclib @@ -747,6 +748,18 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formxpath="//form/input[@name='abc']") + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + fs = _qs(r) + self.assertEqual(fs, {}) + + xpath = u"//form[@name='\u03b1']" + encoded = xpath if six.PY3 else xpath.encode('unicode_escape') + self.assertRaisesRegexp(ValueError, re.escape(encoded), + self.request_class.from_response, + response, formxpath=xpath) + def test_from_response_button_submit(self): response = _buildresponse( """
From 4f98be60be700e0c0c359f3cbaa1535ae9ece7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:56:05 +0200 Subject: [PATCH 0018/3444] FormRequest: fix unicode xpath exception --- scrapy/http/request/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8ff394602..948ad05c9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -85,7 +85,8 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape') + raise ValueError('No element found with %s' % encoded) # If we get here, it means that either formname was None # or invalid From ebfdb9bb03fd876bf519535dd57ec0816e0c3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Sat, 14 Nov 2015 23:24:07 +0200 Subject: [PATCH 0019/3444] readable xpath with exslt --- scrapy/http/request/form.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 948ad05c9..f623a5aa3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -109,12 +109,11 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' - ' translate(., "SUBMIT", "submit") != "submit"' - ' and translate(., "IMAGE", "image") !="image"' - ' and translate(., "RESET", "reset") != "reset"' - ' and (../@checked or (' - ' translate(., "CHECKBOX", "checkbox") != "checkbox"' - ' and translate(., "RADIO", "radio") != "radio"))]]') + ' not(re:test(., "^(?:submit|image|reset)$", "i"))' + ' and (../@checked or' + ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', + namespaces={ + "re": "http://exslt.org/regular-expressions"}) values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -160,8 +159,10 @@ def _get_clickable(clickdata, form): clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' - ' and translate(@type, "SUBMIT", "submit") = "submit"]' - '|descendant::button[not(@type)]')] + ' and re:test(@type, "^submit$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) + ] if not clickables: return From 0025d5a9439a1f737bccb8ac1dfc6ff85daabb85 Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 16 Nov 2015 07:30:17 +0800 Subject: [PATCH 0020/3444] Fixed minor grammar issues. --- docs/faq.rst | 2 +- docs/topics/broad-crawls.rst | 4 ++-- docs/topics/extensions.rst | 4 ++-- docs/topics/item-pipeline.rst | 2 +- docs/topics/items.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/selectors.rst | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2e61f44ee..3d2bd8d4d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -144,7 +144,7 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those message are thrown by the Offsite Spider Middleware, which is a spider +Those messages are thrown by the Offsite Spider Middleware, which is a spider middleware (enabled by default) whose purpose is to filter out requests to domains outside the ones covered by the spider. diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index aaf46bc92..79f0b3b53 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -34,7 +34,7 @@ These are some common properties often found in broad crawls: As said above, Scrapy default settings are optimized for focused crawls, not broad crawls. However, due to its asynchronous architecture, Scrapy is very -well suited for performing fast broad crawls. This page summarize some things +well suited for performing fast broad crawls. This page summarizes some things you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. @@ -46,7 +46,7 @@ Concurrency is the number of requests that are processed in parallel. There is a global limit and a per-domain limit. The default global concurrency limit in Scrapy is not suitable for crawling -many different domains in parallel, so you will want to increase it. How much +many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU you crawler will have available. A good starting point is ``100``, but the best way to find out is by doing some trials and identifying at what concurrency your Scrapy process gets CPU diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 11c0aadb6..f9e709514 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ Extensions use the :ref:`Scrapy settings ` to manage their settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, an +avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like `GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. @@ -143,7 +143,7 @@ Here is the code of such extension:: self.items_scraped += 1 if self.items_scraped % self.item_count == 0: logger.info("scraped %d items", self.items_scraped) - + .. _topics-extensions-ref: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index f74400b4d..28969be61 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -95,7 +95,7 @@ contain a price:: Write items to a JSON file -------------------------- -The following pipeline stores all scraped items (from all spiders) into a a +The following pipeline stores all scraped items (from all spiders) into a single ``items.jl`` file, containing one item per line serialized in JSON format:: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 21ec0ed8c..4a8f47e93 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -61,7 +61,7 @@ the example above. You can specify any kind of metadata for each field. There is no restriction on the values accepted by :class:`Field` objects. For this same reason, there is no reference list of all available metadata keys. Each key -defined in :class:`Field` objects could be used by a different components, and +defined in :class:`Field` objects could be used by a different component, and only those components know about it. You can also define and use any other :class:`Field` key in your project too, for your own needs. The main goal of :class:`Field` objects is to provide a way to define all field metadata in one diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 735137ea2..92590c180 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -97,7 +97,7 @@ subclasses): A real example -------------- -Let's see a concrete example of an hypothetical case of memory leaks. +Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 9ae34f423..60fe2267c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -228,7 +228,7 @@ with varying degrees of sophistication. Getting around those measures can be difficult and tricky, and may sometimes require special infrastructure. Please consider contacting `commercial support`_ if in doubt. -Here are some tips to keep in mind when dealing with these kind of sites: +Here are some tips to keep in mind when dealing with these kinds of sites: * rotate your user agent from a pool of well-known ones from browsers (google around to get a list of them) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 273cae0f8..8dc82dfe5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -579,7 +579,7 @@ Built-in Selectors reference is used together with ``text``. If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follow: + inferred from the response type as follows: * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type @@ -757,7 +757,7 @@ nodes can be accessed directly by their names:: Date: Thu, 26 Nov 2015 17:19:19 +0100 Subject: [PATCH 0024/3444] nextcall repetitive calls (heartbeats). --- scrapy/core/engine.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index eb2779b12..ef4403106 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,7 +7,7 @@ For more information see docs/topics/architecture.rst import logging from time import time -from twisted.internet import defer +from twisted.internet import defer, task from twisted.python.failure import Failure from scrapy import signals @@ -30,6 +30,7 @@ class Slot(object): self.close_if_idle = close_if_idle self.nextcall = nextcall self.scheduler = scheduler + self.heartbeat = task.LoopingCall(nextcall.schedule) def add_request(self, request): self.inprogress.add(request) @@ -47,6 +48,7 @@ class Slot(object): if self.closing and not self.inprogress: if self.nextcall: self.nextcall.cancel() + self.heartbeat.stop() self.closing.callback(None) @@ -113,7 +115,6 @@ class ExecutionEngine(object): return if self.paused: - slot.nextcall.schedule(5) return while not self._needs_backout(spider): @@ -254,6 +255,7 @@ class ExecutionEngine(object): self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) slot.nextcall.schedule() + slot.heartbeat.start(5) def _spider_idle(self, spider): """Called when a spider gets idle. This function is called when there @@ -267,7 +269,6 @@ class ExecutionEngine(object): spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \ for _, x in res): - self.slot.nextcall.schedule(5) return if self.spider_is_idle(spider): From c4d29ecaef534cdab31fb73ec83545010f7a5ad8 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 2 Dec 2015 17:05:44 +0100 Subject: [PATCH 0025/3444] Ignoring xlib/tx folder, depending on Twisted version. --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index f9ca3ab93..b0ac1badd 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ import glob import six import pytest +from twisted import version as twisted_version def _py_files(folder): @@ -26,6 +27,9 @@ collect_ignore = [ ] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): + collect_ignore += _py_files("scrapy/xlib/tx") + if six.PY3: for line in open('tests/py3-ignores.txt'): From 016875fd513c66b7ee1bcc80a1f2ae0d6ffda2d2 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 15:30:06 +0300 Subject: [PATCH 0026/3444] added more verbosity for log and for exception when download is cancelled because of a size limit --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..78f3f74fa 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,11 +239,16 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - logger.error("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s).", - {'size': expected_size, 'maxsize': maxsize}) + error_message = ( + "Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize}).".format( + url=request.url, size=expected_size, maxsize=maxsize + ) + ) + logger.error(error_message) txresponse._transport._producer.loseConnection() - raise defer.CancelledError() + raise defer.CancelledError(error_message) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From 37289815ac98404458c43159abfd46dd54ed4b7b Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 16:15:50 +0300 Subject: [PATCH 0027/3444] code formatting fix --- scrapy/core/downloader/handlers/http11.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 78f3f74fa..9d075e877 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,13 +239,11 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ( - "Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize}).".format( - url=request.url, size=expected_size, maxsize=maxsize - ) - ) + error_message = ("Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize})." + ).format(url=request.url, size=expected_size, maxsize=maxsize) + logger.error(error_message) txresponse._transport._producer.loseConnection() raise defer.CancelledError(error_message) From f8ae99d18fb0365c340152e880245bb4340d7a1f Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Sat, 5 Dec 2015 09:48:17 -0400 Subject: [PATCH 0028/3444] DOC Removed pywin32 from install instructions as it's already declared as dependency. --- docs/intro/install.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index abb94c006..122de47f6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -44,13 +44,10 @@ Anaconda If you already have installed `Anaconda`_ or `Miniconda`_, the company `Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X. -To install Scrapy in Linux or OS X, use: +To install Scrapy using ``conda``, run:: conda install -c scrapinghub scrapy -To install Scrapy in Windows, use: - - conda install -c scrapinghub scrapy pywin32 Windows ------- From 4be4ef038ea8e206bdd3682dab3cb7b5184994eb Mon Sep 17 00:00:00 2001 From: orangain Date: Sat, 12 Dec 2015 16:21:00 +0900 Subject: [PATCH 0029/3444] DOC: Add captions to toctrees which appear in sidebar --- docs/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 0d21f5d40..4cb3eb741 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ First steps =========== .. toctree:: + :caption: First steps :hidden: intro/overview @@ -53,6 +54,7 @@ Basic concepts ============== .. toctree:: + :caption: Basic concepts :hidden: topics/commands @@ -110,6 +112,7 @@ Built-in services ================= .. toctree:: + :caption: Built-in services :hidden: topics/logging @@ -138,6 +141,7 @@ Solving specific problems ========================= .. toctree:: + :caption: Solving specific problems :hidden: faq @@ -203,6 +207,7 @@ Extending Scrapy ================ .. toctree:: + :caption: Extending Scrapy :hidden: topics/architecture @@ -240,6 +245,7 @@ All the rest ============ .. toctree:: + :caption: All the rest :hidden: news From 719b1353a7a49d13f1f54e37a85b8b692915c0c7 Mon Sep 17 00:00:00 2001 From: seales Date: Sun, 13 Dec 2015 19:39:48 -0800 Subject: [PATCH 0030/3444] Spelling fixes --- sep/sep-003.rst | 2 +- sep/sep-014.rst | 2 +- sep/sep-018.rst | 2 +- sep/sep-020.rst | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sep/sep-003.rst b/sep/sep-003.rst index 282574968..184839525 100644 --- a/sep/sep-003.rst +++ b/sep/sep-003.rst @@ -146,7 +146,7 @@ Default values p['numbers'] # returns [] -Accesing and changing nested item values +Accessing and changing nested item values ---------------------------------------- :: diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 98a31b1aa..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -54,7 +54,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. -This is an enhancemente to ``LinkExtractors`` which returns urls (links), +This is an enhancement to ``LinkExtractors`` which returns urls (links), Request Extractors return Request objects. Request Processors diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e30821917..aca7ac342 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -477,7 +477,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def should_follow(self, request, spider): info = self.spiders[spider] - # hostanme can be None for wrong urls (like javascript links) + # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or '' return bool(info.regex.search(host)) diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 7b2c043b7..49d068479 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -23,9 +23,9 @@ Rationale ========= There are certain markup patterns that lend themselves quite nicely to -automated parsing, for example the ```` tag outlilnes such a pattern +automated parsing, for example the ``
`` tag outlines such a pattern for populating a database table with the embedded ```` elements denoting -the rows and the furthur embedded `` --- docs/topics/firefox.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst index beda3b8db..0cf45861a 100644 --- a/docs/topics/firefox.rst +++ b/docs/topics/firefox.rst @@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one after applying some browser clean up and executing Javascript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to -extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things when working with Firefox and XPath: From 7727d87f646d03a9f4d0d30d3dcc1161dd1cb1b5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 7 Nov 2016 16:44:57 +0100 Subject: [PATCH 0546/3444] Test Slot's heartbeat state before stopping it Also add a test on state of looping task in LogStats extension Fixes #2011 and #2362 --- scrapy/core/engine.py | 3 ++- scrapy/extensions/logstats.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3c4bc662c..2b5770138 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -48,7 +48,8 @@ class Slot(object): if self.closing and not self.inprogress: if self.nextcall: self.nextcall.cancel() - self.heartbeat.stop() + if self.heartbeat.running: + self.heartbeat.stop() self.closing.callback(None) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 647e50f8d..b685e7b19 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -15,6 +15,7 @@ class LogStats(object): self.stats = stats self.interval = interval self.multiplier = 60.0 / self.interval + self.task = None @classmethod def from_crawler(cls, crawler): @@ -47,5 +48,5 @@ class LogStats(object): logger.info(msg, log_args, extra={'spider': spider}) def spider_closed(self, spider, reason): - if self.task.running: + if self.task and self.task.running: self.task.stop() From 61efacdd1f6fb55cf1f694e56502a3d8a1d5a325 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Nov 2016 11:35:42 +0100 Subject: [PATCH 0547/3444] Add testcase for catching exception from open_spider() from pipeline --- tests/pipelines.py | 11 +++++++++++ tests/test_crawl.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/pipelines.py diff --git a/tests/pipelines.py b/tests/pipelines.py new file mode 100644 index 000000000..d81cfa93d --- /dev/null +++ b/tests/pipelines.py @@ -0,0 +1,11 @@ +""" +Some pipelines used for testing and benchmarking +""" + +class ZeroDivisionErrorPipeline(object): + + def open_spider(self, spider): + a = 1/0 + + def process_item(self, item, spider): + return item diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 90fd921c8..a0f3c9997 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -250,6 +250,18 @@ with multiples lines yield self.assertFailure(crawler.crawl(), TestError) self.assertFalse(crawler.crawling) + @defer.inlineCallbacks + def test_open_spider_error_on_faulty_pipeline(self): + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) + yield self.assertFailure( + self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + ZeroDivisionError) + @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) From 27456996a904ba25f780d2e3824059e836b6cb90 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Nov 2016 11:46:16 +0100 Subject: [PATCH 0548/3444] Add assertion on crawler not running --- tests/test_crawl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index a0f3c9997..1b4a4b3b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -261,6 +261,7 @@ with multiples lines yield self.assertFailure( self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), ZeroDivisionError) + self.assertFalse(crawler.crawling) @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): From af2280e695f9430fdb0ba452594f3f9c4cb9592d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Nov 2016 13:30:51 +0100 Subject: [PATCH 0549/3444] Update docstring --- tests/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines.py b/tests/pipelines.py index d81cfa93d..ddfbc7a99 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -1,5 +1,5 @@ """ -Some pipelines used for testing and benchmarking +Some pipelines used for testing """ class ZeroDivisionErrorPipeline(object): From 3cd56da0cc68a1c1d0ac4ca38dabc5b8aab3e461 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 8 Nov 2016 20:52:32 -0300 Subject: [PATCH 0550/3444] Ignore explicitly compiled python files. This avoids to include compiled files in the templates directory. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 04b3e1fb9..94de4f3bf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,3 +12,4 @@ prune docs/build recursive-include extras * recursive-include bin * recursive-include tests * +global-exclude __pycache__ *.py[cod] From 76459e1969245f214dd39868c11090cdf6e7cf04 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 8 Nov 2016 21:30:27 -0300 Subject: [PATCH 0551/3444] Update conda channel to conda-forge and show conda version badge. --- README.rst | 4 ++++ docs/intro/install.rst | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 74ae1f30d..b72ebf53d 100644 --- a/README.rst +++ b/README.rst @@ -22,6 +22,10 @@ Scrapy :target: http://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report +.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg + :target: https://anaconda.org/conda-forge/scrapy + :alt: Conda Version + Overview ======== diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 7c806a049..767749ec5 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -245,12 +245,12 @@ Using Anaconda is an alternative to using a virtualenv and installing with ``pip For Windows users, or if you have issues installing through ``pip``, this is the recommended way to install Scrapy. -If you already have `Anaconda`_ or `Miniconda`_ installed, -`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X. +If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_ +community have up-to-date packages for Linux, Windows and OS X. To install Scrapy using ``conda``, run:: - conda install -c scrapinghub scrapy + conda install -c conda-forge scrapy .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ @@ -268,3 +268,4 @@ To install Scrapy using ``conda``, run:: .. _Scrapinghub: http://scrapinghub.com .. _Anaconda: http://docs.continuum.io/anaconda/index .. _Miniconda: http://conda.pydata.org/docs/install/quick.html +.. _conda-forge: https://conda-forge.github.io/ From 28155dfcccc0cd7883ffde714570e8c13b0920a6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 9 Nov 2016 12:20:06 +0100 Subject: [PATCH 0552/3444] Parse robots.txt content as native str Fixes #2373 --- scrapy/downloadermiddlewares/robotstxt.py | 5 ++++- tests/test_downloadermiddleware_robotstxt.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 698f394ad..e26c22a09 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.python import to_native_str logger = logging.getLogger(__name__) @@ -94,7 +95,9 @@ class RobotsTxtMiddleware(object): # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. pass - rp.parse(body.splitlines()) + # stdlib's robotparser expects native 'str' ; + # with unicode input, non-ASCII encoded bytes decoding fails in Python2 + rp.parse(to_native_str(body).splitlines()) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f2e94e171..95208c41f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import re from twisted.internet import reactor, error @@ -30,11 +31,15 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = re.sub(b'^\s+(?m)', b'', b''' + ROBOTS = re.sub(b'^\s+(?m)', b'', u''' User-Agent: * Disallow: /admin/ Disallow: /static/ - ''') + + # taken from https://en.wikipedia.org/robots.txt + Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: + Disallow: /wiki/Käyttäjä: + '''.encode('utf-8')) response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): deferred = Deferred() @@ -48,7 +53,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): return DeferredList([ self.assertNotIgnored(Request('http://site.local/allowed'), middleware), self.assertIgnored(Request('http://site.local/admin/main'), middleware), - self.assertIgnored(Request('http://site.local/static/'), middleware) + self.assertIgnored(Request('http://site.local/static/'), middleware), + self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), + self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): From e8205f67339bdd3cdd6450e104b8969c9b98692d Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 29 Jul 2015 15:34:27 +0000 Subject: [PATCH 0553/3444] LinkExtractor PY3 'unicode' type fix --- scrapy/linkextractors/htmlparser.py | 3 ++- scrapy/linkextractors/sgml.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index dcc261b31..9867e1179 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -3,6 +3,7 @@ HTMLParser-based link extractor """ import warnings +import six from six.moves.html_parser import HTMLParser from six.moves.urllib.parse import urljoin @@ -39,7 +40,7 @@ class HtmlParserLinkExtractor(HTMLParser): ret = [] base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - if isinstance(link.url, unicode): + if isinstance(link.url, six.text_type): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 9938e071f..c68dae4c8 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -1,6 +1,7 @@ """ SGMLParser-based Link extractors """ +import six from six.moves.urllib.parse import urljoin import warnings from sgmllib import SGMLParser @@ -40,7 +41,7 @@ class BaseSgmlLinkExtractor(SGMLParser): if base_url is None: base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - if isinstance(link.url, unicode): + if isinstance(link.url, six.text_type): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) From 7025d6656a127d18ac1ef4f29a912181308c6971 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Tue, 18 Oct 2016 11:51:13 -0200 Subject: [PATCH 0554/3444] document download_latency meta key --- docs/topics/request-response.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75b98d3b3..7e700c7cb 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -299,6 +299,7 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`download_timeout` * :reqmeta:`download_maxsize` +* :reqmeta:`download_latency` * :reqmeta:`proxy` .. reqmeta:: bindaddress @@ -316,6 +317,15 @@ download_timeout The amount of time (in secs) that the downloader will wait before timing out. See also: :setting:`DOWNLOAD_TIMEOUT`. +.. reqmeta:: download_latency + +download_latency +---------------- + +The amount of time spent to fetch the response, since the request has been +started, i.e. HTTP message sent over the network. This meta key only becomes +available when the response has been downloaded. While most other meta keys are +used to control Scrapy behavior, this one is supposed to be read-only. .. _topics-request-response-ref-request-subclasses: From 6cd35c77da6601f218b5042f4b9a9919e642e15c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 15 Nov 2016 17:38:32 +0100 Subject: [PATCH 0555/3444] Pass user-agent as native str when checking URLs against robots.txt --- scrapy/downloadermiddlewares/robotstxt.py | 3 ++- tests/test_downloadermiddleware_robotstxt.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index e26c22a09..c3dfa7819 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -41,7 +41,8 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch(self._useragent, request.url): + if rp is not None and not rp.can_fetch( + to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) raise IgnoreRequest() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 95208c41f..60306eacb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -39,6 +39,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): # taken from https://en.wikipedia.org/robots.txt Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: + + User-Agent: UnicödeBöt + Disallow: /some/randome/page.html '''.encode('utf-8')) response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): From 11fe3751cf96ebbf3af3c1446a736cd577883495 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 16 Nov 2016 11:55:09 +0100 Subject: [PATCH 0556/3444] DOC Remove "Ubuntu" section from sidebar/ToC --- docs/index.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b4272e47f..289fb2b1b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,7 +155,6 @@ Solving specific problems topics/firebug topics/leaks topics/media-pipeline - topics/ubuntu topics/deploy topics/autothrottle topics/benchmarking @@ -188,9 +187,6 @@ Solving specific problems :doc:`topics/media-pipeline` Download files and/or images associated with your scraped items. -:doc:`topics/ubuntu` - Install latest Scrapy packages easily on Ubuntu - :doc:`topics/deploy` Deploying your Scrapy spiders and run them in a remote server. From 8db85453939b4431be8142a7b6733acbc6d71416 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 16 Nov 2016 13:56:58 +0100 Subject: [PATCH 0557/3444] Add "orphan" metadata for Ubuntu packages page As described in http://www.sphinx-doc.org/en/latest/markup/misc.html#metadata --- docs/topics/ubuntu.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index f1a870d62..679bb56ff 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -1,3 +1,5 @@ +:orphan: Ubuntu packages are obsolete + .. _topics-ubuntu: =============== From d62776a8589a7ae8fd98403020ce85d01e6b8977 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 16 Nov 2016 12:19:32 -0300 Subject: [PATCH 0558/3444] mention scrapoxy in best practices doc --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7dae68470..25ae4b5ba 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -238,7 +238,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Google cache`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_ + services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a + super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is `Crawlera`_ @@ -253,3 +254,4 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: http://scrapinghub.com/crawlera +.. _scrapoxy: http://scrapoxy.io/ From 01142e2ae5e3b82d5f8701858931ba0354808720 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 22 Nov 2016 14:48:33 +0100 Subject: [PATCH 0559/3444] Print more dependencies versions in "scrapy version" verbose output --- scrapy/commands/version.py | 13 +++++++++++++ tests/test_command_version.py | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 4bf085c9e..a9954edb0 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -26,12 +26,25 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: + import cssselect + import parsel import lxml.etree + import w3lib + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + print("Scrapy : %s" % scrapy.__version__) print("lxml : %s" % lxml_version) print("libxml2 : %s" % libxml2_version) + print("cssselect : %s" % cssselect.__version__) + print("parsel : %s" % parsel.__version__) + print("w3lib : %s" % w3lib_version) print("Twisted : %s" % twisted.version.short()) print("Python : %s" % sys.version.replace("\n", "- ")) print("pyOpenSSL : %s" % self._get_openssl_version()) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 37e1f2543..2789d207c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -25,5 +25,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute(['-v']) headers = [l.partition(":")[0].strip() for l in out.strip().decode(encoding).splitlines()] - self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'Twisted', - 'Python', 'pyOpenSSL', 'Platform']) + self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', + 'cssselect', 'parsel', 'w3lib', + 'Twisted', 'Python', 'pyOpenSSL', + 'Platform']) From 35b655d2f84d652440c393166f6e19d7384b4f1c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 12:23:22 +0100 Subject: [PATCH 0560/3444] Handle redirects transparently by default in shell and fetch Adds --no-status-aware command line option to have previous behaviour --- scrapy/commands/fetch.py | 5 ++++- scrapy/commands/shell.py | 4 +++- scrapy/shell.py | 12 ++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index f09a873c1..a157b19f8 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,6 +27,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not handle status codes like redirects and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -50,7 +52,8 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + if opts.no_status_aware: + request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7be7f7256..bc0203d89 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,6 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not transparently handle status codes like redirects") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -68,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url) + shell.start(url=url, handle_statuses=opts.no_status_aware) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 183ee1f70..966003f17 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -40,11 +40,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None): + def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider) + self.fetch(url, spider, handle_statuses=handle_statuses) elif request: self.fetch(request, spider) elif response: @@ -98,14 +98,14 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None): + def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): if isinstance(request_or_url, Request): request = request_or_url - url = request.url else: url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + request = Request(url, dont_filter=True, **kwargs) + if handle_statuses: + request.meta['handle_httpstatus_all'] = True response = None try: response, spider = threads.blockingCallFromThread( From 9aefc0a886ca571a49f087e5349e2557fd78d943 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 13:41:51 +0100 Subject: [PATCH 0561/3444] Add test for fetch command with redirections disabled --- scrapy/utils/testsite.py | 8 ++++++++ tests/test_command_fetch.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ad0375443..e50a989b3 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -20,12 +20,20 @@ class SiteTest(object): return urljoin(self.baseurl, path) +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request): + content = util.Redirect.render(self, request) + return content.replace(b'http-equiv=\"refresh\"', + b'http-no-equiv=\"do-not-refresh-me\"') + + def test_site(): r = resource.Resource() r.putChild(b"text", static.Data(b"Works", "text/plain")) r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 4843a9a2f..45d03a129 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/text')]) self.assertEqual(out.strip(), b'Works') + @defer.inlineCallbacks + def test_redirect_default(self): + _, out, _ = yield self.execute([self.url('/redirect')]) + self.assertEqual(out.strip(), b'Redirected here') + + @defer.inlineCallbacks + def test_redirect_disabled(self): + _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')]) + err = err.strip() + self.assertIn(b'downloader/response_status_count/302', err, err) + self.assertNotIn(b'downloader/response_status_count/200', err, err) + @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) From 03cf5f1bd2019127933ea4a6358a7d47743efcf6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:18:57 +0100 Subject: [PATCH 0562/3444] Remove ChunkedTransferMiddleware from default settings --- docs/topics/downloader-middleware.rst | 8 +++++++- docs/topics/settings.rst | 1 - scrapy/settings/default_settings.py | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15069e56e..dca5ec6a0 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -665,7 +665,13 @@ ChunkedTransferMiddleware .. class:: ChunkedTransferMiddleware - This middleware adds support for `chunked transfer encoding`_ + This middleware adds support for `chunked transfer encoding`_. + +.. note:: + This middleware is not enabled nor used by Scrapy downloader anymore. + In fact, Scrapy downloader has built-in support for chunked transfers, + so this middleware has no effect in practice. + HttpProxyMiddleware ------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..2195e4233 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -468,7 +468,6 @@ Default:: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, } diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..61f4bd567 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -102,7 +102,6 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, # Downloader side From e6f174b01535810fffeae2d020bc66a6a25ba4e8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:33:27 +0100 Subject: [PATCH 0563/3444] Add deprecation warning for ChunkedTransfer middleware --- scrapy/downloadermiddlewares/chunked.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 57e97e4d2..fd90aab2a 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -1,6 +1,14 @@ +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer +warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` " + "is deprecated, chunked transfers are supported by default.", + ScrapyDeprecationWarning, stacklevel=2) + + class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding From 8cffb4bbefcebd10ec12ee2678fd490edf149576 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:50:21 +0100 Subject: [PATCH 0564/3444] Update warning wording --- scrapy/downloadermiddlewares/chunked.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index fd90aab2a..64d94c489 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -4,8 +4,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer -warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` " - "is deprecated, chunked transfers are supported by default.", +warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " + "chunked transfers are supported by default.", ScrapyDeprecationWarning, stacklevel=2) From 059085b5b4a54901c4fd0730e43b156a6d8eeb51 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 18:23:34 +0100 Subject: [PATCH 0565/3444] Remove docs for deprecated ChunkedTransfer middleware --- docs/topics/downloader-middleware.rst | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dca5ec6a0..29d9b0298 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -657,22 +657,6 @@ Default: ``True`` Whether the Compression middleware will be enabled. -ChunkedTransferMiddleware -------------------------- - -.. module:: scrapy.downloadermiddlewares.chunked - :synopsis: Chunked Transfer Middleware - -.. class:: ChunkedTransferMiddleware - - This middleware adds support for `chunked transfer encoding`_. - -.. note:: - This middleware is not enabled nor used by Scrapy downloader anymore. - In fact, Scrapy downloader has built-in support for chunked transfers, - so this middleware has no effect in practice. - - HttpProxyMiddleware ------------------- @@ -976,4 +960,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html -.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding From f98ffb53b66cf2e5b38b393f811f59ccc1d68992 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 29 Nov 2016 16:52:54 +0100 Subject: [PATCH 0566/3444] add docs for some scheduler settings --- docs/faq.rst | 2 +- docs/topics/settings.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 415331515..ad11b071b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -317,6 +317,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. .. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/LIFO +.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..91f3b37c9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1033,6 +1033,35 @@ Example entry in logs:: (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) + +.. setting:: SCHEDULER_DISK_QUEUE + + SCHEDULER_DISK_QUEUE +-------------------- + +Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` + +Type of disk queue that will be used by scheduler. Other available types are +``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +``scrapy.squeues.MarshalLifoDiskQueue``. + +.. setting:: SCHEDULER_MEMORY_QUEUE + +SCHEDULER_MEMORY_QUEUE +---------------------- +Default: ``'scrapy.squeues.LifoMemoryQueue'`` + +Type of in-memory queue used by scheduler. Other available type is: +``scrapy.squeues.FifoMemoryQueue``. + +.. setting:: SCHEDULER_PRIORITY_QUEUE + +SCHEDULER_PRIORITY_QUEUE +------------------------ +Default: ``'queuelib.PriorityQueue'`` + +Type of priority queue used by scheduler. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS From 624284e85145aa62b0e7fc21bf0d61bda2cbbefa Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 29 Nov 2016 18:18:59 +0100 Subject: [PATCH 0567/3444] Fix indent --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 91f3b37c9..fe3767a53 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1036,7 +1036,7 @@ Example entry in logs:: .. setting:: SCHEDULER_DISK_QUEUE - SCHEDULER_DISK_QUEUE +SCHEDULER_DISK_QUEUE -------------------- Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` From 27606aad1166afe3eebbee8dc365b1a122639e3c Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 30 Nov 2016 09:47:02 +0200 Subject: [PATCH 0568/3444] Fix #396 re-triggered issue The InteractiveShellEmbed class is a singleton and we need to drop the instance with its clear_instance() method to rebuild the instance from scratch with fresh environment for each subsequent Scrapy's shell drop in. --- scrapy/utils/console.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 567fd51bc..1888d9599 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -15,6 +15,9 @@ def _embed_ipython_shell(namespace={}, banner=''): config = load_default_config() # Always use .instace() to ensure _instance propagation to all parents # this is needed for completion works well for new imports + # and clear the instance to always have the fresh env + # on repeated breaks like with inspect_response() + InteractiveShellEmbed.clear_instance() shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config) shell() From 5ff64ad015aff872ffae78193b67525dd53ae80d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Nov 2016 11:47:25 -0300 Subject: [PATCH 0569/3444] handle relative sitemap urls in robots.txt --- scrapy/spiders/sitemap.py | 2 +- scrapy/utils/sitemap.py | 7 +++++-- tests/test_spider.py | 6 +++++- tests/test_utils_sitemap.py | 9 +++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 89d96c330..9e45637c3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -32,7 +32,7 @@ class SitemapSpider(Spider): def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): - for url in sitemap_urls_from_robots(response.text): + for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) else: body = self._get_sitemap_body(response) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 008196435..4742b3e13 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,9 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + import lxml.etree +from six.moves.urllib.parse import urljoin class Sitemap(object): @@ -34,10 +36,11 @@ class Sitemap(object): yield d -def sitemap_urls_from_robots(robots_text): +def sitemap_urls_from_robots(robots_text, base_url=None): """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith('sitemap:'): - yield line.split(':', 1)[1].strip() + url = line.split(':', 1)[1].strip() + yield urljoin(base_url, url) diff --git a/tests/test_spider.py b/tests/test_spider.py index 1d22c1212..079734a69 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -332,13 +332,17 @@ class SitemapSpiderTest(SpiderTest): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml """ r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://example.com/sitemap.xml', - 'http://example.com/sitemap-product-index.xml']) + 'http://example.com/sitemap-product-index.xml', + 'http://example.com/sitemap-uppercase.xml', + 'http://www.example.com/sitemap-relative-url.xml']) class BaseSpiderDeprecationTest(unittest.TestCase): diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index bd2677956..716bb44eb 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -119,13 +119,18 @@ Disallow: /s*/*tags # Sitemap files Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml # Forums Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots)), - ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')), + ['http://example.com/sitemap.xml', + 'http://example.com/sitemap-product-index.xml', + 'http://example.com/sitemap-uppercase.xml', + 'http://example.com/sitemap-relative-url.xml']) def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before tag""" From d9f43e21ba510ee3c95c89c3d551bc30a8d0f2c9 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 1 Dec 2016 11:56:33 -0300 Subject: [PATCH 0570/3444] TST: Fix duplicated test name. --- tests/test_spiderloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 68dca2e98..83c3a3670 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -66,7 +66,7 @@ class SpiderLoaderTest(unittest.TestCase): self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 1 - def test_load_spider_module(self): + def test_load_spider_module_multiple(self): prefix = 'tests.test_spiderloader.test_spiders.' module = ','.join(prefix + s for s in ('spider1', 'spider2')) settings = Settings({'SPIDER_MODULES': module}) From 6431e7a1386e03fff203ae85b6c47a4f0f4cf797 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 1 Dec 2016 13:24:12 -0300 Subject: [PATCH 0571/3444] DOC State explicitly that spiders are loaded recursively. --- docs/topics/api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index d470a0d41..985cc0433 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -171,7 +171,8 @@ SpiderLoader API This class method is used by Scrapy to create an instance of the class. It's called with the current project settings, and it loads the spiders - found in the modules of the :setting:`SPIDER_MODULES` setting. + found recursively in the modules of the :setting:`SPIDER_MODULES` + setting. :param settings: project settings :type settings: :class:`~scrapy.settings.Settings` instance From 923b974f0a10c3eb007ee4d183aa63a41d6a6db2 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 1 Dec 2016 12:52:52 -0300 Subject: [PATCH 0572/3444] TST Include nested a nested spider in spider loader test. --- tests/test_spiderloader/__init__.py | 6 +++--- tests/test_spiderloader/test_spiders/nested/__init__.py | 0 tests/test_spiderloader/test_spiders/nested/spider4.py | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 tests/test_spiderloader/test_spiders/nested/__init__.py create mode 100644 tests/test_spiderloader/test_spiders/nested/spider4.py diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 68dca2e98..74e461215 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -9,6 +9,7 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy +import tempfile from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings @@ -22,8 +23,7 @@ class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = self.mktemp() - os.mkdir(self.tmpdir) + self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) @@ -40,7 +40,7 @@ class SpiderLoaderTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spider_loader.list()), - set(['spider1', 'spider2', 'spider3'])) + set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_load(self): spider1 = self.spider_loader.load("spider1") diff --git a/tests/test_spiderloader/test_spiders/nested/__init__.py b/tests/test_spiderloader/test_spiders/nested/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py new file mode 100644 index 000000000..35b71870a --- /dev/null +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -0,0 +1,9 @@ +from scrapy.spiders import Spider + +class Spider4(Spider): + name = "spider4" + allowed_domains = ['spider4.com'] + + @classmethod + def handles_request(cls, request): + return request.url == 'http://spider4.com/onlythis' From e1ea0c433a96c2949b0a7c26cde65aea13fb79b4 Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 22:02:10 +0000 Subject: [PATCH 0573/3444] Strip xlib.tx code of Twisted 10 --- scrapy/xlib/tx/__init__.py | 12 ++++----- scrapy/xlib/tx/_newclient.py | 31 +++++++++++++++++------- scrapy/xlib/tx/client.py | 28 +++++++++++++++------ scrapy/xlib/tx/endpoints.py | 26 +++++++++++++++----- scrapy/xlib/tx/interfaces.py | 47 ++++++++++++++++++++++++++---------- scrapy/xlib/tx/iweb.py | 18 ++++++++++---- 6 files changed, 115 insertions(+), 47 deletions(-) diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index 1ac4e0108..1c9bf09e5 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -15,9 +15,9 @@ else: client = endpoints = _Mock() -Agent = client.Agent -ProxyAgent = client.ProxyAgent -ResponseDone = client.ResponseDone -ResponseFailed = client.ResponseFailed -HTTPConnectionPool = client.HTTPConnectionPool -TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint +Agent = client.Agent # since < 11.1 +ProxyAgent = client.ProxyAgent # since 11.1 +ResponseDone = client.ResponseDone # since 11.1 +ResponseFailed = client.ResponseFailed # since 11.1 +HTTPConnectionPool = client.HTTPConnectionPool # since 12.1 +TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index 16d0ca6b4..e902d6683 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder -from .iweb import IResponse, UNKNOWN_LENGTH +from twisted.web._newclient import ( + BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, + RequestGenerationFailed, RequestTransmissionFailed, + WrongBodyLength, ResponseDone, RequestNotSent, + LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder, + TransportProxyProducer, +) +# newer than 10.0.0 +#from twisted.web._newclient import ( +# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser, +# HTTPClientParser, Request, Response, HTTP11ClientProtocol, +#) +from .iweb import IResponse # States HTTPParser can be in STATUS = 'STATUS' @@ -52,7 +65,7 @@ HEADER = 'HEADER' BODY = 'BODY' DONE = 'DONE' - +''' {{{ class BadHeaders(Exception): """ Headers passed to L{Request} were in some way invalid. @@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException): @ivar reasons: A C{list} of one or more L{Failure} instances giving the reasons the request transmission was considered to have failed. """ - +}}} ''' class ConnectionAborted(Exception): @@ -126,7 +139,7 @@ class ConnectionAborted(Exception): """ - +''' {{{ class WrongBodyLength(Exception): """ An L{IBodyProducer} declared the number of bytes it was going to @@ -142,7 +155,7 @@ class ResponseDone(Exception): protocol passed to L{Response.deliverBody} and indicates that the entire response has been delivered. """ - +}}} ''' class ResponseFailed(_WrapperException): @@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed): """ - +''' {{{ class RequestNotSent(Exception): """ L{RequestNotSent} indicates that an attempt was made to issue a request but @@ -178,7 +191,7 @@ class RequestNotSent(Exception): to send a request using a protocol which is no longer connected to a server. """ - +}}} ''' def _callAppFunction(function): @@ -764,7 +777,7 @@ class Request: _callAppFunction(self.bodyProducer.stopProducing) - +''' {{{ class LengthEnforcingConsumer: """ An L{IConsumer} proxy which enforces an exact length requirement on the @@ -1188,7 +1201,7 @@ class TransportProxyProducer: """ if self._producer is not None: self._producer.pauseProducing() - +}}} ''' class HTTP11ClientProtocol(Protocol): diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index c3830dc47..396115985 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocol from twisted.python import failure from twisted.python.components import proxyForInterface from twisted.web import error +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from twisted.web.http_headers import Headers +from twisted.web.client import ( + PartialDownloadError, +) +# newer than 10.0.0 +#from twisted.web.client import ( +# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer, +# HTTPConnectionPool, Agent, ProxyAgent, +#) + from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer - +from .iweb import IResponse +''' {{{ class PartialDownloadError(error.Error): """ Page was only partially downloaded, we got disconnected in middle. @ivar response: All of the response body which was downloaded. """ - +}}} ''' class _URL(tuple): """ @@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, from twisted.web.error import SchemeNotSupported from ._newclient import Request, Response, HTTP11ClientProtocol -from ._newclient import ResponseDone, ResponseFailed -from ._newclient import RequestNotSent, RequestTransmissionFailed +from twisted.web._newclient import ResponseDone +from ._newclient import ResponseFailed +from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed +from twisted.web._newclient import ( + PotentialDataLoss, _WrapperException) from ._newclient import ( - ResponseNeverReceived, PotentialDataLoss, _WrapperException) + ResponseNeverReceived) try: from twisted.internet.ssl import ClientContextFactory @@ -1161,8 +1174,7 @@ def readBody(response): __all__ = [ - 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader', - 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage', + 'PartialDownloadError', 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index d8a92ccd0..21a467433 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions. from __future__ import division, absolute_import import os -import socket +#import socket from zope.interface import implementer, directlyProvides import warnings -from twisted.internet import interfaces, defer, error, fdesc, threads +from twisted.internet import interfaces, defer, error, fdesc from twisted.internet.protocol import ( - ClientFactory, Protocol, ProcessProtocol, Factory) + ClientFactory, Protocol, Factory) +#from twisted.internet import threads, ProcessProtocol from twisted.internet.interfaces import IStreamServerEndpointStringParser from twisted.internet.interfaces import IStreamClientEndpointStringParser from twisted.python.filepath import FilePath -from twisted.python.failure import Failure -from twisted.python import log +#from twisted.python.failure import Failure +#from twisted.python import log from twisted.python.components import proxyForInterface from twisted.plugin import IPlugin, getPlugins -from twisted.internet import stdio +#from twisted.internet import stdio + +# newer than 10.0.0 +#from twisted.internet.endpoints import ( +# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint, +# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol, +# quoteStringArgument, +# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers +# clientFromString, #> using newer _clientParsers +# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint, +# _parseTCP, _parseUNIX, _loadCAsFromDir, +# _parseSSL, _tokenize, +# _parseClientTCP, _parseClientSSL, _parseClientUNIX, +#) from .interfaces import IFileDescriptorReceiver diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index f3e4ed5d8..a715d4a05 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -11,7 +11,28 @@ from __future__ import division, absolute_import from zope.interface import Interface, Attribute +from twisted.internet.interfaces import ( + IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, + IReactorUDP, IReactorMulticast, IReactorProcess, + IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, + IReactorPluggableResolver, IReactorFDSet, + IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, + IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, + ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, + IProtocol, IProcessProtocol, IHalfCloseableProtocol, + IProtocolFactory, ITransport, IProcessTransport, IServiceCollection, + IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, + IMulticastTransport, +) +# newer than 10.0.0 +#from twisted.internet.interfaces import ( +# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket, +# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport, +# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint, +# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, +#) +''' {{{ class IAddress(Interface): """ An address, e.g. a TCP C{(host, port)}. @@ -74,7 +95,7 @@ class IResolverSimple(Interface): @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) if the name cannot be resolved within the specified timeout period. """ - +}}} ''' class IResolver(IResolverSimple): @@ -614,7 +635,7 @@ class IResolver(IResolverSimple): """ - +''' {{{ class IReactorTCP(Interface): def listenTCP(port, factory, backlog=50, interface=''): @@ -701,7 +722,7 @@ class IReactorSSL(Interface): @param interface: the hostname to bind to, defaults to '' (all) """ - +}}} ''' class IReactorUNIX(Interface): @@ -829,7 +850,7 @@ class IReactorWin32Events(Interface): """ - +''' {{{ class IReactorUDP(Interface): """ UDP socket methods. @@ -868,7 +889,7 @@ class IReactorMulticast(Interface): @see: L{twisted.internet.interfaces.IMulticastTransport} @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} """ - +}}} ''' class IReactorSocket(Interface): @@ -970,7 +991,7 @@ class IReactorSocket(Interface): """ - +''' {{{ class IReactorProcess(Interface): def spawnProcess(processProtocol, executable, args=(), env={}, path=None, @@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface): @return: The previously installed resolver. """ - +}}} ''' class IReactorDaemonize(Interface): """ @@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface): """ - +''' {{{ class IReactorFDSet(Interface): """ Implement me to be able to use L{IFileDescriptor} type resources. @@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface): This will never be called for TCP connections as TCP does not support notification of this type of half-close. """ - +}}} ''' class IFileDescriptorReceiver(Interface): @@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface): """ - +''' {{{ class IProtocolFactory(Interface): """ Interface for protocol factories. @@ -1974,7 +1995,7 @@ class ITransport(Interface): @return: An L{IAddress} provider. """ - +}}} ''' class ITCPTransport(ITransport): """ @@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport): Return an object with the peer's certificate info. """ - +''' {{{ class IProcessTransport(ITransport): """ A process transport. @@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface): """ Leave multicast group, return L{Deferred} of success. """ - +}}} ''' class IStreamClientEndpoint(Interface): """ diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index ddcb6ed7a..57a111411 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}. from zope.interface import Interface, Attribute -from twisted.internet.interfaces import IPushProducer +#from twisted.internet.interfaces import IPushProducer +from twisted.web.iweb import ( + ICredentialFactory, IBodyProducer, + UNKNOWN_LENGTH, +) +# newer than 10.0.0 +#from twisted.web.iweb import ( +# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, +#) class IRequest(Interface): """ @@ -320,7 +328,7 @@ class IRequest(Interface): """ - +''' {{{ class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of @@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer): L{Deferred} returned by C{startProducing} is never fired. """ - +}}} ''' class IRenderable(Interface): @@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface): """ - +''' {{{ UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" - +}}} ''' __all__ = [ "ICredentialFactory", "IRequest", "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", From c8cf1a303d2272aa44422035fd1e3e6048cb5606 Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 22:02:11 +0000 Subject: [PATCH 0574/3444] Bump Twisted dependency to 13.1.0 (released June 2013) --- requirements.txt | 2 +- scrapy/xlib/tx/_newclient.py | 36 +++++++++++++--------------- scrapy/xlib/tx/client.py | 38 +++++++++++++---------------- scrapy/xlib/tx/endpoints.py | 38 +++++++++++++---------------- scrapy/xlib/tx/interfaces.py | 46 +++++++++++++++++------------------- scrapy/xlib/tx/iweb.py | 15 +++++------- setup.py | 2 +- 7 files changed, 79 insertions(+), 98 deletions(-) diff --git a/requirements.txt b/requirements.txt index cfa907050..64b6e771c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=10.0.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index e902d6683..d20eda34f 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDone from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol -from twisted.protocols.basic import LineReceiver -from twisted.web.iweb import UNKNOWN_LENGTH -from twisted.web.http_headers import Headers +#from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH, IResponse +#from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder from twisted.web._newclient import ( BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, - RequestGenerationFailed, RequestTransmissionFailed, - WrongBodyLength, ResponseDone, RequestNotSent, - LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder, - TransportProxyProducer, + RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted, + WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent, + ResponseNeverReceived, HTTPParser, HTTPClientParser, Request, + LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder, + TransportProxyProducer, HTTP11ClientProtocol ) -# newer than 10.0.0 -#from twisted.web._newclient import ( -# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser, -# HTTPClientParser, Request, Response, HTTP11ClientProtocol, -#) -from .iweb import IResponse # States HTTPParser can be in STATUS = 'STATUS' @@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException): @ivar reasons: A C{list} of one or more L{Failure} instances giving the reasons the request transmission was considered to have failed. """ -}}} ''' + class ConnectionAborted(Exception): @@ -139,7 +134,7 @@ class ConnectionAborted(Exception): """ -''' {{{ + class WrongBodyLength(Exception): """ An L{IBodyProducer} declared the number of bytes it was going to @@ -155,7 +150,7 @@ class ResponseDone(Exception): protocol passed to L{Response.deliverBody} and indicates that the entire response has been delivered. """ -}}} ''' + class ResponseFailed(_WrapperException): @@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed): """ -''' {{{ + class RequestNotSent(Exception): """ L{RequestNotSent} indicates that an attempt was made to issue a request but @@ -191,7 +186,7 @@ class RequestNotSent(Exception): to send a request using a protocol which is no longer connected to a server. """ -}}} ''' + def _callAppFunction(function): @@ -777,7 +772,7 @@ class Request: _callAppFunction(self.bodyProducer.stopProducing) -''' {{{ + class LengthEnforcingConsumer: """ An L{IConsumer} proxy which enforces an exact length requirement on the @@ -1201,7 +1196,7 @@ class TransportProxyProducer: """ if self._producer is not None: self._producer.pauseProducing() -}}} ''' + class HTTP11ClientProtocol(Protocol): @@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol): d = Deferred() self._abortDeferreds.append(d) return d +}}} ''' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index 396115985..8e0b1df8b 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -29,23 +29,18 @@ from twisted.python.failure import Failure from twisted.web import http from twisted.internet import defer, protocol, task, reactor from twisted.internet.interfaces import IProtocol +from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint from twisted.python import failure from twisted.python.components import proxyForInterface from twisted.web import error -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse from twisted.web.http_headers import Headers from twisted.web.client import ( - PartialDownloadError, + PartialDownloadError, FileBodyProducer, + CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, + Agent, ProxyAgent, HTTPConnectionPool, readBody, ) -# newer than 10.0.0 -#from twisted.web.client import ( -# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer, -# HTTPConnectionPool, Agent, ProxyAgent, -#) - -from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse ''' {{{ class PartialDownloadError(error.Error): @@ -54,7 +49,7 @@ class PartialDownloadError(error.Error): @ivar response: All of the response body which was downloaded. """ -}}} ''' + class _URL(tuple): """ @@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, else: reactor.connectTCP(host, port, factory) return factory - +}}} ''' # The code which follows is based on the new HTTP client implementation. It # should be significantly better than anything above, though it is not yet # feature equivalent. -from twisted.web.error import SchemeNotSupported -from ._newclient import Request, Response, HTTP11ClientProtocol -from twisted.web._newclient import ResponseDone -from ._newclient import ResponseFailed -from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed -from twisted.web._newclient import ( - PotentialDataLoss, _WrapperException) -from ._newclient import ( - ResponseNeverReceived) +#from twisted.web.error import SchemeNotSupported +from twisted.web._newclient import Response +#from twisted.web._newclient import Request, HTTP11ClientProtocol +from twisted.web._newclient import ResponseDone, ResponseFailed +#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed +#from twisted.web._newclient import ( +# ResponseNeverReceived, PotentialDataLoss, _WrapperException) +''' {{{ try: from twisted.internet.ssl import ClientContextFactory except ImportError: @@ -1170,7 +1164,7 @@ def readBody(response): d = defer.Deferred() response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) return d - +}}} ''' __all__ = [ diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index 21a467433..3f4704064 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvides import warnings from twisted.internet import interfaces, defer, error, fdesc -from twisted.internet.protocol import ( - ClientFactory, Protocol, Factory) +#from twisted.internet.protocol import ( +# ClientFactory, Protocol) +from twisted.internet.protocol import Factory #from twisted.internet import threads, ProcessProtocol from twisted.internet.interfaces import IStreamServerEndpointStringParser -from twisted.internet.interfaces import IStreamClientEndpointStringParser +#from twisted.internet.interfaces import IStreamClientEndpointStringParser from twisted.python.filepath import FilePath #from twisted.python.failure import Failure #from twisted.python import log -from twisted.python.components import proxyForInterface +#from twisted.python.components import proxyForInterface from twisted.plugin import IPlugin, getPlugins #from twisted.internet import stdio -# newer than 10.0.0 -#from twisted.internet.endpoints import ( -# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint, -# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol, -# quoteStringArgument, -# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers -# clientFromString, #> using newer _clientParsers -# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint, -# _parseTCP, _parseUNIX, _loadCAsFromDir, -# _parseSSL, _tokenize, -# _parseClientTCP, _parseClientSSL, _parseClientUNIX, -#) - -from .interfaces import IFileDescriptorReceiver +from twisted.internet.endpoints import ( + clientFromString, serverFromString, quoteStringArgument, + TCP4ServerEndpoint, TCP6ServerEndpoint, + TCP4ClientEndpoint, TCP6ClientEndpoint, + UNIXServerEndpoint, UNIXClientEndpoint, + SSL4ServerEndpoint, SSL4ClientEndpoint, + AdoptedStreamServerEndpoint, connectProtocol, +) __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - +''' {{{ class _WrappingProtocol(Protocol): """ Wrap another protocol in order to notify my user when a connection has @@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol): self._wrappedProtocol = wrappedProtocol for iface in [interfaces.IHalfCloseableProtocol, - IFileDescriptorReceiver]: + interfaces.IFileDescriptorReceiver]: if iface.providedBy(self._wrappedProtocol): directlyProvides(self, iface) @@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object): + def _parseTCP(factory, port, interface="", backlog=50): """ Internal parser function for L{_parseServer} to convert the string @@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol): def buildProtocol(self, addr): return protocol return endpoint.connect(OneShotFactory()) - +}}} ''' diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index a715d4a05..7b2a78632 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -13,24 +13,21 @@ from zope.interface import Interface, Attribute from twisted.internet.interfaces import ( IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, - IReactorUDP, IReactorMulticast, IReactorProcess, + IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, - IReactorPluggableResolver, IReactorFDSet, + IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet, IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, IProtocol, IProcessProtocol, IHalfCloseableProtocol, - IProtocolFactory, ITransport, IProcessTransport, IServiceCollection, + IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport, + IUNIXTransport, + ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection, IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, - IMulticastTransport, + IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint, + IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, + IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver ) -# newer than 10.0.0 -#from twisted.internet.interfaces import ( -# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket, -# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport, -# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint, -# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, -#) ''' {{{ class IAddress(Interface): @@ -95,7 +92,7 @@ class IResolverSimple(Interface): @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) if the name cannot be resolved within the specified timeout period. """ -}}} ''' + class IResolver(IResolverSimple): @@ -635,7 +632,7 @@ class IResolver(IResolverSimple): """ -''' {{{ + class IReactorTCP(Interface): def listenTCP(port, factory, backlog=50, interface=''): @@ -722,7 +719,7 @@ class IReactorSSL(Interface): @param interface: the hostname to bind to, defaults to '' (all) """ -}}} ''' + class IReactorUNIX(Interface): @@ -850,7 +847,7 @@ class IReactorWin32Events(Interface): """ -''' {{{ + class IReactorUDP(Interface): """ UDP socket methods. @@ -889,7 +886,7 @@ class IReactorMulticast(Interface): @see: L{twisted.internet.interfaces.IMulticastTransport} @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} """ -}}} ''' + class IReactorSocket(Interface): @@ -991,7 +988,7 @@ class IReactorSocket(Interface): """ -''' {{{ + class IReactorProcess(Interface): def spawnProcess(processProtocol, executable, args=(), env={}, path=None, @@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface): @return: The previously installed resolver. """ -}}} ''' + class IReactorDaemonize(Interface): """ @@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface): """ -''' {{{ + class IReactorFDSet(Interface): """ Implement me to be able to use L{IFileDescriptor} type resources. @@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface): This will never be called for TCP connections as TCP does not support notification of this type of half-close. """ -}}} ''' + class IFileDescriptorReceiver(Interface): @@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface): """ -''' {{{ + class IProtocolFactory(Interface): """ Interface for protocol factories. @@ -1995,7 +1992,7 @@ class ITransport(Interface): @return: An L{IAddress} provider. """ -}}} ''' + class ITCPTransport(ITransport): """ @@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport): Return an object with the peer's certificate info. """ -''' {{{ + class IProcessTransport(ITransport): """ A process transport. @@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface): """ Leave multicast group, return L{Deferred} of success. """ -}}} ''' + class IStreamClientEndpoint(Interface): """ @@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface): @return: a client endpoint @rtype: L{IStreamClientEndpoint} """ +}}} ''' diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index 57a111411..32c88ff2d 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute #from twisted.internet.interfaces import IPushProducer from twisted.web.iweb import ( - ICredentialFactory, IBodyProducer, - UNKNOWN_LENGTH, + IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, + IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, ) -# newer than 10.0.0 -#from twisted.web.iweb import ( -# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, -#) +''' {{{ class IRequest(Interface): """ An HTTP request. @@ -328,7 +325,7 @@ class IRequest(Interface): """ -''' {{{ + class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of @@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer): L{Deferred} returned by C{startProducing} is never fired. """ -}}} ''' + class IRenderable(Interface): @@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface): """ -''' {{{ + UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" }}} ''' __all__ = [ diff --git a/setup.py b/setup.py index 92c114a7a..5e32d4240 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], install_requires=[ - 'Twisted>=10.0.0', + 'Twisted>=13.1.0', 'w3lib>=1.15.0', 'queuelib', 'lxml', From 985755d1fe0aabf922bcdb0e8bc22d67948820cb Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 23:44:15 +0000 Subject: [PATCH 0575/3444] Remove obsolete xlib code for Twisted 13.1.0 --- scrapy/xlib/tx/_newclient.py | 1466 -------------------- scrapy/xlib/tx/client.py | 1116 ---------------- scrapy/xlib/tx/endpoints.py | 1253 ----------------- scrapy/xlib/tx/interfaces.py | 2433 ---------------------------------- scrapy/xlib/tx/iweb.py | 569 -------- 5 files changed, 6837 deletions(-) diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index d20eda34f..39cd20f95 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -38,9 +38,7 @@ from twisted.internet.error import ConnectionDone from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol -#from twisted.protocols.basic import LineReceiver from twisted.web.iweb import UNKNOWN_LENGTH, IResponse -#from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder @@ -59,1467 +57,3 @@ STATUS = 'STATUS' HEADER = 'HEADER' BODY = 'BODY' DONE = 'DONE' - -''' {{{ -class BadHeaders(Exception): - """ - Headers passed to L{Request} were in some way invalid. - """ - - - -class ExcessWrite(Exception): - """ - The body L{IBodyProducer} for a request tried to write data after - indicating it had finished writing data. - """ - - -class ParseError(Exception): - """ - Some received data could not be parsed. - - @ivar data: The string which could not be parsed. - """ - def __init__(self, reason, data): - Exception.__init__(self, reason, data) - self.data = data - - - -class BadResponseVersion(ParseError): - """ - The version string in a status line was unparsable. - """ - - - -class _WrapperException(Exception): - """ - L{_WrapperException} is the base exception type for exceptions which - include one or more other exceptions as the low-level causes. - - @ivar reasons: A list of exceptions. See subclass documentation for more - details. - """ - def __init__(self, reasons): - Exception.__init__(self, reasons) - self.reasons = reasons - - - -class RequestGenerationFailed(_WrapperException): - """ - There was an error while creating the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request generation was considered to have failed. - """ - - - -class RequestTransmissionFailed(_WrapperException): - """ - There was an error while sending the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request transmission was considered to have failed. - """ - - - -class ConnectionAborted(Exception): - """ - The connection was explicitly aborted by application code. - """ - - - -class WrongBodyLength(Exception): - """ - An L{IBodyProducer} declared the number of bytes it was going to - produce (via its C{length} attribute) and then produced a different number - of bytes. - """ - - - -class ResponseDone(Exception): - """ - L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the - protocol passed to L{Response.deliverBody} and indicates that the entire - response has been delivered. - """ - - - -class ResponseFailed(_WrapperException): - """ - L{ResponseFailed} indicates that all of the response to a request was not - received for some reason. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the response was considered to have failed. - - @ivar response: If specified, the L{Response} received from the server (and - in particular the status code and the headers). - """ - - def __init__(self, reasons, response=None): - _WrapperException.__init__(self, reasons) - self.response = response - - - -class ResponseNeverReceived(ResponseFailed): - """ - A L{ResponseFailed} that knows no response bytes at all have been received. - """ - - - -class RequestNotSent(Exception): - """ - L{RequestNotSent} indicates that an attempt was made to issue a request but - for reasons unrelated to the details of the request itself, the request - could not be sent. For example, this may indicate that an attempt was made - to send a request using a protocol which is no longer connected to a - server. - """ - - - -def _callAppFunction(function): - """ - Call C{function}. If it raises an exception, log it with a minimal - description of the source. - - @return: C{None} - """ - try: - function() - except: - log.err(None, "Unexpected exception from %s" % ( - fullyQualifiedName(function),)) - - - -class HTTPParser(LineReceiver): - """ - L{HTTPParser} handles the parsing side of HTTP processing. With a suitable - subclass, it can parse either the client side or the server side of the - connection. - - @ivar headers: All of the non-connection control message headers yet - received. - - @ivar state: State indicator for the response parsing state machine. One - of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}. - - @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline - header while that header is being received. - """ - - # NOTE: According to HTTP spec, we're supposed to eat the - # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that - # doesn't sound like a good idea to me, because it makes it impossible to - # have a non-authenticating transparent proxy in front of an authenticating - # proxy. An authenticating proxy can eat them itself. -jknight - # - # Further, quoting - # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html - # regarding the 'Proxy-Connection' header: - # - # The Proxy-Connection: header is a mistake in how some web browsers - # use HTTP. Its name is the result of a false analogy. It is not a - # standard part of the protocol. There is a different standard - # protocol mechanism for doing what it does. And its existence - # imposes a requirement upon HTTP servers such that no proxy HTTP - # server can be standards-conforming in practice. - # - # -exarkun - - # Some servers (like http://news.ycombinator.com/) return status lines and - # HTTP headers delimited by \n instead of \r\n. - delimiter = '\n' - - CONNECTION_CONTROL_HEADERS = set([ - 'content-length', 'connection', 'keep-alive', 'te', 'trailers', - 'transfer-encoding', 'upgrade', 'proxy-connection']) - - def connectionMade(self): - self.headers = Headers() - self.connHeaders = Headers() - self.state = STATUS - self._partialHeader = None - - - def switchToBodyMode(self, decoder): - """ - Switch to body parsing mode - interpret any more bytes delivered as - part of the message body and deliver them to the given decoder. - """ - if self.state == BODY: - raise RuntimeError("already in body mode") - - self.bodyDecoder = decoder - self.state = BODY - self.setRawMode() - - - def lineReceived(self, line): - """ - Handle one line from a response. - """ - # Handle the normal CR LF case. - if line[-1:] == '\r': - line = line[:-1] - - if self.state == STATUS: - self.statusReceived(line) - self.state = HEADER - elif self.state == HEADER: - if not line or line[0] not in ' \t': - if self._partialHeader is not None: - header = ''.join(self._partialHeader) - name, value = header.split(':', 1) - value = value.strip() - self.headerReceived(name, value) - if not line: - # Empty line means the header section is over. - self.allHeadersReceived() - else: - # Line not beginning with LWS is another header. - self._partialHeader = [line] - else: - # A line beginning with LWS is a continuation of a header - # begun on a previous line. - self._partialHeader.append(line) - - - def rawDataReceived(self, data): - """ - Pass data from the message body to the body decoder object. - """ - self.bodyDecoder.dataReceived(data) - - - def isConnectionControlHeader(self, name): - """ - Return C{True} if the given lower-cased name is the name of a - connection control header (rather than an entity header). - - According to RFC 2616, section 14.10, the tokens in the Connection - header are probably relevant here. However, I am not sure what the - practical consequences of either implementing or ignoring that are. - So I leave it unimplemented for the time being. - """ - return name in self.CONNECTION_CONTROL_HEADERS - - - def statusReceived(self, status): - """ - Callback invoked whenever the first line of a new message is received. - Override this. - - @param status: The first line of an HTTP request or response message - without trailing I{CR LF}. - @type status: C{str} - """ - - - def headerReceived(self, name, value): - """ - Store the given header in C{self.headers}. - """ - name = name.lower() - if self.isConnectionControlHeader(name): - headers = self.connHeaders - else: - headers = self.headers - headers.addRawHeader(name, value) - - - def allHeadersReceived(self): - """ - Callback invoked after the last header is passed to C{headerReceived}. - Override this to change to the C{BODY} or C{DONE} state. - """ - self.switchToBodyMode(None) - - - -class HTTPClientParser(HTTPParser): - """ - An HTTP parser which only handles HTTP responses. - - @ivar request: The request with which the expected response is associated. - @type request: L{Request} - - @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a - body. - - @ivar finisher: A callable to invoke when this response is fully parsed. - - @ivar _responseDeferred: A L{Deferred} which will be called back with the - response when all headers in the response have been received. - Thereafter, C{None}. - - @ivar _everReceivedData: C{True} if any bytes have been received. - """ - NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED]) - - _transferDecoders = { - 'chunked': _ChunkedTransferDecoder, - } - - bodyDecoder = None - - def __init__(self, request, finisher): - self.request = request - self.finisher = finisher - self._responseDeferred = Deferred() - self._everReceivedData = False - - - def dataReceived(self, data): - """ - Override so that we know if any response has been received. - """ - self._everReceivedData = True - HTTPParser.dataReceived(self, data) - - - def parseVersion(self, strversion): - """ - Parse version strings of the form Protocol '/' Major '.' Minor. E.g. - 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError - on bad syntax. - """ - try: - proto, strnumber = strversion.split('/') - major, minor = strnumber.split('.') - major, minor = int(major), int(minor) - except ValueError as e: - raise BadResponseVersion(str(e), strversion) - if major < 0 or minor < 0: - raise BadResponseVersion("version may not be negative", strversion) - return (proto, major, minor) - - - def statusReceived(self, status): - """ - Parse the status line into its components and create a response object - to keep track of this response's state. - """ - parts = status.split(' ', 2) - if len(parts) != 3: - raise ParseError("wrong number of parts", status) - - try: - statusCode = int(parts[1]) - except ValueError: - raise ParseError("non-integer status code", status) - - self.response = Response( - self.parseVersion(parts[0]), - statusCode, - parts[2], - self.headers, - self.transport) - - - def _finished(self, rest): - """ - Called to indicate that an entire response has been received. No more - bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are - passed up and the state of this L{HTTPClientParser} is set to I{DONE}. - - @param rest: A C{str} giving any extra bytes delivered to this - L{HTTPClientParser} which are not part of the response being - parsed. - """ - self.state = DONE - self.finisher(rest) - - - def isConnectionControlHeader(self, name): - """ - Content-Length in the response to a HEAD request is an entity header, - not a connection control header. - """ - if self.request.method == 'HEAD' and name == 'content-length': - return False - return HTTPParser.isConnectionControlHeader(self, name) - - - def allHeadersReceived(self): - """ - Figure out how long the response body is going to be by examining - headers and stuff. - """ - if (self.response.code in self.NO_BODY_CODES - or self.request.method == 'HEAD'): - self.response.length = 0 - self._finished(self.clearLineBuffer()) - else: - transferEncodingHeaders = self.connHeaders.getRawHeaders( - 'transfer-encoding') - if transferEncodingHeaders: - - # This could be a KeyError. However, that would mean we do not - # know how to decode the response body, so failing the request - # is as good a behavior as any. Perhaps someday we will want - # to normalize/document/test this specifically, but failing - # seems fine to me for now. - transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()] - - # If anyone ever invents a transfer encoding other than - # chunked (yea right), and that transfer encoding can predict - # the length of the response body, it might be sensible to - # allow the transfer decoder to set the response object's - # length attribute. - else: - contentLengthHeaders = self.connHeaders.getRawHeaders('content-length') - if contentLengthHeaders is None: - contentLength = None - elif len(contentLengthHeaders) == 1: - contentLength = int(contentLengthHeaders[0]) - self.response.length = contentLength - else: - # "HTTP Message Splitting" or "HTTP Response Smuggling" - # potentially happening. Or it's just a buggy server. - raise ValueError( - "Too many Content-Length headers; response is invalid") - - if contentLength == 0: - self._finished(self.clearLineBuffer()) - transferDecoder = None - else: - transferDecoder = lambda x, y: _IdentityTransferDecoder( - contentLength, x, y) - - if transferDecoder is None: - self.response._bodyDataFinished() - else: - # Make sure as little data as possible from the response body - # gets delivered to the response object until the response - # object actually indicates it is ready to handle bytes - # (probably because an application gave it a way to interpret - # them). - self.transport.pauseProducing() - self.switchToBodyMode(transferDecoder( - self.response._bodyDataReceived, - self._finished)) - - # This must be last. If it were first, then application code might - # change some state (for example, registering a protocol to receive the - # response body). Then the pauseProducing above would be wrong since - # the response is ready for bytes and nothing else would ever resume - # the transport. - self._responseDeferred.callback(self.response) - del self._responseDeferred - - - def connectionLost(self, reason): - if self.bodyDecoder is not None: - try: - try: - self.bodyDecoder.noMoreData() - except PotentialDataLoss: - self.response._bodyDataFinished(Failure()) - except _DataLoss: - self.response._bodyDataFinished( - Failure(ResponseFailed([reason, Failure()], - self.response))) - else: - self.response._bodyDataFinished() - except: - # Handle exceptions from both the except suites and the else - # suite. Those functions really shouldn't raise exceptions, - # but maybe there's some buggy application code somewhere - # making things difficult. - log.err() - elif self.state != DONE: - if self._everReceivedData: - exceptionClass = ResponseFailed - else: - exceptionClass = ResponseNeverReceived - self._responseDeferred.errback(Failure(exceptionClass([reason]))) - del self._responseDeferred - - - -class Request: - """ - A L{Request} instance describes an HTTP request to be sent to an HTTP - server. - - @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD', - 'POST', etc. - @type method: C{str} - - @ivar uri: The relative URI of the resource to request. For example, - C{'/foo/bar?baz=quux'}. - @type uri: C{str} - - @ivar headers: Headers to be sent to the server. It is important to - note that this object does not create any implicit headers. So it - is up to the HTTP Client to add required headers such as 'Host'. - @type headers: L{twisted.web.http_headers.Headers} - - @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which - produces the content body to send to the remote HTTP server. - - @ivar persistent: Set to C{True} when you use HTTP persistent connection. - @type persistent: C{bool} - """ - def __init__(self, method, uri, headers, bodyProducer, persistent=False): - self.method = method - self.uri = uri - self.headers = headers - self.bodyProducer = bodyProducer - self.persistent = persistent - - - def _writeHeaders(self, transport, TEorCL): - hosts = self.headers.getRawHeaders('host', ()) - if len(hosts) != 1: - raise BadHeaders("Exactly one Host header required") - - # In the future, having the protocol version be a parameter to this - # method would probably be good. It would be nice if this method - # weren't limited to issuing HTTP/1.1 requests. - requestLines = [] - requestLines.append( - '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) - if not self.persistent: - requestLines.append('Connection: close\r\n') - if TEorCL is not None: - requestLines.append(TEorCL) - for name, values in self.headers.getAllRawHeaders(): - requestLines.extend(['%s: %s\r\n' % (name, v) for v in values]) - requestLines.append('\r\n') - transport.writeSequence(requestLines) - - - def _writeToChunked(self, transport): - """ - Write this request to the given transport using chunked - transfer-encoding to frame the body. - """ - self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n') - encoder = ChunkedEncoder(transport) - encoder.registerProducer(self.bodyProducer, True) - d = self.bodyProducer.startProducing(encoder) - - def cbProduced(ignored): - encoder.unregisterProducer() - def ebProduced(err): - encoder._allowNoMoreWrites() - # Don't call the encoder's unregisterProducer because it will write - # a zero-length chunk. This would indicate to the server that the - # request body is complete. There was an error, though, so we - # don't want to do that. - transport.unregisterProducer() - return err - d.addCallbacks(cbProduced, ebProduced) - return d - - - def _writeToContentLength(self, transport): - """ - Write this request to the given transport using content-length to frame - the body. - """ - self._writeHeaders( - transport, - 'Content-Length: %d\r\n' % (self.bodyProducer.length,)) - - # This Deferred is used to signal an error in the data written to the - # encoder below. It can only errback and it will only do so before too - # many bytes have been written to the encoder and before the producer - # Deferred fires. - finishedConsuming = Deferred() - - # This makes sure the producer writes the correct number of bytes for - # the request body. - encoder = LengthEnforcingConsumer( - self.bodyProducer, transport, finishedConsuming) - - transport.registerProducer(self.bodyProducer, True) - - finishedProducing = self.bodyProducer.startProducing(encoder) - - def combine(consuming, producing): - # This Deferred is returned and will be fired when the first of - # consuming or producing fires. If it's cancelled, forward that - # cancellation to the producer. - def cancelConsuming(ign): - finishedProducing.cancel() - ultimate = Deferred(cancelConsuming) - - # Keep track of what has happened so far. This initially - # contains None, then an integer uniquely identifying what - # sequence of events happened. See the callbacks and errbacks - # defined below for the meaning of each value. - state = [None] - - def ebConsuming(err): - if state == [None]: - # The consuming Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. The - # producing Deferred should not fire later (because the - # consumer should have called stopProducing on the - # producer), but if it does, a callback will be ignored - # and an errback will be logged. - state[0] = 1 - ultimate.errback(err) - else: - # The consuming Deferred errbacked after the producing - # Deferred fired. This really shouldn't ever happen. - # If it does, I goofed. Log the error anyway, just so - # there's a chance someone might notice and complain. - log.err( - err, - "Buggy state machine in %r/[%d]: " - "ebConsuming called" % (self, state[0])) - - def cbProducing(result): - if state == [None]: - # The producing Deferred succeeded first. Nothing will - # ever happen to the consuming Deferred. Tell the - # encoder we're done so it can check what the producer - # wrote and make sure it was right. - state[0] = 2 - try: - encoder._noMoreWritesExpected() - except: - # Fail the overall writeTo Deferred - something the - # producer did was wrong. - ultimate.errback() - else: - # Success - succeed the overall writeTo Deferred. - ultimate.callback(None) - # Otherwise, the consuming Deferred already errbacked. The - # producing Deferred wasn't supposed to fire, but it did - # anyway. It's buggy, but there's not really anything to be - # done about it. Just ignore this result. - - def ebProducing(err): - if state == [None]: - # The producing Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. - # Tell the encoder that we're done so it knows to reject - # further writes from the producer (which should not - # happen, but the producer may be buggy). - state[0] = 3 - encoder._allowNoMoreWrites() - ultimate.errback(err) - else: - # The producing Deferred failed after the consuming - # Deferred failed. It shouldn't have, so it's buggy. - # Log the exception in case anyone who can fix the code - # is watching. - log.err(err, "Producer is buggy") - - consuming.addErrback(ebConsuming) - producing.addCallbacks(cbProducing, ebProducing) - - return ultimate - - d = combine(finishedConsuming, finishedProducing) - def f(passthrough): - # Regardless of what happens with the overall Deferred, once it - # fires, the producer registered way up above the definition of - # combine should be unregistered. - transport.unregisterProducer() - return passthrough - d.addBoth(f) - return d - - - def writeTo(self, transport): - """ - Format this L{Request} as an HTTP/1.1 request and write it to the given - transport. If bodyProducer is not None, it will be associated with an - L{IConsumer}. - - @return: A L{Deferred} which fires with C{None} when the request has - been completely written to the transport or with a L{Failure} if - there is any problem generating the request bytes. - """ - if self.bodyProducer is not None: - if self.bodyProducer.length is UNKNOWN_LENGTH: - return self._writeToChunked(transport) - else: - return self._writeToContentLength(transport) - else: - self._writeHeaders(transport, None) - return succeed(None) - - - def stopWriting(self): - """ - Stop writing this request to the transport. This can only be called - after C{writeTo} and before the L{Deferred} returned by C{writeTo} - fires. It should cancel any asynchronous task started by C{writeTo}. - The L{Deferred} returned by C{writeTo} need not be fired if this method - is called. - """ - # If bodyProducer is None, then the Deferred returned by writeTo has - # fired already and this method cannot be called. - _callAppFunction(self.bodyProducer.stopProducing) - - - -class LengthEnforcingConsumer: - """ - An L{IConsumer} proxy which enforces an exact length requirement on the - total data written to it. - - @ivar _length: The number of bytes remaining to be written. - - @ivar _producer: The L{IBodyProducer} which is writing to this - consumer. - - @ivar _consumer: The consumer to which at most C{_length} bytes will be - forwarded. - - @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too - many bytes are written to this consumer. - """ - def __init__(self, producer, consumer, finished): - self._length = producer.length - self._producer = producer - self._consumer = consumer - self._finished = finished - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self._finished = None - - - def write(self, bytes): - """ - Write C{bytes} to the underlying consumer unless - C{_noMoreWritesExpected} has been called or there are/have been too - many bytes. - """ - if self._finished is None: - # No writes are supposed to happen any more. Try to convince the - # calling code to stop calling this method by calling its - # stopProducing method and then throwing an exception at it. This - # exception isn't documented as part of the API because you're - # never supposed to expect it: only buggy code will ever receive - # it. - self._producer.stopProducing() - raise ExcessWrite() - - if len(bytes) <= self._length: - self._length -= len(bytes) - self._consumer.write(bytes) - else: - # No synchronous exception is raised in *this* error path because - # we still have _finished which we can use to report the error to a - # better place than the direct caller of this method (some - # arbitrary application code). - _callAppFunction(self._producer.stopProducing) - self._finished.errback(WrongBodyLength("too many bytes written")) - self._allowNoMoreWrites() - - - def _noMoreWritesExpected(self): - """ - Called to indicate no more bytes will be written to this consumer. - Check to see that the correct number have been written. - - @raise WrongBodyLength: If not enough bytes have been written. - """ - if self._finished is not None: - self._allowNoMoreWrites() - if self._length: - raise WrongBodyLength("too few bytes written") - - - -def makeStatefulDispatcher(name, template): - """ - Given a I{dispatch} name and a function, return a function which can be - used as a method and which, when called, will call another method defined - on the instance and return the result. The other method which is called is - determined by the value of the C{_state} attribute of the instance. - - @param name: A string which is used to construct the name of the subsidiary - method to invoke. The subsidiary method is named like C{'_%s_%s' % - (name, _state)}. - - @param template: A function object which is used to give the returned - function a docstring. - - @return: The dispatcher function. - """ - def dispatcher(self, *args, **kwargs): - func = getattr(self, '_' + name + '_' + self._state, None) - if func is None: - raise RuntimeError( - "%r has no %s method in state %s" % (self, name, self._state)) - return func(*args, **kwargs) - dispatcher.__doc__ = template.__doc__ - return dispatcher - - - -class Response: - """ - A L{Response} instance describes an HTTP response received from an HTTP - server. - - L{Response} should not be subclassed or instantiated. - - @ivar _transport: The transport which is delivering this response. - - @ivar _bodyProtocol: The L{IProtocol} provider to which the body is - delivered. C{None} before one has been registered with - C{deliverBody}. - - @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived} - before C{deliverBody} is called. C{None} afterwards. - - @ivar _state: Indicates what state this L{Response} instance is in, - particularly with respect to delivering bytes from the response body - to an application-suppled protocol object. This may be one of - C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'}, - with the following meanings: - - - INITIAL: This is the state L{Response} objects start in. No - protocol has yet been provided and the underlying transport may - still have bytes to deliver to it. - - - DEFERRED_CLOSE: If the underlying transport indicates all bytes - have been delivered but no application-provided protocol is yet - available, the L{Response} moves to this state. Data is - buffered and waiting for a protocol to be delivered to. - - - CONNECTED: If a protocol is provided when the state is INITIAL, - the L{Response} moves to this state. Any buffered data is - delivered and any data which arrives from the transport - subsequently is given directly to the protocol. - - - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state, - the L{Response} moves to this state after delivering all - buffered data to the protocol. Otherwise, if the L{Response} is - in the CONNECTED state, if the transport indicates there is no - more data, the L{Response} moves to this state. Nothing else - can happen once the L{Response} is in this state. - """ - implements(IResponse) - - length = UNKNOWN_LENGTH - - _bodyProtocol = None - _bodyFinished = False - - def __init__(self, version, code, phrase, headers, _transport): - self.version = version - self.code = code - self.phrase = phrase - self.headers = headers - self._transport = _transport - self._bodyBuffer = [] - self._state = 'INITIAL' - - - def deliverBody(self, protocol): - """ - Dispatch the given L{IProtocol} depending of the current state of the - response. - """ - deliverBody = makeStatefulDispatcher('deliverBody', deliverBody) - - - def _deliverBody_INITIAL(self, protocol): - """ - Deliver any buffered data to C{protocol} and prepare to deliver any - future data to it. Move to the C{'CONNECTED'} state. - """ - # Now that there's a protocol to consume the body, resume the - # transport. It was previously paused by HTTPClientParser to avoid - # reading too much data before it could be handled. - self._transport.resumeProducing() - - protocol.makeConnection(self._transport) - self._bodyProtocol = protocol - for data in self._bodyBuffer: - self._bodyProtocol.dataReceived(data) - self._bodyBuffer = None - self._state = 'CONNECTED' - - - def _deliverBody_CONNECTED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol when it is - already being delivered to another protocol. - """ - raise RuntimeError( - "Response already has protocol %r, cannot deliverBody " - "again" % (self._bodyProtocol,)) - - - def _deliverBody_DEFERRED_CLOSE(self, protocol): - """ - Deliver any buffered data to C{protocol} and then disconnect the - protocol. Move to the C{'FINISHED'} state. - """ - # Unlike _deliverBody_INITIAL, there is no need to resume the - # transport here because all of the response data has been received - # already. Some higher level code may want to resume the transport if - # that code expects further data to be received over it. - - protocol.makeConnection(self._transport) - - for data in self._bodyBuffer: - protocol.dataReceived(data) - self._bodyBuffer = None - protocol.connectionLost(self._reason) - self._state = 'FINISHED' - - - def _deliverBody_FINISHED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol after the - response body has been delivered to another protocol. - """ - raise RuntimeError( - "Response already finished, cannot deliverBody now.") - - - def _bodyDataReceived(self, data): - """ - Called by HTTPClientParser with chunks of data from the response body. - They will be buffered or delivered to the protocol passed to - deliverBody. - """ - _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived', - _bodyDataReceived) - - - def _bodyDataReceived_INITIAL(self, data): - """ - Buffer any data received for later delivery to a protocol passed to - C{deliverBody}. - - Little or no data should be buffered by this method, since the - transport has been paused and will not be resumed until a protocol - is supplied. - """ - self._bodyBuffer.append(data) - - - def _bodyDataReceived_CONNECTED(self, data): - """ - Deliver any data received to the protocol to which this L{Response} - is connected. - """ - self._bodyProtocol.dataReceived(data) - - - def _bodyDataReceived_DEFERRED_CLOSE(self, data): - """ - It is invalid for data to be delivered after it has been indicated - that the response body has been completely delivered. - """ - raise RuntimeError("Cannot receive body data after _bodyDataFinished") - - - def _bodyDataReceived_FINISHED(self, data): - """ - It is invalid for data to be delivered after the response body has - been delivered to a protocol. - """ - raise RuntimeError("Cannot receive body data after protocol disconnected") - - - def _bodyDataFinished(self, reason=None): - """ - Called by HTTPClientParser when no more body data is available. If the - optional reason is supplied, this indicates a problem or potential - problem receiving all of the response body. - """ - _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished', - _bodyDataFinished) - - - def _bodyDataFinished_INITIAL(self, reason=None): - """ - Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to - which to deliver the response body. - """ - self._state = 'DEFERRED_CLOSE' - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._reason = reason - - - def _bodyDataFinished_CONNECTED(self, reason=None): - """ - Disconnect the protocol and move to the C{'FINISHED'} state. - """ - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._bodyProtocol.connectionLost(reason) - self._bodyProtocol = None - self._state = 'FINISHED' - - - def _bodyDataFinished_DEFERRED_CLOSE(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data more than once") - - - def _bodyDataFinished_FINISHED(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data after protocol disconnected") - - - -class ChunkedEncoder: - """ - Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol} - for streaming request bodies to the server. - """ - implements(IConsumer) - - def __init__(self, transport): - self.transport = transport - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self.transport = None - - - def registerProducer(self, producer, streaming): - """ - Register the given producer with C{self.transport}. - """ - self.transport.registerProducer(producer, streaming) - - - def write(self, data): - """ - Write the given request body bytes to the transport using chunked - encoding. - - @type data: C{str} - """ - if self.transport is None: - raise ExcessWrite() - self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n")) - - - def unregisterProducer(self): - """ - Indicate that the request body is complete and finish the request. - """ - self.write('') - self.transport.unregisterProducer() - self._allowNoMoreWrites() - - - -class TransportProxyProducer: - """ - An L{IPushProducer} implementation which wraps another such thing and - proxies calls to it until it is told to stop. - - @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after - this proxy has been stopped. - """ - implements(IPushProducer) - - # LineReceiver uses this undocumented attribute of transports to decide - # when to stop calling lineReceived or rawDataReceived (if it finds it to - # be true, it doesn't bother to deliver any more data). Set disconnecting - # to False here and never change it to true so that all data is always - # delivered to us and so that LineReceiver doesn't fail with an - # AttributeError. - disconnecting = False - - def __init__(self, producer): - self._producer = producer - - - def _stopProxying(self): - """ - Stop forwarding calls of L{IPushProducer} methods to the underlying - L{IPushProvider} provider. - """ - self._producer = None - - - def stopProducing(self): - """ - Proxy the stoppage to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.stopProducing() - - - def resumeProducing(self): - """ - Proxy the resumption to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.resumeProducing() - - - def pauseProducing(self): - """ - Proxy the pause to the underlying producer, unless this proxy has been - stopped. - """ - if self._producer is not None: - self._producer.pauseProducing() - - - -class HTTP11ClientProtocol(Protocol): - """ - L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client - protocol. It supports as few features as possible. - - @ivar _parser: After a request is issued, the L{HTTPClientParser} to - which received data making up the response to that request is - delivered. - - @ivar _finishedRequest: After a request is issued, the L{Deferred} which - will fire when a L{Response} object corresponding to that request is - available. This allows L{HTTP11ClientProtocol} to fail the request - if there is a connection or parsing problem. - - @ivar _currentRequest: After a request is issued, the L{Request} - instance used to make that request. This allows - L{HTTP11ClientProtocol} to stop request generation if necessary (for - example, if the connection is lost). - - @ivar _transportProxy: After a request is issued, the - L{TransportProxyProducer} to which C{_parser} is connected. This - allows C{_parser} to pause and resume the transport in a way which - L{HTTP11ClientProtocol} can exert some control over. - - @ivar _responseDeferred: After a request is issued, the L{Deferred} from - C{_parser} which will fire with a L{Response} when one has been - received. This is eventually chained with C{_finishedRequest}, but - only in certain cases to avoid double firing that Deferred. - - @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance - is in with respect to transmission of a request and reception of a - response. This may be one of the following strings: - - - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances - start in. Nothing is happening: no request is being sent and no - response is being received or expected. - - - TRANSMITTING: When a request is made (via L{request}), the - instance moves to this state. L{Request.writeTo} has been used - to start to send a request but it has not yet finished. - - - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a - complete response but the request has not yet been fully sent - yet. The instance will remain in this state until the request - is fully sent. - - - GENERATION_FAILED: There was an error while the request. The - request was not fully sent to the network. - - - WAITING: The request was fully sent to the network. The - instance is now waiting for the response to be fully received. - - - ABORTING: Application code has requested that the HTTP connection - be aborted. - - - CONNECTION_LOST: The connection has been lost. - - @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when - the connection is lost. - """ - _state = 'QUIESCENT' - _parser = None - _finishedRequest = None - _currentRequest = None - _transportProxy = None - _responseDeferred = None - - - def __init__(self, quiescentCallback=lambda c: None): - self._quiescentCallback = quiescentCallback - self._abortDeferreds = [] - - - @property - def state(self): - return self._state - - - def request(self, request): - """ - Issue C{request} over C{self.transport} and return a L{Deferred} which - will fire with a L{Response} instance or an error. - - @param request: The object defining the parameters of the request to - issue. - @type request: L{Request} - - @rtype: L{Deferred} - @return: The deferred may errback with L{RequestGenerationFailed} if - the request was not fully written to the transport due to a local - error. It may errback with L{RequestTransmissionFailed} if it was - not fully written to the transport due to a network error. It may - errback with L{ResponseFailed} if the request was sent (not - necessarily received) but some or all of the response was lost. It - may errback with L{RequestNotSent} if it is not possible to send - any more requests using this L{HTTP11ClientProtocol}. - """ - if self._state != 'QUIESCENT': - return fail(RequestNotSent()) - - self._state = 'TRANSMITTING' - _requestDeferred = maybeDeferred(request.writeTo, self.transport) - - def cancelRequest(ign): - # Explicitly cancel the request's deferred if it's still trying to - # write when this request is cancelled. - if self._state in ( - 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'): - _requestDeferred.cancel() - else: - self.transport.abortConnection() - self._disconnectParser(Failure(CancelledError())) - self._finishedRequest = Deferred(cancelRequest) - - # Keep track of the Request object in case we need to call stopWriting - # on it. - self._currentRequest = request - - self._transportProxy = TransportProxyProducer(self.transport) - self._parser = HTTPClientParser(request, self._finishResponse) - self._parser.makeConnection(self._transportProxy) - self._responseDeferred = self._parser._responseDeferred - - def cbRequestWrotten(ignored): - if self._state == 'TRANSMITTING': - self._state = 'WAITING' - self._responseDeferred.chainDeferred(self._finishedRequest) - - def ebRequestWriting(err): - if self._state == 'TRANSMITTING': - self._state = 'GENERATION_FAILED' - self.transport.abortConnection() - self._finishedRequest.errback( - Failure(RequestGenerationFailed([err]))) - else: - log.err(err, 'Error writing request, but not in valid state ' - 'to finalize request: %s' % self._state) - - _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) - - return self._finishedRequest - - - def _finishResponse(self, rest): - """ - Called by an L{HTTPClientParser} to indicate that it has parsed a - complete response. - - @param rest: A C{str} giving any trailing bytes which were given to - the L{HTTPClientParser} which were not part of the response it - was parsing. - """ - _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse) - - - def _finishResponse_WAITING(self, rest): - # Currently the rest parameter is ignored. Don't forget to use it if - # we ever add support for pipelining. And maybe check what trailers - # mean. - if self._state == 'WAITING': - self._state = 'QUIESCENT' - else: - # The server sent the entire response before we could send the - # whole request. That sucks. Oh well. Fire the request() - # Deferred with the response. But first, make sure that if the - # request does ever finish being written that it won't try to fire - # that Deferred. - self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' - self._responseDeferred.chainDeferred(self._finishedRequest) - - # This will happen if we're being called due to connection being lost; - # if so, no need to disconnect parser again, or to call - # _quiescentCallback. - if self._parser is None: - return - - reason = ConnectionDone("synthetic!") - connHeaders = self._parser.connHeaders.getRawHeaders('connection', ()) - if (('close' in connHeaders) or self._state != "QUIESCENT" or - not self._currentRequest.persistent): - self._giveUp(Failure(reason)) - else: - # We call the quiescent callback first, to ensure connection gets - # added back to connection pool before we finish the request. - try: - self._quiescentCallback(self) - except: - # If callback throws exception, just log it and disconnect; - # keeping persistent connections around is an optimisation: - log.err() - self.transport.loseConnection() - self._disconnectParser(reason) - - - _finishResponse_TRANSMITTING = _finishResponse_WAITING - - - def _disconnectParser(self, reason): - """ - If there is still a parser, call its C{connectionLost} method with the - given reason. If there is not, do nothing. - - @type reason: L{Failure} - """ - if self._parser is not None: - parser = self._parser - self._parser = None - self._currentRequest = None - self._finishedRequest = None - self._responseDeferred = None - - # The parser is no longer allowed to do anything to the real - # transport. Stop proxying from the parser's transport to the real - # transport before telling the parser it's done so that it can't do - # anything. - self._transportProxy._stopProxying() - self._transportProxy = None - parser.connectionLost(reason) - - - def _giveUp(self, reason): - """ - Lose the underlying connection and disconnect the parser with the given - L{Failure}. - - Use this method instead of calling the transport's loseConnection - method directly otherwise random things will break. - """ - self.transport.loseConnection() - self._disconnectParser(reason) - - - def dataReceived(self, bytes): - """ - Handle some stuff from some place. - """ - try: - self._parser.dataReceived(bytes) - except: - self._giveUp(Failure()) - - - def connectionLost(self, reason): - """ - The underlying transport went away. If appropriate, notify the parser - object. - """ - connectionLost = makeStatefulDispatcher('connectionLost', connectionLost) - - - def _connectionLost_QUIESCENT(self, reason): - """ - Nothing is currently happening. Move to the C{'CONNECTION_LOST'} - state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_GENERATION_FAILED(self, reason): - """ - The connection was in an inconsistent state. Move to the - C{'CONNECTION_LOST'} state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_TRANSMITTING(self, reason): - """ - Fail the L{Deferred} for the current request, notify the request - object that it does not need to continue transmitting itself, and - move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - self._finishedRequest.errback( - Failure(RequestTransmissionFailed([reason]))) - del self._finishedRequest - - # Tell the request that it should stop bothering now. - self._currentRequest.stopWriting() - - - def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason): - """ - Move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_WAITING(self, reason): - """ - Disconnect the response parser so that it can propagate the event as - necessary (for example, to call an application protocol's - C{connectionLost} method, or to fail a request L{Deferred}) and move - to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(reason) - self._state = 'CONNECTION_LOST' - - - def _connectionLost_ABORTING(self, reason): - """ - Disconnect the response parser with a L{ConnectionAborted} failure, and - move to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(Failure(ConnectionAborted())) - self._state = 'CONNECTION_LOST' - for d in self._abortDeferreds: - d.callback(None) - self._abortDeferreds = [] - - - def abort(self): - """ - Close the connection and cause all outstanding L{request} L{Deferred}s - to fire with an error. - """ - if self._state == "CONNECTION_LOST": - return succeed(None) - self.transport.loseConnection() - self._state = 'ABORTING' - d = Deferred() - self._abortDeferreds.append(d) - return d -}}} ''' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index 8e0b1df8b..c2d50648a 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -42,1129 +42,13 @@ from twisted.web.client import ( Agent, ProxyAgent, HTTPConnectionPool, readBody, ) -''' {{{ -class PartialDownloadError(error.Error): - """ - Page was only partially downloaded, we got disconnected in middle. - - @ivar response: All of the response body which was downloaded. - """ - - -class _URL(tuple): - """ - A parsed URL. - - At some point this should be replaced with a better URL implementation. - """ - def __new__(self, scheme, host, port, path): - return tuple.__new__(_URL, (scheme, host, port, path)) - - - def __init__(self, scheme, host, port, path): - self.scheme = scheme - self.host = host - self.port = port - self.path = path - - -def _parse(url, defaultPort=None): - """ - Split the given URL into the scheme, host, port, and path. - - @type url: C{bytes} - @param url: An URL to parse. - - @type defaultPort: C{int} or C{None} - @param defaultPort: An alternate value to use as the port if the URL does - not include one. - - @return: A four-tuple of the scheme, host, port, and path of the URL. All - of these are C{bytes} instances except for port, which is an C{int}. - """ - url = url.strip() - parsed = http.urlparse(url) - scheme = parsed[0] - path = urlunparse((b'', b'') + parsed[2:]) - - if defaultPort is None: - if scheme == b'https': - defaultPort = 443 - else: - defaultPort = 80 - - host, port = parsed[1], defaultPort - if b':' in host: - host, port = host.split(b':') - try: - port = int(port) - except ValueError: - port = defaultPort - - if path == b'': - path = b'/' - - return _URL(scheme, host, port, path) - - -def _makeGetterFactory(url, factoryFactory, contextFactory=None, - *args, **kwargs): - """ - Create and connect an HTTP page getting factory. - - Any additional positional or keyword arguments are used when calling - C{factoryFactory}. - - @param factoryFactory: Factory factory that is called with C{url}, C{args} - and C{kwargs} to produce the getter - - @param contextFactory: Context factory to use when creating a secure - connection, defaulting to C{None} - - @return: The factory created by C{factoryFactory} - """ - scheme, host, port, path = _parse(url) - factory = factoryFactory(url, *args, **kwargs) - if scheme == b'https': - from twisted.internet import ssl - if contextFactory is None: - contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(host, port, factory, contextFactory) - else: - reactor.connectTCP(host, port, factory) - return factory -}}} ''' # The code which follows is based on the new HTTP client implementation. It # should be significantly better than anything above, though it is not yet # feature equivalent. -#from twisted.web.error import SchemeNotSupported from twisted.web._newclient import Response -#from twisted.web._newclient import Request, HTTP11ClientProtocol from twisted.web._newclient import ResponseDone, ResponseFailed -#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed -#from twisted.web._newclient import ( -# ResponseNeverReceived, PotentialDataLoss, _WrapperException) - -''' {{{ -try: - from twisted.internet.ssl import ClientContextFactory -except ImportError: - class WebClientContextFactory(object): - """ - A web context factory which doesn't work because the necessary SSL - support is missing. - """ - def getContext(self, hostname, port): - raise NotImplementedError("SSL support unavailable") -else: - class WebClientContextFactory(ClientContextFactory): - """ - A web context factory which ignores the hostname and port and does no - certificate verification. - """ - def getContext(self, hostname, port): - return ClientContextFactory.getContext(self) - - - -class _WebToNormalContextFactory(object): - """ - Adapt a web context factory to a normal context factory. - - @ivar _webContext: A web context factory which accepts a hostname and port - number to its C{getContext} method. - - @ivar _hostname: The hostname which will be passed to - C{_webContext.getContext}. - - @ivar _port: The port number which will be passed to - C{_webContext.getContext}. - """ - def __init__(self, webContext, hostname, port): - self._webContext = webContext - self._hostname = hostname - self._port = port - - - def getContext(self): - """ - Called the wrapped web context factory's C{getContext} method with a - hostname and port number and return the resulting context object. - """ - return self._webContext.getContext(self._hostname, self._port) - - - -@implementer(IBodyProducer) -class FileBodyProducer(object): - """ - L{FileBodyProducer} produces bytes from an input file object incrementally - and writes them to a consumer. - - Since file-like objects cannot be read from in an event-driven manner, - L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from - the file. This process is also paused and resumed based on notifications - from the L{IConsumer} provider being written to. - - The file is closed after it has been read, or if the producer is stopped - early. - - @ivar _inputFile: Any file-like object, bytes read from which will be - written to a consumer. - - @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to - schedule all reads. - - @ivar _readSize: The number of bytes to read from C{_inputFile} at a time. - """ - - # Python 2.4 doesn't have these symbolic constants - _SEEK_SET = getattr(os, 'SEEK_SET', 0) - _SEEK_END = getattr(os, 'SEEK_END', 2) - - def __init__(self, inputFile, cooperator=task, readSize=2 ** 16): - self._inputFile = inputFile - self._cooperate = cooperator.cooperate - self._readSize = readSize - self.length = self._determineLength(inputFile) - - - def _determineLength(self, fObj): - """ - Determine how many bytes can be read out of C{fObj} (assuming it is not - modified from this point on). If the determination cannot be made, - return C{UNKNOWN_LENGTH}. - """ - try: - seek = fObj.seek - tell = fObj.tell - except AttributeError: - return UNKNOWN_LENGTH - originalPosition = tell() - seek(0, self._SEEK_END) - end = tell() - seek(originalPosition, self._SEEK_SET) - return end - originalPosition - - - def stopProducing(self): - """ - Permanently stop writing bytes from the file to the consumer by - stopping the underlying L{CooperativeTask}. - """ - self._inputFile.close() - self._task.stop() - - - def startProducing(self, consumer): - """ - Start a cooperative task which will read bytes from the input file and - write them to C{consumer}. Return a L{Deferred} which fires after all - bytes have been written. - - @param consumer: Any L{IConsumer} provider - """ - self._task = self._cooperate(self._writeloop(consumer)) - d = self._task.whenDone() - def maybeStopped(reason): - # IBodyProducer.startProducing's Deferred isn't support to fire if - # stopProducing is called. - reason.trap(task.TaskStopped) - return defer.Deferred() - d.addCallbacks(lambda ignored: None, maybeStopped) - return d - - - def _writeloop(self, consumer): - """ - Return an iterator which reads one chunk of bytes from the input file - and writes them to the consumer for each time it is iterated. - """ - while True: - bytes = self._inputFile.read(self._readSize) - if not bytes: - self._inputFile.close() - break - consumer.write(bytes) - yield None - - - def pauseProducing(self): - """ - Temporarily suspend copying bytes from the input file to the consumer - by pausing the L{CooperativeTask} which drives that activity. - """ - self._task.pause() - - - def resumeProducing(self): - """ - Undo the effects of a previous C{pauseProducing} and resume copying - bytes to the consumer by resuming the L{CooperativeTask} which drives - the write activity. - """ - self._task.resume() - - - -class _HTTP11ClientFactory(protocol.Factory): - """ - A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}. - - @ivar _quiescentCallback: The quiescent callback to be passed to protocol - instances, used to return them to the connection pool. - - @since: 11.1 - """ - def __init__(self, quiescentCallback): - self._quiescentCallback = quiescentCallback - - - def buildProtocol(self, addr): - return HTTP11ClientProtocol(self._quiescentCallback) - - - -class _RetryingHTTP11ClientProtocol(object): - """ - A wrapper for L{HTTP11ClientProtocol} that automatically retries requests. - - @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}. - - @ivar _newConnection: A callable that creates a new connection for a - retry. - """ - - def __init__(self, clientProtocol, newConnection): - self._clientProtocol = clientProtocol - self._newConnection = newConnection - - - def _shouldRetry(self, method, exception, bodyProducer): - """ - Indicate whether request should be retried. - - Only returns C{True} if method is idempotent, no response was - received, the reason for the failed request was not due to - user-requested cancellation, and no body was sent. The latter - requirement may be relaxed in the future, and PUT added to approved - method list. - """ - if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"): - return False - if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed, - ResponseNeverReceived)): - return False - if isinstance(exception, _WrapperException): - for failure in exception.reasons: - if failure.check(defer.CancelledError): - return False - if bodyProducer is not None: - return False - return True - - - def request(self, request): - """ - Do a request, and retry once (with a new connection) it it fails in - a retryable manner. - - @param request: A L{Request} instance that will be requested using the - wrapped protocol. - """ - d = self._clientProtocol.request(request) - - def failed(reason): - if self._shouldRetry(request.method, reason.value, - request.bodyProducer): - return self._newConnection().addCallback( - lambda connection: connection.request(request)) - else: - return reason - d.addErrback(failed) - return d - - - -class HTTPConnectionPool(object): - """ - A pool of persistent HTTP connections. - - Features: - - Cached connections will eventually time out. - - Limits on maximum number of persistent connections. - - Connections are stored using keys, which should be chosen such that any - connections stored under a given key can be used interchangeably. - - Failed requests done using previously cached connections will be retried - once if they use an idempotent method (e.g. GET), in case the HTTP server - timed them out. - - @ivar persistent: Boolean indicating whether connections should be - persistent. Connections are persistent by default. - - @ivar maxPersistentPerHost: The maximum number of cached persistent - connections for a C{host:port} destination. - @type maxPersistentPerHost: C{int} - - @ivar cachedConnectionTimeout: Number of seconds a cached persistent - connection will stay open before disconnecting. - - @ivar retryAutomatically: C{boolean} indicating whether idempotent - requests should be retried once if no response was received. - - @ivar _factory: The factory used to connect to the proxy. - - @ivar _connections: Map (scheme, host, port) to lists of - L{HTTP11ClientProtocol} instances. - - @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a - C{IDelayedCall} instance of their timeout. - - @since: 12.1 - """ - - _factory = _HTTP11ClientFactory - maxPersistentPerHost = 2 - cachedConnectionTimeout = 240 - retryAutomatically = True - - def __init__(self, reactor, persistent=True): - self._reactor = reactor - self.persistent = persistent - self._connections = {} - self._timeouts = {} - - - def getConnection(self, key, endpoint): - """ - Supply a connection, newly created or retrieved from the pool, to be - used for one HTTP request. - - The connection will remain out of the pool (not available to be - returned from future calls to this method) until one HTTP request has - been completed over it. - - Afterwards, if the connection is still open, it will automatically be - added to the pool. - - @param key: A unique key identifying connections that can be used - interchangeably. - - @param endpoint: An endpoint that can be used to open a new connection - if no cached connection is available. - - @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol} - (or a wrapper) that can be used to send a single HTTP request. - """ - # Try to get cached version: - connections = self._connections.get(key) - while connections: - connection = connections.pop(0) - # Cancel timeout: - self._timeouts[connection].cancel() - del self._timeouts[connection] - if connection.state == "QUIESCENT": - if self.retryAutomatically: - newConnection = lambda: self._newConnection(key, endpoint) - connection = _RetryingHTTP11ClientProtocol( - connection, newConnection) - return defer.succeed(connection) - - return self._newConnection(key, endpoint) - - - def _newConnection(self, key, endpoint): - """ - Create a new connection. - - This implements the new connection code path for L{getConnection}. - """ - def quiescentCallback(protocol): - self._putConnection(key, protocol) - factory = self._factory(quiescentCallback) - return endpoint.connect(factory) - - - def _removeConnection(self, key, connection): - """ - Remove a connection from the cache and disconnect it. - """ - connection.transport.loseConnection() - self._connections[key].remove(connection) - del self._timeouts[connection] - - - def _putConnection(self, key, connection): - """ - Return a persistent connection to the pool. This will be called by - L{HTTP11ClientProtocol} when the connection becomes quiescent. - """ - if connection.state != "QUIESCENT": - # Log with traceback for debugging purposes: - try: - raise RuntimeError( - "BUG: Non-quiescent protocol added to connection pool.") - except: - log.err() - return - connections = self._connections.setdefault(key, []) - if len(connections) == self.maxPersistentPerHost: - dropped = connections.pop(0) - dropped.transport.loseConnection() - self._timeouts[dropped].cancel() - del self._timeouts[dropped] - connections.append(connection) - cid = self._reactor.callLater(self.cachedConnectionTimeout, - self._removeConnection, - key, connection) - self._timeouts[connection] = cid - - - def closeCachedConnections(self): - """ - Close all persistent connections and remove them from the pool. - - @return: L{defer.Deferred} that fires when all connections have been - closed. - """ - results = [] - for protocols in self._connections.itervalues(): - for p in protocols: - results.append(p.abort()) - self._connections = {} - for dc in self._timeouts.values(): - dc.cancel() - self._timeouts = {} - return defer.gatherResults(results).addCallback(lambda ign: None) - - - -class _AgentBase(object): - """ - Base class offering common facilities for L{Agent}-type classes. - - @ivar _reactor: The C{IReactorTime} implementation which will be used by - the pool, and perhaps by subclasses as well. - - @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections. - """ - - def __init__(self, reactor, pool): - if pool is None: - pool = HTTPConnectionPool(reactor, False) - self._reactor = reactor - self._pool = pool - - - def _computeHostValue(self, scheme, host, port): - """ - Compute the string to use for the value of the I{Host} header, based on - the given scheme, host name, and port number. - """ - if (scheme, port) in (('http', 80), ('https', 443)): - return host - return '%s:%d' % (host, port) - - - def _requestWithEndpoint(self, key, endpoint, method, parsedURI, - headers, bodyProducer, requestPath): - """ - Issue a new request, given the endpoint and the path sent as part of - the request. - """ - # Create minimal headers, if necessary: - if headers is None: - headers = Headers() - if not headers.hasHeader('host'): - #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us - headers.addRawHeader( - 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, - parsedURI.port)) - - d = self._pool.getConnection(key, endpoint) - def cbConnected(proto): - return proto.request( - Request(method, requestPath, headers, bodyProducer, - persistent=self._pool.persistent)) - d.addCallback(cbConnected) - return d - - - -class Agent(_AgentBase): - """ - L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS} - scheme URIs (but performs no certificate checking by default). - - @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a - non-persistent L{HTTPConnectionPool} instance will be created. - - @ivar _contextFactory: A web context factory which will be used to create - SSL context objects for any SSL connections the agent needs to make. - - @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP} - or C{connectSSL} for specifying the connection timeout. - - @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or - C{connectSSL} for specifying the local address to bind to. - - @since: 9.0 - """ - - def __init__(self, reactor, contextFactory=WebClientContextFactory(), - connectTimeout=None, bindAddress=None, - pool=None): - _AgentBase.__init__(self, reactor, pool) - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - - - def _wrapContextFactory(self, host, port): - """ - Create and return a normal context factory wrapped around - C{self._contextFactory} in such a way that C{self._contextFactory} will - have the host and port information passed to it. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: A context factory suitable to be passed to - C{reactor.connectSSL}. - """ - return _WebToNormalContextFactory(self._contextFactory, host, port) - - - def _getEndpoint(self, scheme, host, port): - """ - Get an endpoint for the given host and port, using a transport - selected based on scheme. - - @param scheme: A string like C{'http'} or C{'https'} (the only two - supported values) to use to determine how to establish the - connection. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: An endpoint which can be used to connect to given address. - """ - kwargs = {} - if self._connectTimeout is not None: - kwargs['timeout'] = self._connectTimeout - kwargs['bindAddress'] = self._bindAddress - if scheme == 'http': - return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) - elif scheme == 'https': - return SSL4ClientEndpoint(self._reactor, host, port, - self._wrapContextFactory(host, port), - **kwargs) - else: - raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,)) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request. - - @param method: The request method to send. - @type method: C{str} - - @param uri: The request URI send. - @type uri: C{str} - - @param headers: The request headers to send. If no I{Host} header is - included, one will be added based on the request URI. - @type headers: L{Headers} - - @param bodyProducer: An object which will produce the request body or, - if the request body is to be empty, L{None}. - @type bodyProducer: L{IBodyProducer} provider - - @return: A L{Deferred} which fires with the result of the request (a - L{twisted.web.iweb.IResponse} provider), or fails if there is a - problem setting up a connection over which to issue the request. - It may also fail with L{SchemeNotSupported} if the scheme of the - given URI is not supported. - @rtype: L{Deferred} - """ - parsedURI = _parse(uri) - try: - endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host, - parsedURI.port) - except SchemeNotSupported: - return defer.fail(Failure()) - key = (parsedURI.scheme, parsedURI.host, parsedURI.port) - return self._requestWithEndpoint(key, endpoint, method, parsedURI, - headers, bodyProducer, parsedURI.path) - - - -class ProxyAgent(_AgentBase): - """ - An HTTP agent able to cross HTTP proxies. - - @ivar _proxyEndpoint: The endpoint used to connect to the proxy. - - @since: 11.1 - """ - - def __init__(self, endpoint, reactor=None, pool=None): - if reactor is None: - from twisted.internet import reactor - _AgentBase.__init__(self, reactor, pool) - self._proxyEndpoint = endpoint - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request via the configured proxy. - """ - # Cache *all* connections under the same key, since we are only - # connecting to a single destination, the proxy: - key = ("http-proxy", self._proxyEndpoint) - - # To support proxying HTTPS via CONNECT, we will use key - # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that - # wraps _proxyEndpoint with an additional callback to do the CONNECT. - return self._requestWithEndpoint(key, self._proxyEndpoint, method, - _parse(uri), headers, bodyProducer, - uri) - - - -class _FakeUrllib2Request(object): - """ - A fake C{urllib2.Request} object for C{cookielib} to work with. - - @see: U{http://docs.python.org/library/urllib2.html#request-objects} - - @type uri: C{str} - @ivar uri: Request URI. - - @type headers: L{twisted.web.http_headers.Headers} - @ivar headers: Request headers. - - @type type: C{str} - @ivar type: The scheme of the URI. - - @type host: C{str} - @ivar host: The host[:port] of the URI. - - @since: 11.1 - """ - def __init__(self, uri): - self.uri = uri - self.headers = Headers() - self.type, rest = splittype(self.uri) - self.host, rest = splithost(rest) - - - def has_header(self, header): - return self.headers.hasHeader(header) - - - def add_unredirected_header(self, name, value): - self.headers.addRawHeader(name, value) - - - def get_full_url(self): - return self.uri - - - def get_header(self, name, default=None): - headers = self.headers.getRawHeaders(name, default) - if headers is not None: - return headers[0] - return None - - - def get_host(self): - return self.host - - - def get_type(self): - return self.type - - - def is_unverifiable(self): - # In theory this shouldn't be hardcoded. - return False - - - -class _FakeUrllib2Response(object): - """ - A fake C{urllib2.Response} object for C{cookielib} to work with. - - @type response: C{twisted.web.iweb.IResponse} - @ivar response: Underlying Twisted Web response. - - @since: 11.1 - """ - def __init__(self, response): - self.response = response - - - def info(self): - class _Meta(object): - def getheaders(zelf, name): - return self.response.headers.getRawHeaders(name, []) - return _Meta() - - - -class CookieAgent(object): - """ - L{CookieAgent} extends the basic L{Agent} to add RFC-compliant - handling of HTTP cookies. Cookies are written to and extracted - from a C{cookielib.CookieJar} instance. - - The same cookie jar instance will be used for any requests through this - agent, mutating it whenever a I{Set-Cookie} header appears in a response. - - @type _agent: L{twisted.web.client.Agent} - @ivar _agent: Underlying Twisted Web agent to issue requests through. - - @type cookieJar: C{cookielib.CookieJar} - @ivar cookieJar: Initialized cookie jar to read cookies from and store - cookies to. - - @since: 11.1 - """ - def __init__(self, agent, cookieJar): - self._agent = agent - self.cookieJar = cookieJar - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request to the wrapped L{Agent}. - - Send a I{Cookie} header if a cookie for C{uri} is stored in - L{CookieAgent.cookieJar}. Cookies are automatically extracted and - stored from requests. - - If a C{'cookie'} header appears in C{headers} it will override the - automatic cookie header obtained from the cookie jar. - - @see: L{Agent.request} - """ - if headers is None: - headers = Headers() - lastRequest = _FakeUrllib2Request(uri) - # Setting a cookie header explicitly will disable automatic request - # cookies. - if not headers.hasHeader('cookie'): - self.cookieJar.add_cookie_header(lastRequest) - cookieHeader = lastRequest.get_header('Cookie', None) - if cookieHeader is not None: - headers = headers.copy() - headers.addRawHeader('cookie', cookieHeader) - - d = self._agent.request(method, uri, headers, bodyProducer) - d.addCallback(self._extractCookies, lastRequest) - return d - - - def _extractCookies(self, response, request): - """ - Extract response cookies and store them in the cookie jar. - - @type response: L{twisted.web.iweb.IResponse} - @param response: Twisted Web response. - - @param request: A urllib2 compatible request object. - """ - resp = _FakeUrllib2Response(response) - self.cookieJar.extract_cookies(resp, request) - return response - - - -class GzipDecoder(proxyForInterface(IResponse)): - """ - A wrapper for a L{Response} instance which handles gzip'ed body. - - @ivar original: The original L{Response} object. - - @since: 11.1 - """ - - def __init__(self, response): - self.original = response - self.length = UNKNOWN_LENGTH - - - def deliverBody(self, protocol): - """ - Override C{deliverBody} to wrap the given C{protocol} with - L{_GzipProtocol}. - """ - self.original.deliverBody(_GzipProtocol(protocol, self.original)) - - - -class _GzipProtocol(proxyForInterface(IProtocol)): - """ - A L{Protocol} implementation which wraps another one, transparently - decompressing received data. - - @ivar _zlibDecompress: A zlib decompress object used to decompress the data - stream. - - @ivar _response: A reference to the original response, in case of errors. - - @since: 11.1 - """ - - def __init__(self, protocol, response): - self.original = protocol - self._response = response - self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS) - - - def dataReceived(self, data): - """ - Decompress C{data} with the zlib decompressor, forwarding the raw data - to the original protocol. - """ - try: - rawData = self._zlibDecompress.decompress(data) - except zlib.error: - raise ResponseFailed([failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - - - def connectionLost(self, reason): - """ - Forward the connection lost event, flushing remaining data from the - decompressor if any. - """ - try: - rawData = self._zlibDecompress.flush() - except zlib.error: - raise ResponseFailed([reason, failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - self.original.connectionLost(reason) - - - -class ContentDecoderAgent(object): - """ - An L{Agent} wrapper to handle encoded content. - - It takes care of declaring the support for content in the - I{Accept-Encoding} header, and automatically decompresses the received data - if it's effectively using compression. - - @param decoders: A list or tuple of (name, decoder) objects. The name - declares which decoding the decoder supports, and the decoder must - return a response object when called/instantiated. For example, - C{(('gzip', GzipDecoder))}. The order determines how the decoders are - going to be advertized to the server. - - @since: 11.1 - """ - - def __init__(self, agent, decoders): - self._agent = agent - self._decoders = dict(decoders) - self._supported = ','.join([decoder[0] for decoder in decoders]) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request which declares supporting compressed content. - - @see: L{Agent.request}. - """ - if headers is None: - headers = Headers() - else: - headers = headers.copy() - headers.addRawHeader('accept-encoding', self._supported) - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback(self._handleResponse) - - - def _handleResponse(self, response): - """ - Check if the response is encoded, and wrap it to handle decompression. - """ - contentEncodingHeaders = response.headers.getRawHeaders( - 'content-encoding', []) - contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',') - while contentEncodingHeaders: - name = contentEncodingHeaders.pop().strip() - decoder = self._decoders.get(name) - if decoder is not None: - response = decoder(response) - else: - # Add it back - contentEncodingHeaders.append(name) - break - if contentEncodingHeaders: - response.headers.setRawHeaders( - 'content-encoding', [','.join(contentEncodingHeaders)]) - else: - response.headers.removeHeader('content-encoding') - return response - - - -class RedirectAgent(object): - """ - An L{Agent} wrapper which handles HTTP redirects. - - The implementation is rather strict: 301 and 302 behaves like 307, not - redirecting automatically on methods different from C{GET} and C{HEAD}. - - @param redirectLimit: The maximum number of times the agent is allowed to - follow redirects before failing with a L{error.InfiniteRedirection}. - - @since: 11.1 - """ - - def __init__(self, agent, redirectLimit=20): - self._agent = agent - self._redirectLimit = redirectLimit - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request following HTTP redirects. - - @see: L{Agent.request}. - """ - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback( - self._handleResponse, method, uri, headers, 0) - - - def _handleRedirect(self, response, method, uri, headers, redirectCount): - """ - Handle a redirect response, checking the number of redirects already - followed, and extracting the location header fields. - """ - if redirectCount >= self._redirectLimit: - err = error.InfiniteRedirection( - response.code, - 'Infinite redirection detected', - location=uri) - raise ResponseFailed([failure.Failure(err)], response) - locationHeaders = response.headers.getRawHeaders('location', []) - if not locationHeaders: - err = error.RedirectWithNoLocation( - response.code, 'No location header field', uri) - raise ResponseFailed([failure.Failure(err)], response) - location = locationHeaders[0] - deferred = self._agent.request(method, location, headers) - return deferred.addCallback( - self._handleResponse, method, uri, headers, redirectCount + 1) - - - def _handleResponse(self, response, method, uri, headers, redirectCount): - """ - Handle the response, making another request if it indicates a redirect. - """ - if response.code in (http.MOVED_PERMANENTLY, http.FOUND, - http.TEMPORARY_REDIRECT): - if method not in ('GET', 'HEAD'): - err = error.PageRedirect(response.code, location=uri) - raise ResponseFailed([failure.Failure(err)], response) - return self._handleRedirect(response, method, uri, headers, - redirectCount) - elif response.code == http.SEE_OTHER: - return self._handleRedirect(response, 'GET', uri, headers, - redirectCount) - return response - - - -class _ReadBodyProtocol(protocol.Protocol): - """ - Protocol that collects data sent to it. - - This is a helper for L{IResponse.deliverBody}, which collects the body and - fires a deferred with it. - - @ivar deferred: See L{__init__}. - @ivar status: See L{__init__}. - @ivar message: See L{__init__}. - - @ivar dataBuffer: list of byte-strings received - @type dataBuffer: L{list} of L{bytes} - """ - - def __init__(self, status, message, deferred): - """ - @param status: Status of L{IResponse} - @ivar status: L{int} - - @param message: Message of L{IResponse} - @type message: L{bytes} - - @param deferred: deferred to fire when response is complete - @type deferred: L{Deferred} firing with L{bytes} - """ - self.deferred = deferred - self.status = status - self.message = message - self.dataBuffer = [] - - - def dataReceived(self, data): - """ - Accumulate some more bytes from the response. - """ - self.dataBuffer.append(data) - - - def connectionLost(self, reason): - """ - Deliver the accumulated response bytes to the waiting L{Deferred}, if - the response body has been completely received without error. - """ - if reason.check(ResponseDone): - self.deferred.callback(b''.join(self.dataBuffer)) - elif reason.check(PotentialDataLoss): - self.deferred.errback( - PartialDownloadError(self.status, self.message, - b''.join(self.dataBuffer))) - else: - self.deferred.errback(reason) - - - -def readBody(response): - """ - Get the body of an L{IResponse} and return it as a byte string. - - This is a helper function for clients that don't want to incrementally - receive the body of an HTTP response. - - @param response: The HTTP response for which the body will be read. - @type response: L{IResponse} provider - - @return: A L{Deferred} which will fire with the body of the response. - """ - d = defer.Deferred() - response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) - return d -}}} ''' __all__ = [ diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index 3f4704064..197e43ed3 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -14,27 +14,6 @@ parsed by the L{clientFromString} and L{serverFromString} functions. from __future__ import division, absolute_import -import os -#import socket - -from zope.interface import implementer, directlyProvides -import warnings - -from twisted.internet import interfaces, defer, error, fdesc -#from twisted.internet.protocol import ( -# ClientFactory, Protocol) -from twisted.internet.protocol import Factory -#from twisted.internet import threads, ProcessProtocol -from twisted.internet.interfaces import IStreamServerEndpointStringParser -#from twisted.internet.interfaces import IStreamClientEndpointStringParser -from twisted.python.filepath import FilePath -#from twisted.python.failure import Failure -#from twisted.python import log -#from twisted.python.components import proxyForInterface - -from twisted.plugin import IPlugin, getPlugins -#from twisted.internet import stdio - from twisted.internet.endpoints import ( clientFromString, serverFromString, quoteStringArgument, TCP4ServerEndpoint, TCP6ServerEndpoint, @@ -44,1236 +23,4 @@ from twisted.internet.endpoints import ( AdoptedStreamServerEndpoint, connectProtocol, ) - __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - -''' {{{ -class _WrappingProtocol(Protocol): - """ - Wrap another protocol in order to notify my user when a connection has - been made. - """ - - def __init__(self, connectedDeferred, wrappedProtocol): - """ - @param connectedDeferred: The L{Deferred} that will callback - with the C{wrappedProtocol} when it is connected. - - @param wrappedProtocol: An L{IProtocol} provider that will be - connected. - """ - self._connectedDeferred = connectedDeferred - self._wrappedProtocol = wrappedProtocol - - for iface in [interfaces.IHalfCloseableProtocol, - interfaces.IFileDescriptorReceiver]: - if iface.providedBy(self._wrappedProtocol): - directlyProvides(self, iface) - - - def logPrefix(self): - """ - Transparently pass through the wrapped protocol's log prefix. - """ - if interfaces.ILoggingContext.providedBy(self._wrappedProtocol): - return self._wrappedProtocol.logPrefix() - return self._wrappedProtocol.__class__.__name__ - - - def connectionMade(self): - """ - Connect the C{self._wrappedProtocol} to our C{self.transport} and - callback C{self._connectedDeferred} with the C{self._wrappedProtocol} - """ - self._wrappedProtocol.makeConnection(self.transport) - self._connectedDeferred.callback(self._wrappedProtocol) - - - def dataReceived(self, data): - """ - Proxy C{dataReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.dataReceived(data) - - - def fileDescriptorReceived(self, descriptor): - """ - Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.fileDescriptorReceived(descriptor) - - - def connectionLost(self, reason): - """ - Proxy C{connectionLost} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.connectionLost(reason) - - - def readConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.readConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.readConnectionLost() - - - def writeConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.writeConnectionLost() - - - -class _WrappingFactory(ClientFactory): - """ - Wrap a factory in order to wrap the protocols it builds. - - @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol - method will be called and whose resulting protocol will be wrapped. - - @ivar _onConnection: A L{Deferred} that fires when the protocol is - connected - - @ivar _connector: A L{connector } - that is managing the current or previous connection attempt. - """ - protocol = _WrappingProtocol - - def __init__(self, wrappedFactory): - """ - @param wrappedFactory: A provider of I{IProtocolFactory} whose - buildProtocol method will be called and whose resulting protocol - will be wrapped. - """ - self._wrappedFactory = wrappedFactory - self._onConnection = defer.Deferred(canceller=self._canceller) - - - def startedConnecting(self, connector): - """ - A connection attempt was started. Remember the connector which started - said attempt, for use later. - """ - self._connector = connector - - - def _canceller(self, deferred): - """ - The outgoing connection attempt was cancelled. Fail that L{Deferred} - with an L{error.ConnectingCancelledError}. - - @param deferred: The L{Deferred } that was cancelled; - should be the same as C{self._onConnection}. - @type deferred: L{Deferred } - - @note: This relies on startedConnecting having been called, so it may - seem as though there's a race condition where C{_connector} may not - have been set. However, using public APIs, this condition is - impossible to catch, because a connection API - (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a - L{_WrappingFactory}'s L{Deferred } is returned to - C{connect()}'s caller. - - @return: C{None} - """ - deferred.errback( - error.ConnectingCancelledError( - self._connector.getDestination())) - self._connector.stopConnecting() - - - def doStart(self): - """ - Start notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStart() - - - def doStop(self): - """ - Stop notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStop() - - - def buildProtocol(self, addr): - """ - Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback - the C{self._onConnection} L{Deferred}. - - @return: An instance of L{_WrappingProtocol} or C{None} - """ - try: - proto = self._wrappedFactory.buildProtocol(addr) - except: - self._onConnection.errback() - else: - return self.protocol(self._onConnection, proto) - - - def clientConnectionFailed(self, connector, reason): - """ - Errback the C{self._onConnection} L{Deferred} when the - client connection fails. - """ - if not self._onConnection.called: - self._onConnection.errback(reason) - - - - - -@implementer(interfaces.ITransport) -class _ProcessEndpointTransport(proxyForInterface( - interfaces.IProcessTransport, '_process')): - """ - An L{ITransport} provider for the L{IProtocol} instance passed to the - process endpoint. - - @ivar _process: An active process transport which will be used by write - methods on this object to write data to a child process. - @type _process: L{interfaces.IProcessTransport} provider - """ - - def write(self, data): - """ - Write to the child process's standard input. - - @param data: The data to write on stdin. - """ - self._process.writeToChild(0, data) - - - def writeSequence(self, data): - """ - Write a list of strings to child process's stdin. - - @param data: The list of chunks to write on stdin. - """ - for chunk in data: - self._process.writeToChild(0, chunk) - - -@implementer(interfaces.IStreamServerEndpoint) -class _TCPServerEndpoint(object): - """ - A TCP server endpoint interface - """ - - def __init__(self, reactor, port, backlog, interface): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - self._reactor = reactor - self._port = port - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a TCP - socket - """ - return defer.execute(self._reactor.listenTCP, - self._port, - protocolFactory, - backlog=self._backlog, - interface=self._interface) - - - -class TCP4ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv4 configuration - """ - def __init__(self, reactor, port, backlog=50, interface=''): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -class TCP6ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv6 configuration - """ - def __init__(self, reactor, port, backlog=50, interface='::'): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class TCP4ClientEndpoint(object): - """ - TCP client endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, host, port, timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorTCP} provider - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param timeout: The number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectTCP( - self._host, self._port, wf, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - - -@implementer(interfaces.IStreamServerEndpoint) -class SSL4ServerEndpoint(object): - """ - SSL secured TCP server endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, port, sslContextFactory, - backlog=50, interface=''): - """ - @param reactor: An L{IReactorSSL} provider. - - @param port: The port number used for listening - @type port: int - - @param sslContextFactory: An instance of - L{twisted.internet.ssl.ContextFactory}. - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - self._reactor = reactor - self._port = port - self._sslContextFactory = sslContextFactory - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen for SSL on a - TCP socket. - """ - return defer.execute(self._reactor.listenSSL, self._port, - protocolFactory, - contextFactory=self._sslContextFactory, - backlog=self._backlog, - interface=self._interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class SSL4ClientEndpoint(object): - """ - SSL secured TCP client endpoint with an IPv4 configuration - """ - - def __init__(self, reactor, host, port, sslContextFactory, - timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorSSL} provider. - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param sslContextFactory: SSL Configuration information as an instance - of L{twisted.internet.ssl.ContextFactory}. - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._sslContextFactory = sslContextFactory - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect with SSL over - TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectSSL( - self._host, self._port, wf, self._sslContextFactory, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class UNIXServerEndpoint(object): - """ - UnixSocket server endpoint. - """ - def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - @param address: The path to the Unix socket file, used when listening - @param backlog: number of connections to allow in backlog. - @param mode: mode to set on the unix socket. This parameter is - deprecated. Permissions should be set on the directory which - contains the UNIX socket. - @param wantPID: If True, create a pidfile for the socket. - """ - self._reactor = reactor - self._address = address - self._backlog = backlog - self._mode = mode - self._wantPID = wantPID - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket. - """ - return defer.execute(self._reactor.listenUNIX, self._address, - protocolFactory, - backlog=self._backlog, - mode=self._mode, - wantPID=self._wantPID) - - - -@implementer(interfaces.IStreamClientEndpoint) -class UNIXClientEndpoint(object): - """ - UnixSocket client endpoint. - """ - def __init__(self, reactor, path, timeout=30, checkPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - - @param path: The path to the Unix socket file, used when connecting - @type path: str - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param checkPID: If True, check for a pid file to verify that a server - is listening. - @type checkPID: bool - """ - self._reactor = reactor - self._path = path - self._timeout = timeout - self._checkPID = checkPID - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via a - UNIX Socket - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectUNIX( - self._path, wf, - timeout=self._timeout, - checkPID=self._checkPID) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class AdoptedStreamServerEndpoint(object): - """ - An endpoint for listening on a file descriptor initialized outside of - Twisted. - - @ivar _used: A C{bool} indicating whether this endpoint has been used to - listen with a factory yet. C{True} if so. - """ - _close = os.close - _setNonBlocking = staticmethod(fdesc.setNonBlocking) - - def __init__(self, reactor, fileno, addressFamily): - """ - @param reactor: An L{IReactorSocket} provider. - - @param fileno: An integer file descriptor corresponding to a listening - I{SOCK_STREAM} socket. - - @param addressFamily: The address family of the socket given by - C{fileno}. - """ - self.reactor = reactor - self.fileno = fileno - self.addressFamily = addressFamily - self._used = False - - - def listen(self, factory): - """ - Implement L{IStreamServerEndpoint.listen} to start listening on, and - then close, C{self._fileno}. - """ - if self._used: - return defer.fail(error.AlreadyListened()) - self._used = True - - try: - self._setNonBlocking(self.fileno) - port = self.reactor.adoptStreamPort( - self.fileno, self.addressFamily, factory) - self._close(self.fileno) - except: - return defer.fail() - return defer.succeed(port) - - - - -def _parseTCP(factory, port, interface="", backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a TCP(IPv4) stream endpoint into the structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments - to L{TCP4ServerEndpoint}. - """ - return (int(port), factory), {'interface': interface, - 'backlog': int(backlog)} - - - -def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the - structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param address: the pathname of the unix socket - @type address: C{str} - - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param lockfile: A string '0' or '1', mapping to True and False - respectively. See the C{wantPID} argument to C{listenUNIX} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory, - arguments to L{UNIXServerEndpoint}. - """ - return ( - (address, factory), - {'mode': int(mode, 8), 'backlog': int(backlog), - 'wantPID': bool(int(lockfile))}) - - - -def _parseSSL(factory, port, privateKey="server.pem", certKey=None, - sslmethod=None, interface='', backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for an SSL (over TCP/IPv4) stream endpoint into the structured - arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param privateKey: The file name of a PEM format private key file. - @type privateKey: C{str} - - @param certKey: The file name of a PEM format certificate file. - @type certKey: C{str} - - @param sslmethod: The string name of an SSL method, based on the name of a - constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD", - "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD". - @type sslmethod: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments - to L{SSL4ServerEndpoint}. - """ - from twisted.internet import ssl - if certKey is None: - certKey = privateKey - kw = {} - if sslmethod is not None: - kw['method'] = getattr(ssl.SSL, sslmethod) - else: - kw['method'] = ssl.SSL.SSLv23_METHOD - certPEM = FilePath(certKey).getContent() - keyPEM = FilePath(privateKey).getContent() - privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM) - cf = ssl.CertificateOptions( - privateKey=privateCertificate.privateKey.original, - certificate=privateCertificate.original, - **kw - ) - return ((int(port), factory, cf), - {'interface': interface, 'backlog': int(backlog)}) - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _StandardIOParser(object): - """ - Stream server endpoint string parser for the Standard I/O type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "stdio" - - def _parseServer(self, reactor): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{StandardIOEndpoint} - - @param reactor: Reactor for the endpoint - """ - return StandardIOEndpoint(reactor) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor) - - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _TCP6ServerParser(object): - """ - Stream server endpoint string parser for the TCP6ServerEndpoint type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type - - def _parseServer(self, reactor, port, backlog=50, interface='::'): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{TCP6ServerEndpoint} - - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - port = int(port) - backlog = int(backlog) - return TCP6ServerEndpoint(reactor, port, backlog, interface) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor, *args, **kwargs) - - - -_serverParsers = {"tcp": _parseTCP, - "unix": _parseUNIX, - "ssl": _parseSSL, - } - -_OP, _STRING = range(2) - -def _tokenize(description): - """ - Tokenize a strports string and yield each token. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples - starting with L{_OP} will contain a second element of either ':' (i.e. - 'next parameter') or '=' (i.e. 'assign parameter value'). For example, - the string 'hello:greet\=ing=world' would result in a generator - yielding these values:: - - _STRING, 'hello' - _OP, ':' - _STRING, 'greet=ing' - _OP, '=' - _STRING, 'world' - """ - current = '' - ops = ':=' - nextOps = {':': ':=', '=': ':'} - description = iter(description) - for n in description: - if n in ops: - yield _STRING, current - yield _OP, n - current = '' - ops = nextOps[n] - elif n == '\\': - current += next(description) - else: - current += n - yield _STRING, current - - - -def _parse(description): - """ - Convert a description string into a list of positional and keyword - parameters, using logic vaguely like what Python does. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all - ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of - all C{str}s which do contain an '='. For example, the result of - C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}. - """ - args, kw = [], {} - def add(sofar): - if len(sofar) == 1: - args.append(sofar[0]) - else: - kw[sofar[0]] = sofar[1] - sofar = () - for (type, value) in _tokenize(description): - if type is _STRING: - sofar += (value,) - elif value == ':': - add(sofar) - sofar = () - add(sofar) - return args, kw - - -# Mappings from description "names" to endpoint constructors. -_endpointServerFactories = { - 'TCP': TCP4ServerEndpoint, - 'SSL': SSL4ServerEndpoint, - 'UNIX': UNIXServerEndpoint, - } - -_endpointClientFactories = { - 'TCP': TCP4ClientEndpoint, - 'SSL': SSL4ClientEndpoint, - 'UNIX': UNIXClientEndpoint, - } - - -_NO_DEFAULT = object() - -def _parseServer(description, factory, default=None): - """ - Parse a stports description into a 2-tuple of arguments and keyword values. - - @param description: A description in the format explained by - L{serverFromString}. - @type description: C{str} - - @param factory: A 'factory' argument; this is left-over from - twisted.application.strports, it's not really used. - @type factory: L{IProtocolFactory} or L{None} - - @param default: Deprecated argument, specifying the default parser mode to - use for unqualified description strings (those which do not have a ':' - and prefix). - @type default: C{str} or C{NoneType} - - @return: a 3-tuple of (plugin or name, arguments, keyword arguments) - """ - args, kw = _parse(description) - if not args or (len(args) == 1 and not kw): - deprecationMessage = ( - "Unqualified strport description passed to 'service'." - "Use qualified endpoint descriptions; for example, 'tcp:%s'." - % (description,)) - if default is None: - default = 'tcp' - warnings.warn( - deprecationMessage, category=DeprecationWarning, stacklevel=4) - elif default is _NO_DEFAULT: - raise ValueError(deprecationMessage) - # If the default has been otherwise specified, the user has already - # been warned. - args[0:0] = [default] - endpointType = args[0] - parser = _serverParsers.get(endpointType) - if parser is None: - # If the required parser is not found in _server, check if - # a plugin exists for the endpointType - for plugin in getPlugins(IStreamServerEndpointStringParser): - if plugin.prefix == endpointType: - return (plugin, args[1:], kw) - raise ValueError("Unknown endpoint type: '%s'" % (endpointType,)) - return (endpointType.upper(),) + parser(factory, *args[1:], **kw) - - - -def _serverFromStringLegacy(reactor, description, default): - """ - Underlying implementation of L{serverFromString} which avoids exposing the - deprecated 'default' argument to anything but L{strports.service}. - """ - nameOrPlugin, args, kw = _parseServer(description, None, default) - if type(nameOrPlugin) is not str: - plugin = nameOrPlugin - return plugin.parseStreamServer(reactor, *args, **kw) - else: - name = nameOrPlugin - # Chop out the factory. - args = args[:1] + args[2:] - return _endpointServerFactories[name](reactor, *args, **kw) - - - -def serverFromString(reactor, description): - """ - Construct a stream server endpoint from an endpoint description string. - - The format for server endpoint descriptions is a simple string. It is a - prefix naming the type of endpoint, then a colon, then the arguments for - that endpoint. - - For example, you can call it like this to create an endpoint that will - listen on TCP port 80:: - - serverFromString(reactor, "tcp:80") - - Additional arguments may be specified as keywords, separated with colons. - For example, you can specify the interface for a TCP server endpoint to - bind to like this:: - - serverFromString(reactor, "tcp:80:interface=127.0.0.1") - - SSL server endpoints may be specified with the 'ssl' prefix, and the - private key and certificate files may be specified by the C{privateKey} and - C{certKey} arguments:: - - serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem") - - If a private key file name (C{privateKey}) isn't provided, a "server.pem" - file is assumed to exist which contains the private key. If the certificate - file name (C{certKey}) isn't provided, the private key file is assumed to - contain the certificate as well. - - You may escape colons in arguments with a backslash, which you will need to - use if you want to specify a full pathname argument on Windows:: - - serverFromString(reactor, - "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem") - - finally, the 'unix' prefix may be used to specify a filesystem UNIX socket, - optionally with a 'mode' argument to specify the mode of the socket file - created by C{listen}:: - - serverFromString(reactor, "unix:/var/run/finger") - serverFromString(reactor, "unix:/var/run/finger:mode=660") - - This function is also extensible; new endpoint types may be registered as - L{IStreamServerEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The server endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to listen with the parameters - given by by C{description}. - - @rtype: L{IStreamServerEndpoint} - - @raise ValueError: when the 'description' string cannot be parsed. - - @since: 10.2 - """ - return _serverFromStringLegacy(reactor, description, _NO_DEFAULT) - - - -def quoteStringArgument(argument): - """ - Quote an argument to L{serverFromString} and L{clientFromString}. Since - arguments are separated with colons and colons are escaped with - backslashes, some care is necessary if, for example, you have a pathname, - you may be tempted to interpolate into a string like this:: - - serverFromString("ssl:443:privateKey=%s" % (myPathName,)) - - This may appear to work, but will have portability issues (Windows - pathnames, for example). Usually you should just construct the appropriate - endpoint type rather than interpolating strings, which in this case would - be L{SSL4ServerEndpoint}. There are some use-cases where you may need to - generate such a string, though; for example, a tool to manipulate a - configuration file which has strports descriptions in it. To be correct in - those cases, do this instead:: - - serverFromString("ssl:443:privateKey=%s" % - (quoteStringArgument(myPathName),)) - - @param argument: The part of the endpoint description string you want to - pass through. - - @type argument: C{str} - - @return: The quoted argument. - - @rtype: C{str} - """ - return argument.replace('\\', '\\\\').replace(':', '\\:') - - - -def _parseClientTCP(*args, **kwargs): - """ - Perform any argument value coercion necessary for TCP client parameters. - - Valid positional arguments to this function are host and port. - - Valid keyword arguments to this function are all L{IReactorTCP.connectTCP} - arguments. - - @return: The coerced values as a C{dict}. - """ - - if len(args) == 2: - kwargs['port'] = int(args[1]) - kwargs['host'] = args[0] - elif len(args) == 1: - if 'host' in kwargs: - kwargs['port'] = int(args[0]) - else: - kwargs['host'] = args[0] - - try: - kwargs['port'] = int(kwargs['port']) - except KeyError: - pass - - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - - - -def _loadCAsFromDir(directoryPath): - """ - Load certificate-authority certificate objects in a given directory. - - @param directoryPath: a L{FilePath} pointing at a directory to load .pem - files from. - - @return: a C{list} of L{OpenSSL.crypto.X509} objects. - """ - from twisted.internet import ssl - - caCerts = {} - for child in directoryPath.children(): - if not child.basename().split('.')[-1].lower() == 'pem': - continue - try: - data = child.getContent() - except IOError: - # Permission denied, corrupt disk, we don't care. - continue - try: - theCert = ssl.Certificate.loadPEM(data) - except ssl.SSL.Error: - # Duplicate certificate, invalid certificate, etc. We don't care. - pass - else: - caCerts[theCert.digest()] = theCert.original - return caCerts.values() - - - -def _parseClientSSL(*args, **kwargs): - """ - Perform any argument value coercion necessary for SSL client parameters. - - Valid keyword arguments to this function are all L{IReactorSSL.connectSSL} - arguments except for C{contextFactory}. Instead, C{certKey} (the path name - of the certificate file) C{privateKey} (the path name of the private key - associated with the certificate) are accepted and used to construct a - context factory. - - Valid positional arguments to this function are host and port. - - @param caCertsDir: The one parameter which is not part of - L{IReactorSSL.connectSSL}'s signature, this is a path name used to - construct a list of certificate authority certificates. The directory - will be scanned for files ending in C{.pem}, all of which will be - considered valid certificate authorities for this connection. - - @type caCertsDir: C{str} - - @return: The coerced values as a C{dict}. - """ - from twisted.internet import ssl - kwargs = _parseClientTCP(*args, **kwargs) - certKey = kwargs.pop('certKey', None) - privateKey = kwargs.pop('privateKey', None) - caCertsDir = kwargs.pop('caCertsDir', None) - if certKey is not None: - certx509 = ssl.Certificate.loadPEM( - FilePath(certKey).getContent()).original - else: - certx509 = None - if privateKey is not None: - privateKey = ssl.PrivateCertificate.loadPEM( - FilePath(privateKey).getContent()).privateKey.original - else: - privateKey = None - if caCertsDir is not None: - verify = True - caCerts = _loadCAsFromDir(FilePath(caCertsDir)) - else: - verify = False - caCerts = None - kwargs['sslContextFactory'] = ssl.CertificateOptions( - method=ssl.SSL.SSLv23_METHOD, - certificate=certx509, - privateKey=privateKey, - verify=verify, - caCerts=caCerts - ) - return kwargs - - - -def _parseClientUNIX(*args, **kwargs): - """ - Perform any argument value coercion necessary for UNIX client parameters. - - Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX} - keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted - and has the same meaning. Also C{path} is used instead of C{address}. - - Valid positional arguments to this function are C{path}. - - @return: The coerced values as a C{dict}. - """ - if len(args) == 1: - kwargs['path'] = args[0] - - try: - kwargs['checkPID'] = bool(int(kwargs.pop('lockfile'))) - except KeyError: - pass - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - -_clientParsers = { - 'TCP': _parseClientTCP, - 'SSL': _parseClientSSL, - 'UNIX': _parseClientUNIX, - } - - - -def clientFromString(reactor, description): - """ - Construct a client endpoint from a description string. - - Client description strings are much like server description strings, - although they take all of their arguments as keywords, aside from host and - port. - - You can create a TCP client endpoint with the 'host' and 'port' arguments, - like so:: - - clientFromString(reactor, "tcp:host=www.example.com:port=80") - - or, without specifying host and port keywords:: - - clientFromString(reactor, "tcp:www.example.com:80") - - Or you can specify only one or the other, as in the following 2 examples:: - - clientFromString(reactor, "tcp:host=www.example.com:80") - clientFromString(reactor, "tcp:www.example.com:port=80") - - or an SSL client endpoint with those arguments, plus the arguments used by - the server SSL, for a client certificate:: - - clientFromString(reactor, "ssl:web.example.com:443:" - "privateKey=foo.pem:certKey=foo.pem") - - to specify your certificate trust roots, you can identify a directory with - PEM files in it with the C{caCertsDir} argument:: - - clientFromString(reactor, "ssl:host=web.example.com:port=443:" - "caCertsDir=/etc/ssl/certs") - - You can create a UNIX client endpoint with the 'path' argument and optional - 'lockfile' and 'timeout' arguments:: - - clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9") - - or, with the path as a positional argument with or without optional - arguments as in the following 2 examples:: - - clientFromString(reactor, "unix:/var/foo/bar") - clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9") - - This function is also extensible; new endpoint types may be registered as - L{IStreamClientEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The client endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to connect with the parameters - given by by C{description}. - @rtype: L{IStreamClientEndpoint} - - @since: 10.2 - """ - args, kwargs = _parse(description) - aname = args.pop(0) - name = aname.upper() - for plugin in getPlugins(IStreamClientEndpointStringParser): - if plugin.prefix.upper() == name: - return plugin.parseStreamClient(*args, **kwargs) - if name not in _clientParsers: - raise ValueError("Unknown endpoint type: %r" % (aname,)) - kwargs = _clientParsers[name](*args, **kwargs) - return _endpointClientFactories[name](reactor, **kwargs) - - - -def connectProtocol(endpoint, protocol): - """ - Connect a protocol instance to an endpoint. - - This allows using a client endpoint without having to create a factory. - - @param endpoint: A client endpoint to connect to. - - @param protocol: A protocol instance. - - @return: The result of calling C{connect} on the endpoint, i.e. a - L{Deferred} that will fire with the protocol when connected, or an - appropriate error. - """ - class OneShotFactory(Factory): - def buildProtocol(self, addr): - return protocol - return endpoint.connect(OneShotFactory()) -}}} ''' diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index 7b2a78632..fdcbf3977 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -9,8 +9,6 @@ Maintainer: Itamar Shtull-Trauring from __future__ import division, absolute_import -from zope.interface import Interface, Attribute - from twisted.internet.interfaces import ( IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, @@ -28,2434 +26,3 @@ from twisted.internet.interfaces import ( IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver ) - -''' {{{ -class IAddress(Interface): - """ - An address, e.g. a TCP C{(host, port)}. - - Default implementations are in L{twisted.internet.address}. - """ - -### Reactor Interfaces - -class IConnector(Interface): - """ - Object used to interface between connections and protocols. - - Each L{IConnector} manages one connection. - """ - - def stopConnecting(): - """ - Stop attempting to connect. - """ - - def disconnect(): - """ - Disconnect regardless of the connection state. - - If we are connected, disconnect, if we are trying to connect, - stop trying. - """ - - def connect(): - """ - Try to connect to remote address. - """ - - def getDestination(): - """ - Return destination this will try to connect to. - - @return: An object which provides L{IAddress}. - """ - - - -class IResolverSimple(Interface): - def getHostByName(name, timeout = (1, 3, 11, 45)): - """ - Resolve the domain name C{name} into an IP address. - - @type name: C{str} - @type timeout: C{tuple} - @rtype: L{twisted.internet.defer.Deferred} - @return: The callback of the Deferred that is returned will be - passed a string that represents the IP address of the specified - name, or the errback will be called if the lookup times out. If - multiple types of address records are associated with the name, - A6 records will be returned in preference to AAAA records, which - will be returned in preference to A records. If there are multiple - records of the type to be returned, one will be selected at random. - - @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) - if the name cannot be resolved within the specified timeout period. - """ - - - -class IResolver(IResolverSimple): - def query(query, timeout=None): - """ - Dispatch C{query} to the method which can handle its type. - - @type query: L{twisted.names.dns.Query} - @param query: The DNS query being issued, to which a response is to be - generated. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress(name, timeout=None): - """ - Perform an A record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress6(name, timeout=None): - """ - Perform an A6 record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupIPV6Address(name, timeout=None): - """ - Perform an AAAA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailExchange(name, timeout=None): - """ - Perform an MX record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNameservers(name, timeout=None): - """ - Perform an NS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupCanonicalName(name, timeout=None): - """ - Perform a CNAME record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailBox(name, timeout=None): - """ - Perform an MB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailGroup(name, timeout=None): - """ - Perform an MG record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailRename(name, timeout=None): - """ - Perform an MR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupPointer(name, timeout=None): - """ - Perform a PTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAuthority(name, timeout=None): - """ - Perform an SOA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNull(name, timeout=None): - """ - Perform a NULL record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupWellKnownServices(name, timeout=None): - """ - Perform a WKS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupHostInfo(name, timeout=None): - """ - Perform a HINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailboxInfo(name, timeout=None): - """ - Perform an MINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupText(name, timeout=None): - """ - Perform a TXT record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupResponsibility(name, timeout=None): - """ - Perform an RP record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAFSDatabase(name, timeout=None): - """ - Perform an AFSDB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupService(name, timeout=None): - """ - Perform an SRV record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAllRecords(name, timeout=None): - """ - Perform an ALL_RECORD lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupSenderPolicy(name, timeout= 10): - """ - Perform a SPF record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNamingAuthorityPointer(name, timeout=None): - """ - Perform a NAPTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupZone(name, timeout=None): - """ - Perform an AXFR record lookup. - - NB This is quite different from other DNS requests. See - U{http://cr.yp.to/djbdns/axfr-notes.html} for more - information. - - NB Unlike other C{lookup*} methods, the timeout here is not a - list of ints, it is a single int. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: C{int} - @param timeout: When this timeout expires, the query is - considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. - The first element of the tuple gives answers. - The second and third elements are always empty. - The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - -class IReactorTCP(Interface): - - def listenTCP(port, factory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param backlog: size of the listen queue - - @param interface: The local IPv4 or IPv6 address to which to bind; - defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6 - addresses, you must call this method twice. - - @return: an object that provides L{IListeningPort}. - - @raise CannotListenError: as defined here - L{twisted.internet.error.CannotListenError}, - if it cannot listen on this port (e.g., it - cannot bind to the required port number) - """ - - def connectTCP(host, port, factory, timeout=30, bindAddress=None): - """ - Connect a TCP client. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind - to, or None. - - @return: An object which provides L{IConnector}. This connector will - call various callbacks on the factory when a connection is - made, failed, or lost - see - L{ClientFactory} - docs for details. - """ - -class IReactorSSL(Interface): - - def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None): - """ - Connect a client Protocol to a remote SSL socket. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object. - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind to, - or C{None}. - - @return: An object which provides L{IConnector}. - """ - - def listenSSL(port, factory, contextFactory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - The connection is a SSL one, using contexts created by the context - factory. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance - - @param backlog: size of the listen queue - - @param interface: the hostname to bind to, defaults to '' (all) - """ - - - -class IReactorUNIX(Interface): - """ - UNIX socket methods. - """ - - def connectUNIX(address, factory, timeout=30, checkPID=0): - """ - Connect a client protocol to a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the connection - has failed. - - @param checkPID: if True, check for a pid file to verify that a server - is listening. If C{address} is a Linux abstract namespace path, - this must be C{False}. - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0): - """ - Listen on a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.Factory} instance. - - @param backlog: number of connections to allow in backlog. - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param wantPID: if True, create a pidfile for the socket. If C{address} - is a Linux abstract namespace path, this must be C{False}. - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorUNIXDatagram(Interface): - """ - Datagram UNIX socket methods. - """ - - def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None): - """ - Connect a client protocol to a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param bindAddress: address to bind to - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666): - """ - Listen on a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance. - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorWin32Events(Interface): - """ - Win32 Event API methods - - @since: 10.2 - """ - - def addEvent(event, fd, action): - """ - Add a new win32 event to the event loop. - - @param event: a Win32 event object created using win32event.CreateEvent() - - @param fd: an instance of L{twisted.internet.abstract.FileDescriptor} - - @param action: a string that is a method name of the fd instance. - This method is called in response to the event. - - @return: None - """ - - - def removeEvent(event): - """ - Remove an event. - - @param event: a Win32 event object added using L{IReactorWin32Events.addEvent} - - @return: None - """ - - - -class IReactorUDP(Interface): - """ - UDP socket methods. - """ - - def listenUDP(port, protocol, interface='', maxPacketSize=8192): - """ - Connects a given DatagramProtocol to the given numeric UDP port. - - @return: object which provides L{IListeningPort}. - """ - - - -class IReactorMulticast(Interface): - """ - UDP socket methods that support multicast. - - IMPORTANT: This is an experimental new interface. It may change - without backwards compatibility. Suggestions are welcome. - """ - - def listenMulticast(port, protocol, interface='', maxPacketSize=8192, - listenMultiple=False): - """ - Connects a given - L{DatagramProtocol} to the - given numeric UDP port. - - @param listenMultiple: If set to True, allows multiple sockets to - bind to the same address and port number at the same time. - @type listenMultiple: C{bool} - - @returns: An object which provides L{IListeningPort}. - - @see: L{twisted.internet.interfaces.IMulticastTransport} - @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} - """ - - - -class IReactorSocket(Interface): - """ - Methods which allow a reactor to use externally created sockets. - - For example, to use C{adoptStreamPort} to implement behavior equivalent - to that of L{IReactorTCP.listenTCP}, you might write code like this:: - - from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket - portSocket = socket(AF_INET, SOCK_STREAM) - # Set FD_CLOEXEC on port, left as an exercise. Then make it into a - # non-blocking listening port: - portSocket.setblocking(False) - portSocket.bind(('192.168.1.2', 12345)) - portSocket.listen(SOMAXCONN) - - # Now have the reactor use it as a TCP port - port = reactor.adoptStreamPort( - portSocket.fileno(), AF_INET, YourFactory()) - - # portSocket itself is no longer necessary, and needs to be cleaned - # up by us. - portSocket.close() - - # Whenever the server is no longer needed, stop it as usual. - stoppedDeferred = port.stopListening() - - Another potential use is to inherit a listening descriptor from a parent - process (for example, systemd or launchd), or to receive one over a UNIX - domain socket. - - Some plans for extending this interface exist. See: - - - U{http://twistedmatrix.com/trac/ticket/5570}: established connections - - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports - - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets - """ - - def adoptStreamPort(fileDescriptor, addressFamily, factory): - """ - Add an existing listening I{SOCK_STREAM} socket to the reactor to - monitor for new connections to accept and handle. - - @param fileDescriptor: A file descriptor associated with a socket which - is already bound to an address and marked as listening. The socket - must be set non-blocking. Any additional flags (for example, - close-on-exec) must also be set by application code. Application - code is responsible for closing the file descriptor, which may be - done as soon as C{adoptStreamPort} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create new - protocols to handle connections accepted via this socket. - - @return: An object providing L{IListeningPort}. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - def adoptStreamConnection(fileDescriptor, addressFamily, factory): - """ - Add an existing connected I{SOCK_STREAM} socket to the reactor to - monitor for data. - - Note that the given factory won't have its C{startFactory} and - C{stopFactory} methods called, as there is no sensible time to call - them in this situation. - - @param fileDescriptor: A file descriptor associated with a socket which - is already connected. The socket must be set non-blocking. Any - additional flags (for example, close-on-exec) must also be set by - application code. Application code is responsible for closing the - file descriptor, which may be done as soon as - C{adoptStreamConnection} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create a new - protocol to handle the connection via this socket. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - -class IReactorProcess(Interface): - - def spawnProcess(processProtocol, executable, args=(), env={}, path=None, - uid=None, gid=None, usePTY=0, childFDs=None): - """ - Spawn a process, with a process protocol. - - @type processProtocol: L{IProcessProtocol} provider - @param processProtocol: An object which will be notified of all - events related to the created process. - - @param executable: the file name to spawn - the full path should be - used. - - @param args: the command line arguments to pass to the process; a - sequence of strings. The first string should be the - executable's name. - - @type env: a C{dict} mapping C{str} to C{str}, or C{None}. - @param env: the environment variables to pass to the child process. The - resulting behavior varies between platforms. If - - C{env} is not set: - - On POSIX: pass an empty environment. - - On Windows: pass C{os.environ}. - - C{env} is C{None}: - - On POSIX: pass C{os.environ}. - - On Windows: pass C{os.environ}. - - C{env} is a C{dict}: - - On POSIX: pass the key/value pairs in C{env} as the - complete environment. - - On Windows: update C{os.environ} with the key/value - pairs in the C{dict} before passing it. As a - consequence of U{bug #1640 - }, passing - keys with empty values in an effort to unset - environment variables I{won't} unset them. - - @param path: the path to run the subprocess in - defaults to the - current directory. - - @param uid: user ID to run the subprocess as. (Only available on - POSIX systems.) - - @param gid: group ID to run the subprocess as. (Only available on - POSIX systems.) - - @param usePTY: if true, run this process in a pseudo-terminal. - optionally a tuple of C{(masterfd, slavefd, ttyname)}, - in which case use those file descriptors. - (Not available on all systems.) - - @param childFDs: A dictionary mapping file descriptors in the new child - process to an integer or to the string 'r' or 'w'. - - If the value is an integer, it specifies a file - descriptor in the parent process which will be mapped - to a file descriptor (specified by the key) in the - child process. This is useful for things like inetd - and shell-like file redirection. - - If it is the string 'r', a pipe will be created and - attached to the child at that file descriptor: the - child will be able to write to that file descriptor - and the parent will receive read notification via the - L{IProcessProtocol.childDataReceived} callback. This - is useful for the child's stdout and stderr. - - If it is the string 'w', similar setup to the previous - case will occur, with the pipe being readable by the - child instead of writeable. The parent process can - write to that file descriptor using - L{IProcessTransport.writeToChild}. This is useful for - the child's stdin. - - If childFDs is not passed, the default behaviour is to - use a mapping that opens the usual stdin/stdout/stderr - pipes. - - @see: L{twisted.internet.protocol.ProcessProtocol} - - @return: An object which provides L{IProcessTransport}. - - @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are - insufficient system resources to create a new process. - """ - -class IReactorTime(Interface): - """ - Time methods that a Reactor should implement. - """ - - def seconds(): - """ - Get the current time in seconds. - - @return: A number-like object of some sort. - """ - - - def callLater(delay, callable, *args, **kw): - """ - Call a function later. - - @type delay: C{float} - @param delay: the number of seconds to wait. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: An object which provides L{IDelayedCall} and can be used to - cancel the scheduled call, by calling its C{cancel()} method. - It also may be rescheduled by calling its C{delay()} or - C{reset()} methods. - """ - - - def getDelayedCalls(): - """ - Retrieve all currently scheduled delayed calls. - - @return: A tuple of all L{IDelayedCall} providers representing all - currently scheduled calls. This is everything that has been - returned by C{callLater} but not yet called or canceled. - """ - - -class IDelayedCall(Interface): - """ - A scheduled call. - - There are probably other useful methods we can add to this interface; - suggestions are welcome. - """ - - def getTime(): - """ - Get time when delayed call will happen. - - @return: time in seconds since epoch (a float). - """ - - def cancel(): - """ - Cancel the scheduled call. - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def delay(secondsLater): - """ - Delay the scheduled call. - - @param secondsLater: how many seconds from its current firing time to delay - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def reset(secondsFromNow): - """ - Reset the scheduled call's timer. - - @param secondsFromNow: how many seconds from now it should fire, - equivalent to C{.cancel()} and then doing another - C{reactor.callLater(secondsLater, ...)} - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def active(): - """ - @return: True if this call is still active, False if it has been - called or cancelled. - """ - -class IReactorThreads(Interface): - """ - Dispatch methods to be run in threads. - - Internally, this should use a thread pool and dispatch methods to them. - """ - - def getThreadPool(): - """ - Return the threadpool used by L{callInThread}. Create it first if - necessary. - - @rtype: L{twisted.python.threadpool.ThreadPool} - """ - - - def callInThread(callable, *args, **kwargs): - """ - Run the callable object in a separate thread. - """ - - - def callFromThread(callable, *args, **kw): - """ - Cause a function to be executed by the reactor thread. - - Use this method when you want to run a function in the reactor's thread - from another thread. Calling L{callFromThread} should wake up the main - thread (where L{reactor.run()} is executing) and run the - given callable in that thread. - - If you're writing a multi-threaded application the C{callable} may need - to be thread safe, but this method doesn't require it as such. If you - want to call a function in the next mainloop iteration, but you're in - the same thread, use L{callLater} with a delay of 0. - """ - - - def suggestThreadPoolSize(size): - """ - Suggest the size of the internal threadpool used to dispatch functions - passed to L{callInThread}. - """ - - -class IReactorCore(Interface): - """ - Core methods that a Reactor must implement. - """ - - running = Attribute( - "A C{bool} which is C{True} from I{during startup} to " - "I{during shutdown} and C{False} the rest of the time.") - - - def resolve(name, timeout=10): - """ - Return a L{twisted.internet.defer.Deferred} that will resolve a hostname. - """ - - def run(): - """ - Fire 'startup' System Events, move the reactor to the 'running' - state, then run the main loop until it is stopped with C{stop()} or - C{crash()}. - """ - - def stop(): - """ - Fire 'shutdown' System Events, which will move the reactor to the - 'stopped' state and cause C{reactor.run()} to exit. - """ - - def crash(): - """ - Stop the main loop *immediately*, without firing any system events. - - This is named as it is because this is an extremely "rude" thing to do; - it is possible to lose data and put your system in an inconsistent - state by calling this. However, it is necessary, as sometimes a system - can become wedged in a pre-shutdown call. - """ - - def iterate(delay=0): - """ - Run the main loop's I/O polling function for a period of time. - - This is most useful in applications where the UI is being drawn "as - fast as possible", such as games. All pending L{IDelayedCall}s will - be called. - - The reactor must have been started (via the C{run()} method) prior to - any invocations of this method. It must also be stopped manually - after the last call to this method (via the C{stop()} method). This - method is not re-entrant: you must not call it recursively; in - particular, you must not call it while the reactor is running. - """ - - def fireSystemEvent(eventType): - """ - Fire a system-wide event. - - System-wide events are things like 'startup', 'shutdown', and - 'persist'. - """ - - def addSystemEventTrigger(phase, eventType, callable, *args, **kw): - """ - Add a function to be called when a system event occurs. - - Each "system event" in Twisted, such as 'startup', 'shutdown', and - 'persist', has 3 phases: 'before', 'during', and 'after' (in that - order, of course). These events will be fired internally by the - Reactor. - - An implementor of this interface must only implement those events - described here. - - Callbacks registered for the "before" phase may return either None or a - Deferred. The "during" phase will not execute until all of the - Deferreds from the "before" phase have fired. - - Once the "during" phase is running, all of the remaining triggers must - execute; their return values must be ignored. - - @param phase: a time to call the event -- either the string 'before', - 'after', or 'during', describing when to call it - relative to the event's execution. - - @param eventType: this is a string describing the type of event. - - @param callable: the object to call before shutdown. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: an ID that can be used to remove this call with - removeSystemEventTrigger. - """ - - def removeSystemEventTrigger(triggerID): - """ - Removes a trigger added with addSystemEventTrigger. - - @param triggerID: a value returned from addSystemEventTrigger. - - @raise KeyError: If there is no system event trigger for the given - C{triggerID}. - - @raise ValueError: If there is no system event trigger for the given - C{triggerID}. - - @raise TypeError: If there is no system event trigger for the given - C{triggerID}. - """ - - def callWhenRunning(callable, *args, **kw): - """ - Call a function when the reactor is running. - - If the reactor has not started, the callable will be scheduled - to run when it does start. Otherwise, the callable will be invoked - immediately. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: None if the callable was invoked, otherwise a system - event id for the scheduled call. - """ - - -class IReactorPluggableResolver(Interface): - """ - A reactor with a pluggable name resolver interface. - """ - - def installResolver(resolver): - """ - Set the internal resolver to use to for name lookups. - - @type resolver: An object implementing the L{IResolverSimple} interface - @param resolver: The new resolver to use. - - @return: The previously installed resolver. - """ - - -class IReactorDaemonize(Interface): - """ - A reactor which provides hooks that need to be called before and after - daemonization. - - Notes: - - This interface SHOULD NOT be called by applications. - - This interface should only be implemented by reactors as a workaround - (in particular, it's implemented currently only by kqueue()). - For details please see the comments on ticket #1918. - """ - - def beforeDaemonize(): - """ - Hook to be called immediately before daemonization. No reactor methods - may be called until L{afterDaemonize} is called. - - @return: C{None}. - """ - - - def afterDaemonize(): - """ - Hook to be called immediately after daemonization. This may only be - called after L{beforeDaemonize} had been called previously. - - @return: C{None}. - """ - - - -class IReactorFDSet(Interface): - """ - Implement me to be able to use L{IFileDescriptor} type resources. - - This assumes that your main-loop uses UNIX-style numeric file descriptors - (or at least similarly opaque IDs returned from a .fileno() method) - """ - - def addReader(reader): - """ - I add reader to the set of file descriptors to get read events for. - - @param reader: An L{IReadDescriptor} provider that will be checked for - read events until it is removed from the reactor with - L{removeReader}. - - @return: C{None}. - """ - - def addWriter(writer): - """ - I add writer to the set of file descriptors to get write events for. - - @param writer: An L{IWriteDescriptor} provider that will be checked for - write events until it is removed from the reactor with - L{removeWriter}. - - @return: C{None}. - """ - - def removeReader(reader): - """ - Removes an object previously added with L{addReader}. - - @return: C{None}. - """ - - def removeWriter(writer): - """ - Removes an object previously added with L{addWriter}. - - @return: C{None}. - """ - - def removeAll(): - """ - Remove all readers and writers. - - Should not remove reactor internal reactor connections (like a waker). - - @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers - which were removed. - """ - - def getReaders(): - """ - Return the list of file descriptors currently monitored for input - events by the reactor. - - @return: the list of file descriptors monitored for input events. - @rtype: C{list} of C{IReadDescriptor} - """ - - def getWriters(): - """ - Return the list file descriptors currently monitored for output events - by the reactor. - - @return: the list of file descriptors monitored for output events. - @rtype: C{list} of C{IWriteDescriptor} - """ - - -class IListeningPort(Interface): - """ - A listening port. - """ - - def startListening(): - """ - Start listening on this port. - - @raise CannotListenError: If it cannot listen on this port (e.g., it is - a TCP port and it cannot bind to the required - port number). - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return Deferred that fires - upon completion. - """ - - def getHost(): - """ - Get the host that this port is listening for. - - @return: An L{IAddress} provider. - """ - - -class ILoggingContext(Interface): - """ - Give context information that will be used to log events generated by - this item. - """ - - def logPrefix(): - """ - @return: Prefix used during log formatting to indicate context. - @rtype: C{str} - """ - - - -class IFileDescriptor(ILoggingContext): - """ - An interface representing a UNIX-style numeric file descriptor. - """ - - def fileno(): - """ - @raise: If the descriptor no longer has a valid file descriptor - number associated with it. - - @return: The platform-specified representation of a file descriptor - number. Or C{-1} if the descriptor no longer has a valid file - descriptor number associated with it. As long as the descriptor - is valid, calls to this method on a particular instance must - return the same value. - """ - - - def connectionLost(reason): - """ - Called when the connection was lost. - - This is called when the connection on a selectable object has been - lost. It will be called whether the connection was closed explicitly, - an exception occurred in an event handler, or the other end of the - connection closed it first. - - See also L{IHalfCloseableDescriptor} if your descriptor wants to be - notified separately of the two halves of the connection being closed. - - @param reason: A failure instance indicating the reason why the - connection was lost. L{error.ConnectionLost} and - L{error.ConnectionDone} are of special note, but the - failure may be of other classes as well. - """ - - - -class IReadDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can read. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doRead(): - """ - Some data is available for reading on your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IWriteDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can write. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doWrite(): - """ - Some data can be written to your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor): - """ - An L{IFileDescriptor} that can both read and write. - """ - - -class IHalfCloseableDescriptor(Interface): - """ - A descriptor that can be half-closed. - """ - - def writeConnectionLost(reason): - """ - Indicates write connection was lost. - """ - - def readConnectionLost(reason): - """ - Indicates read connection was lost. - """ - - -class ISystemHandle(Interface): - """ - An object that wraps a networking OS-specific handle. - """ - - def getHandle(): - """ - Return a system- and reactor-specific handle. - - This might be a socket.socket() object, or some other type of - object, depending on which reactor is being used. Use and - manipulate at your own risk. - - This might be used in cases where you want to set specific - options not exposed by the Twisted APIs. - """ - - -class IConsumer(Interface): - """ - A consumer consumes data from a producer. - """ - - def registerProducer(producer, streaming): - """ - Register to receive data from a producer. - - This sets self to be a consumer for a producer. When this object runs - out of data (as when a send(2) call on a socket succeeds in moving the - last data from a userspace buffer into a kernelspace buffer), it will - ask the producer to resumeProducing(). - - For L{IPullProducer} providers, C{resumeProducing} will be called once - each time data is required. - - For L{IPushProducer} providers, C{pauseProducing} will be called - whenever the write buffer fills up and C{resumeProducing} will only be - called when it empties. - - @type producer: L{IProducer} provider - - @type streaming: C{bool} - @param streaming: C{True} if C{producer} provides L{IPushProducer}, - C{False} if C{producer} provides L{IPullProducer}. - - @raise RuntimeError: If a producer is already registered. - - @return: C{None} - """ - - - def unregisterProducer(): - """ - Stop consuming data from a producer, without disconnecting. - """ - - - def write(data): - """ - The producer will write data by calling this method. - - The implementation must be non-blocking and perform whatever - buffering is necessary. If the producer has provided enough data - for now and it is a L{IPushProducer}, the consumer may call its - C{pauseProducing} method. - """ - - - -class IProducer(Interface): - """ - A producer produces data for a consumer. - - Typically producing is done by calling the write method of an class - implementing L{IConsumer}. - """ - - def stopProducing(): - """ - Stop producing data. - - This tells a producer that its consumer has died, so it must stop - producing data for good. - """ - - -class IPushProducer(IProducer): - """ - A push producer, also known as a streaming producer is expected to - produce (write to this consumer) data on a continuous basis, unless - it has been paused. A paused push producer will resume producing - after its resumeProducing() method is called. For a push producer - which is not pauseable, these functions may be noops. - """ - - def pauseProducing(): - """ - Pause producing data. - - Tells a producer that it has produced too much data to process for - the time being, and to stop until resumeProducing() is called. - """ - def resumeProducing(): - """ - Resume producing data. - - This tells a producer to re-add itself to the main loop and produce - more data for its consumer. - """ - -class IPullProducer(IProducer): - """ - A pull producer, also known as a non-streaming producer, is - expected to produce data each time resumeProducing() is called. - """ - - def resumeProducing(): - """ - Produce data for the consumer a single time. - - This tells a producer to produce data for the consumer once - (not repeatedly, once only). Typically this will be done - by calling the consumer's write() method a single time with - produced data. - """ - -class IProtocol(Interface): - - def dataReceived(data): - """ - Called whenever data is received. - - Use this method to translate to a higher-level message. Usually, some - callback will be made upon the receipt of each complete protocol - message. - - @param data: a string of indeterminate length. Please keep in mind - that you will probably need to buffer some data, as partial - (or multiple) protocol messages may be received! I recommend - that unit tests for protocols call through to this method with - differing chunk sizes, down to one byte at a time. - """ - - def connectionLost(reason): - """ - Called when the connection is shut down. - - Clear any circular references here, and any external references - to this Protocol. The connection has been closed. The C{reason} - Failure wraps a L{twisted.internet.error.ConnectionDone} or - L{twisted.internet.error.ConnectionLost} instance (or a subclass - of one of those). - - @type reason: L{twisted.python.failure.Failure} - """ - - def makeConnection(transport): - """ - Make a connection to a transport and a server. - """ - - def connectionMade(): - """ - Called when a connection is made. - - This may be considered the initializer of the protocol, because - it is called when the connection is completed. For clients, - this is called once the connection to the server has been - established; for servers, this is called after an accept() call - stops blocking and a socket has been received. If you need to - send any greeting or initial message, do it here. - """ - - -class IProcessProtocol(Interface): - """ - Interface for process-related event handlers. - """ - - def makeConnection(process): - """ - Called when the process has been created. - - @type process: L{IProcessTransport} provider - @param process: An object representing the process which has been - created and associated with this protocol. - """ - - - def childDataReceived(childFD, data): - """ - Called when data arrives from the child process. - - @type childFD: C{int} - @param childFD: The file descriptor from which the data was - received. - - @type data: C{str} - @param data: The data read from the child's file descriptor. - """ - - - def childConnectionLost(childFD): - """ - Called when a file descriptor associated with the child process is - closed. - - @type childFD: C{int} - @param childFD: The file descriptor which was closed. - """ - - - def processExited(reason): - """ - Called when the child process exits. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - - @since: 8.2 - """ - - - def processEnded(reason): - """ - Called when the child process exits and all file descriptors associated - with it have been closed. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - """ - - - -class IHalfCloseableProtocol(Interface): - """ - Implemented to indicate they want notification of half-closes. - - TCP supports the notion of half-closing the connection, e.g. - closing the write side but still not stopping reading. A protocol - that implements this interface will be notified of such events, - instead of having connectionLost called. - """ - - def readConnectionLost(): - """ - Notification of the read connection being closed. - - This indicates peer did half-close of write side. It is now - the responsibility of the this protocol to call - loseConnection(). In addition, the protocol MUST make sure a - reference to it still exists (i.e. by doing a callLater with - one of its methods, etc.) as the reactor will only have a - reference to it if it is writing. - - If the protocol does not do so, it might get garbage collected - without the connectionLost method ever being called. - """ - - def writeConnectionLost(): - """ - Notification of the write connection being closed. - - This will never be called for TCP connections as TCP does not - support notification of this type of half-close. - """ - - - -class IFileDescriptorReceiver(Interface): - """ - Protocols may implement L{IFileDescriptorReceiver} to receive file - descriptors sent to them. This is useful in conjunction with - L{IUNIXTransport}, which allows file descriptors to be sent between - processes on a single host. - """ - def fileDescriptorReceived(descriptor): - """ - Called when a file descriptor is received over the connection. - - @param descriptor: The descriptor which was received. - @type descriptor: C{int} - - @return: C{None} - """ - - - -class IProtocolFactory(Interface): - """ - Interface for protocol factories. - """ - - def buildProtocol(addr): - """ - Called when a connection has been established to addr. - - If None is returned, the connection is assumed to have been refused, - and the Port will close the connection. - - @type addr: (host, port) - @param addr: The address of the newly-established connection - - @return: None if the connection was refused, otherwise an object - providing L{IProtocol}. - """ - - def doStart(): - """ - Called every time this is connected to a Port or Connector. - """ - - def doStop(): - """ - Called every time this is unconnected from a Port or Connector. - """ - - -class ITransport(Interface): - """ - I am a transport for bytes. - - I represent (and wrap) the physical connection and synchronicity - of the framework which is talking to the network. I make no - representations about whether calls to me will happen immediately - or require returning to a control loop, or whether they will happen - in the same or another thread. Consider methods of this class - (aside from getPeer) to be 'thrown over the wall', to happen at some - indeterminate time. - """ - - def write(data): - """ - Write some data to the physical connection, in sequence, in a - non-blocking fashion. - - If possible, make sure that it is all written. No data will - ever be lost, although (obviously) the connection may be closed - before it all gets through. - """ - - def writeSequence(data): - """ - Write a list of strings to the physical connection. - - If possible, make sure that all of the data is written to - the socket at once, without first copying it all into a - single string. - """ - - def loseConnection(): - """ - Close my connection, after writing all pending data. - - Note that if there is a registered producer on a transport it - will not be closed until the producer has been unregistered. - """ - - def getPeer(): - """ - Get the remote address of this connection. - - Treat this method with caution. It is the unfortunate result of the - CGI and Jabber standards, but should not be considered reliable for - the usual host of reasons; port forwarding, proxying, firewalls, IP - masquerading, etc. - - @return: An L{IAddress} provider. - """ - - def getHost(): - """ - Similar to getPeer, but returns an address describing this side of the - connection. - - @return: An L{IAddress} provider. - """ - - -class ITCPTransport(ITransport): - """ - A TCP based transport. - """ - - def loseWriteConnection(): - """ - Half-close the write side of a TCP connection. - - If the protocol instance this is attached to provides - IHalfCloseableProtocol, it will get notified when the operation is - done. When closing write connection, as with loseConnection this will - only happen when buffer has emptied and there is no registered - producer. - """ - - - def abortConnection(): - """ - Close the connection abruptly. - - Discards any buffered data, stops any registered producer, - and, if possible, notifies the other end of the unclean - closure. - - @since: 11.1 - """ - - - def getTcpNoDelay(): - """ - Return if C{TCP_NODELAY} is enabled. - """ - - def setTcpNoDelay(enabled): - """ - Enable/disable C{TCP_NODELAY}. - - Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are - sent sooner, possibly at the expense of overall throughput. - """ - - def getTcpKeepAlive(): - """ - Return if C{SO_KEEPALIVE} is enabled. - """ - - def setTcpKeepAlive(enabled): - """ - Enable/disable C{SO_KEEPALIVE}. - - Enabling C{SO_KEEPALIVE} sends packets periodically when the connection - is otherwise idle, usually once every two hours. They are intended - to allow detection of lost peers in a non-infinite amount of time. - """ - - def getHost(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - def getPeer(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - - -class IUNIXTransport(ITransport): - """ - Transport for stream-oriented unix domain connections. - """ - def sendFileDescriptor(descriptor): - """ - Send a duplicate of this (file, socket, pipe, etc) descriptor to the - other end of this connection. - - The send is non-blocking and will be queued if it cannot be performed - immediately. The send will be processed in order with respect to other - C{sendFileDescriptor} calls on this transport, but not necessarily with - respect to C{write} calls on this transport. The send can only be - processed if there are also bytes in the normal connection-oriented send - buffer (ie, you must call C{write} at least as many times as you call - C{sendFileDescriptor}). - - @param descriptor: An C{int} giving a valid file descriptor in this - process. Note that a I{file descriptor} may actually refer to a - socket, a pipe, or anything else POSIX tries to treat in the same - way as a file. - - @return: C{None} - """ - - - -class ITLSTransport(ITCPTransport): - """ - A TCP transport that supports switching to TLS midstream. - - Once TLS mode is started the transport will implement L{ISSLTransport}. - """ - - def startTLS(contextFactory): - """ - Initiate TLS negotiation. - - @param contextFactory: A context factory (see L{ssl.py}) - """ - -class ISSLTransport(ITCPTransport): - """ - A SSL/TLS based transport. - """ - - def getPeerCertificate(): - """ - Return an object with the peer's certificate info. - """ - - -class IProcessTransport(ITransport): - """ - A process transport. - """ - - pid = Attribute( - "From before L{IProcessProtocol.makeConnection} is called to before " - "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} " - "giving the platform process ID of this process. C{pid} is L{None} " - "at all other times.") - - def closeStdin(): - """ - Close stdin after all data has been written out. - """ - - def closeStdout(): - """ - Close stdout. - """ - - def closeStderr(): - """ - Close stderr. - """ - - def closeChildFD(descriptor): - """ - Close a file descriptor which is connected to the child process, identified - by its FD in the child process. - """ - - def writeToChild(childFD, data): - """ - Similar to L{ITransport.write} but also allows the file descriptor in - the child process which will receive the bytes to be specified. - - @type childFD: C{int} - @param childFD: The file descriptor to which to write. - - @type data: C{str} - @param data: The bytes to write. - - @return: C{None} - - @raise KeyError: If C{childFD} is not a file descriptor that was mapped - in the child when L{IReactorProcess.spawnProcess} was used to create - it. - """ - - def loseConnection(): - """ - Close stdin, stderr and stdout. - """ - - def signalProcess(signalID): - """ - Send a signal to the process. - - @param signalID: can be - - one of C{"KILL"}, C{"TERM"}, or C{"INT"}. - These will be implemented in a - cross-platform manner, and so should be used - if possible. - - an integer, where it represents a POSIX - signal ID. - - @raise twisted.internet.error.ProcessExitedAlready: If the process has - already exited. - @raise OSError: If the C{os.kill} call fails with an errno different - from C{ESRCH}. - """ - - -class IServiceCollection(Interface): - """ - An object which provides access to a collection of services. - """ - - def getServiceNamed(serviceName): - """ - Retrieve the named service from this application. - - Raise a C{KeyError} if there is no such service name. - """ - - def addService(service): - """ - Add a service to this collection. - """ - - def removeService(service): - """ - Remove a service from this collection. - """ - - -class IUDPTransport(Interface): - """ - Transport for UDP DatagramProtocols. - """ - - def write(packet, addr=None): - """ - Write packet to given address. - - @param addr: a tuple of (ip, port). For connected transports must - be the address the transport is connected to, or None. - In non-connected mode this is mandatory. - - @raise twisted.internet.error.MessageLengthError: C{packet} was too - long. - """ - - def connect(host, port): - """ - Connect the transport to an address. - - This changes it to connected mode. Datagrams can only be sent to - this address, and will only be received from this address. In addition - the protocol's connectionRefused method might get called if destination - is not receiving datagrams. - - @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost') - @param port: port to connect to. - """ - - def getHost(): - """ - Returns L{IPv4Address}. - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return L{Deferred} that fires - upon completion. - """ - - - -class IUNIXDatagramTransport(Interface): - """ - Transport for UDP PacketProtocols. - """ - - def write(packet, address): - """ - Write packet to given address. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - -class IUNIXDatagramConnectedTransport(Interface): - """ - Transport for UDP ConnectedPacketProtocols. - """ - - def write(packet): - """ - Write packet to address we are connected to. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - def getPeer(): - """ - Returns L{UNIXAddress}. - """ - - -class IMulticastTransport(Interface): - """ - Additional functionality for multicast UDP. - """ - - def getOutgoingInterface(): - """ - Return interface of outgoing multicast packets. - """ - - def setOutgoingInterface(addr): - """ - Set interface for outgoing multicast packets. - - Returns Deferred of success. - """ - - def getLoopbackMode(): - """ - Return if loopback mode is enabled. - """ - - def setLoopbackMode(mode): - """ - Set if loopback mode is enabled. - """ - - def getTTL(): - """ - Get time to live for multicast packets. - """ - - def setTTL(ttl): - """ - Set time to live on multicast packets. - """ - - def joinGroup(addr, interface=""): - """ - Join a multicast group. Returns L{Deferred} of success or failure. - - If an error occurs, the returned L{Deferred} will fail with - L{error.MulticastJoinError}. - """ - - def leaveGroup(addr, interface=""): - """ - Leave multicast group, return L{Deferred} of success. - """ - - -class IStreamClientEndpoint(Interface): - """ - A stream client endpoint is a place that L{ClientFactory} can connect to. - For example, a remote TCP host/port pair would be a TCP client endpoint. - - @since: 10.1 - """ - - def connect(protocolFactory): - """ - Connect the C{protocolFactory} to the location specified by this - L{IStreamClientEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IProtocol} upon successful - connection otherwise a L{ConnectError} - """ - - - -class IStreamServerEndpoint(Interface): - """ - A stream server endpoint is a place that a L{Factory} can listen for - incoming connections. - - @since: 10.1 - """ - - def listen(protocolFactory): - """ - Listen with C{protocolFactory} at the location specified by this - L{IStreamServerEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IListeningPort} or an - L{CannotListenError} - """ - - - -class IStreamServerEndpointStringParser(Interface): - """ - An L{IStreamServerEndpointStringParser} is like an - L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s - instead of clients. It integrates with L{endpoints.serverFromString} in - much the same way. - """ - - prefix = Attribute( - """ - @see: L{IStreamClientEndpointStringParser.prefix} - """ - ) - - - def parseStreamServer(reactor, *args, **kwargs): - """ - Parse a stream server endpoint from a reactor and string-only arguments - and keyword arguments. - - @see: L{IStreamClientEndpointStringParser.parseStreamClient} - - @return: a stream server endpoint - @rtype: L{IStreamServerEndpoint} - """ - - - -class IStreamClientEndpointStringParser(Interface): - """ - An L{IStreamClientEndpointStringParser} is a parser which can convert - a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint} - provider. - - This interface is really only useful in the context of the plugin system - for L{endpoints.clientFromString}. See the document entitled "I{The - Twisted Plugin System}" for more details on how to write a plugin. - - If you place an L{IStreamClientEndpointStringParser} plugin in the - C{twisted.plugins} package, that plugin's C{parseStreamClient} method will - be used to produce endpoints for any description string that begins with - the result of that L{IStreamClientEndpointStringParser}'s prefix attribute. - """ - - prefix = Attribute( - """ - A C{str}, the description prefix to respond to. For example, an - L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its - C{prefix} attribute would be called for endpoint descriptions like - C{"foo:bar:baz"} or C{"foo:"}. - """ - ) - - - def parseStreamClient(*args, **kwargs): - """ - This method is invoked by L{endpoints.clientFromString}, if the type of - endpoint matches the return value from this - L{IStreamClientEndpointStringParser}'s C{prefix} method. - - @param args: The string arguments, minus the endpoint type, in the - endpoint description string, parsed according to the rules - described in L{endpoints.quoteStringArgument}. For example, if the - description were C{"my-type:foo:bar:baz=qux"}, C{args} would be - C{('foo','bar')} - - @param kwargs: The string arguments from the endpoint description - passed as keyword arguments. For example, if the description were - C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be - C{dict(baz='qux')}. - - @return: a client endpoint - @rtype: L{IStreamClientEndpoint} - """ -}}} ''' diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index 32c88ff2d..fd814dc22 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -10,580 +10,11 @@ Interface definitions for L{twisted.web}. body is not known in advance. """ -from zope.interface import Interface, Attribute - -#from twisted.internet.interfaces import IPushProducer - from twisted.web.iweb import ( IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, ) -''' {{{ -class IRequest(Interface): - """ - An HTTP request. - - @since: 9.0 - """ - - method = Attribute("A C{str} giving the HTTP method that was used.") - uri = Attribute( - "A C{str} giving the full encoded URI which was requested (including " - "query arguments).") - path = Attribute( - "A C{str} giving the encoded query path of the request URI.") - args = Attribute( - "A mapping of decoded query argument names as C{str} to " - "corresponding query argument values as C{list}s of C{str}. " - "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} " - "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], " - "'quux': ['spam']}}.") - - received_headers = Attribute( - "Backwards-compatibility access to C{requestHeaders}. Use " - "C{requestHeaders} instead. C{received_headers} behaves mostly " - "like a C{dict} and does not provide access to all header values.") - - requestHeaders = Attribute( - "A L{http_headers.Headers} instance giving all received HTTP request " - "headers.") - - content = Attribute( - "A file-like object giving the request body. This may be a file on " - "disk, a C{StringIO}, or some other type. The implementation is free " - "to decide on a per-request basis.") - - headers = Attribute( - "Backwards-compatibility access to C{responseHeaders}. Use" - "C{responseHeaders} instead. C{headers} behaves mostly like a " - "C{dict} and does not provide access to all header values nor " - "does it allow multiple values for one header to be set.") - - responseHeaders = Attribute( - "A L{http_headers.Headers} instance holding all HTTP response " - "headers to be sent.") - - def getHeader(key): - """ - Get an HTTP request header. - - @type key: C{str} - @param key: The name of the header to get the value of. - - @rtype: C{str} or C{NoneType} - @return: The value of the specified header, or C{None} if that header - was not present in the request. - """ - - - def getCookie(key): - """ - Get a cookie that was sent from the network. - """ - - - def getAllHeaders(): - """ - Return dictionary mapping the names of all received headers to the last - value received for each. - - Since this method does not return all header information, - C{requestHeaders.getAllRawHeaders()} may be preferred. - """ - - - def getRequestHostname(): - """ - Get the hostname that the user passed in to the request. - - This will either use the Host: header (if it is available) or the - host we are listening on if the header is unavailable. - - @returns: the requested hostname - @rtype: C{str} - """ - - - def getHost(): - """ - Get my originally requesting transport's host. - - @return: An L{IAddress}. - """ - - - def getClientIP(): - """ - Return the IP address of the client who submitted this request. - - @returns: the client IP address or C{None} if the request was submitted - over a transport where IP addresses do not make sense. - @rtype: L{str} or C{NoneType} - """ - - - def getClient(): - """ - Return the hostname of the IP address of the client who submitted this - request, if possible. - - This method is B{deprecated}. See L{getClientIP} instead. - - @rtype: C{NoneType} or L{str} - @return: The canonical hostname of the client, as determined by - performing a name lookup on the IP address of the client. - """ - - - def getUser(): - """ - Return the HTTP user sent with this request, if any. - - If no user was supplied, return the empty string. - - @returns: the HTTP user, if any - @rtype: C{str} - """ - - - def getPassword(): - """ - Return the HTTP password sent with this request, if any. - - If no password was supplied, return the empty string. - - @returns: the HTTP password, if any - @rtype: C{str} - """ - - - def isSecure(): - """ - Return True if this request is using a secure transport. - - Normally this method returns True if this request's HTTPChannel - instance is using a transport that implements ISSLTransport. - - This will also return True if setHost() has been called - with ssl=True. - - @returns: True if this request is secure - @rtype: C{bool} - """ - - - def getSession(sessionInterface=None): - """ - Look up the session associated with this request or create a new one if - there is not one. - - @return: The L{Session} instance identified by the session cookie in - the request, or the C{sessionInterface} component of that session - if C{sessionInterface} is specified. - """ - - - def URLPath(): - """ - @return: A L{URLPath} instance which identifies the URL for which this - request is. - """ - - - def prePathURL(): - """ - @return: At any time during resource traversal, a L{str} giving an - absolute URL to the most nested resource which has yet been - reached. - """ - - - def rememberRootURL(): - """ - Remember the currently-processed part of the URL for later - recalling. - """ - - - def getRootURL(): - """ - Get a previously-remembered URL. - """ - - - # Methods for outgoing response - def finish(): - """ - Indicate that the response to this request is complete. - """ - - - def write(data): - """ - Write some data to the body of the response to this request. Response - headers are written the first time this method is called, after which - new response headers may not be added. - """ - - - def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): - """ - Set an outgoing HTTP cookie. - - In general, you should consider using sessions instead of cookies, see - L{twisted.web.server.Request.getSession} and the - L{twisted.web.server.Session} class for details. - """ - - - def setResponseCode(code, message=None): - """ - Set the HTTP response code. - """ - - - def setHeader(k, v): - """ - Set an HTTP response header. Overrides any previously set values for - this header. - - @type name: C{str} - @param name: The name of the header for which to set the value. - - @type value: C{str} - @param value: The value to set for the named header. - """ - - - def redirect(url): - """ - Utility function that does a redirect. - - The request should have finish() called after this. - """ - - - def setLastModified(when): - """ - Set the C{Last-Modified} time for the response to this request. - - If I am called more than once, I ignore attempts to set Last-Modified - earlier, only replacing the Last-Modified time if it is to a later - value. - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} if appropriate for the time given. - - @param when: The last time the resource being returned was modified, in - seconds since the epoch. - @type when: L{int}, L{long} or L{float} - - @return: If I am a C{If-Modified-Since} conditional request and the time - given is not newer than the condition, I return - L{CACHED} to indicate that you should write no body. - Otherwise, I return a false value. - """ - - - def setETag(etag): - """ - Set an C{entity tag} for the outgoing response. - - That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for - comparing two or more entities from the same requested resource." - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} or - L{PRECONDITION_FAILED}, if appropriate for the - tag given. - - @param etag: The entity tag for the resource being returned. - @type etag: C{str} - - @return: If I am a C{If-None-Match} conditional request and the tag - matches one in the request, I return L{CACHED} to - indicate that you should write no body. Otherwise, I return a - false value. - """ - - - def setHost(host, port, ssl=0): - """ - Change the host and port the request thinks it's using. - - This method is useful for working with reverse HTTP proxies (e.g. both - Squid and Apache's mod_proxy can do this), when the address the HTTP - client is using is different than the one we're listening on. - - For example, Apache may be listening on https://www.example.com, and - then forwarding requests to http://localhost:8080, but we don't want - HTML produced by Twisted to say 'http://localhost:8080', they should - say 'https://www.example.com', so we do:: - - request.setHost('www.example.com', 443, ssl=1) - """ - - - -class ICredentialFactory(Interface): - """ - A credential factory defines a way to generate a particular kind of - authentication challenge and a way to interpret the responses to these - challenges. It creates - L{ICredentials} providers from - responses. These objects will be used with L{twisted.cred} to authenticate - an authorize requests. - """ - scheme = Attribute( - "A C{str} giving the name of the authentication scheme with which " - "this factory is associated. For example, C{'basic'} or C{'digest'}.") - - - def getChallenge(request): - """ - Generate a new challenge to be sent to a client. - - @type peer: L{twisted.web.http.Request} - @param peer: The request the response to which this challenge will be - included. - - @rtype: C{dict} - @return: A mapping from C{str} challenge fields to associated C{str} - values. - """ - - - def decode(response, request): - """ - Create a credentials object from the given response. - - @type response: C{str} - @param response: scheme specific response string - - @type request: L{twisted.web.http.Request} - @param request: The request being processed (from which the response - was taken). - - @raise twisted.cred.error.LoginFailed: If the response is invalid. - - @rtype: L{twisted.cred.credentials.ICredentials} provider - @return: The credentials represented by the given response. - """ - - - -class IBodyProducer(IPushProducer): - """ - Objects which provide L{IBodyProducer} write bytes to an object which - provides L{IConsumer} by calling its - C{write} method repeatedly. - - L{IBodyProducer} providers may start producing as soon as they have an - L{IConsumer} provider. That is, they - should not wait for a C{resumeProducing} call to begin writing data. - - L{IConsumer.unregisterProducer} - must not be called. Instead, the - L{Deferred} returned from C{startProducing} - must be fired when all bytes have been written. - - L{IConsumer.write} may - synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or - C{stopProducing}. These methods must be implemented with this in mind. - - @since: 9.0 - """ - - # Despite the restrictions above and the additional requirements of - # stopProducing documented below, this interface still needs to be an - # IPushProducer subclass. Providers of it will be passed to IConsumer - # providers which only know about IPushProducer and IPullProducer, not - # about this interface. This interface needs to remain close enough to one - # of those interfaces for consumers to work with it. - - length = Attribute( - """ - C{length} is a C{int} indicating how many bytes in total this - L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH} - if this is not known in advance. - """) - - def startProducing(consumer): - """ - Start producing to the given - L{IConsumer} provider. - - @return: A L{Deferred} which fires with - C{None} when all bytes have been produced or with a - L{Failure} if there is any problem - before all bytes have been produced. - """ - - - def stopProducing(): - """ - In addition to the standard behavior of - L{IProducer.stopProducing} - (stop producing data), make sure the - L{Deferred} returned by - C{startProducing} is never fired. - """ - - - -class IRenderable(Interface): - """ - An L{IRenderable} is an object that may be rendered by the - L{twisted.web.template} templating system. - """ - - def lookupRenderMethod(name): - """ - Look up and return the render method associated with the given name. - - @type name: C{str} - @param name: The value of a render directive encountered in the - document returned by a call to L{IRenderable.render}. - - @return: A two-argument callable which will be invoked with the request - being responded to and the tag object on which the render directive - was encountered. - """ - - - def render(request): - """ - Get the document for this L{IRenderable}. - - @type request: L{IRequest} provider or C{NoneType} - @param request: The request in response to which this method is being - invoked. - - @return: An object which can be flattened. - """ - - - -class ITemplateLoader(Interface): - """ - A loader for templates; something usable as a value for - L{twisted.web.template.Element}'s C{loader} attribute. - """ - - def load(): - """ - Load a template suitable for rendering. - - @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and - other L{IRenderable} providers. - """ - - - -class IResponse(Interface): - """ - An object representing an HTTP response received from an HTTP server. - - @since: 11.1 - """ - - version = Attribute( - "A three-tuple describing the protocol and protocol version " - "of the response. The first element is of type C{str}, the second " - "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.") - - - code = Attribute("The HTTP status code of this response, as a C{int}.") - - - phrase = Attribute( - "The HTTP reason phrase of this response, as a C{str}.") - - - headers = Attribute("The HTTP response L{Headers} of this response.") - - - length = Attribute( - "The C{int} number of bytes expected to be in the body of this " - "response or L{UNKNOWN_LENGTH} if the server did not indicate how " - "many bytes to expect. For I{HEAD} responses, this will be 0; if " - "the response includes a I{Content-Length} header, it will be " - "available in C{headers}.") - - - def deliverBody(protocol): - """ - Register an L{IProtocol} provider - to receive the response body. - - The protocol will be connected to a transport which provides - L{IPushProducer}. The protocol's C{connectionLost} method will be - called with: - - - ResponseDone, which indicates that all bytes from the response - have been successfully delivered. - - - PotentialDataLoss, which indicates that it cannot be determined - if the entire response body has been delivered. This only occurs - when making requests to HTTP servers which do not set - I{Content-Length} or a I{Transfer-Encoding} in the response. - - - ResponseFailed, which indicates that some bytes from the response - were lost. The C{reasons} attribute of the exception may provide - more specific indications as to why. - """ - - - -class _IRequestEncoder(Interface): - """ - An object encoding data passed to L{IRequest.write}, for example for - compression purpose. - - @since: 12.3 - """ - - def encode(data): - """ - Encode the data given and return the result. - - @param data: The content to encode. - @type data: C{str} - - @return: The encoded data. - @rtype: C{str} - """ - - - def finish(): - """ - Callback called when the request is closing. - - @return: If necessary, the pending data accumulated from previous - C{encode} calls. - @rtype: C{str} - """ - - - -class _IRequestEncoderFactory(Interface): - """ - A factory for returing L{_IRequestEncoder} instances. - - @since: 12.3 - """ - - def encoderForRequest(request): - """ - If applicable, returns a L{_IRequestEncoder} instance which will encode - the request. - """ - - - -UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" -}}} ''' __all__ = [ "ICredentialFactory", "IRequest", "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", From 534772f6ea8389ec4f51d5ea4c4572080a3d3d7b Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 2 Dec 2016 01:15:37 +0000 Subject: [PATCH 0576/3444] Import xlib.tx code from twisted proper --- scrapy/core/downloader/handlers/http11.py | 5 +++-- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- tests/test_downloadermiddleware_retry.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 404e9160b..54aa359fb 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -13,8 +13,9 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import PotentialDataLoss -from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, TCP4ClientEndpoint +from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ + HTTPConnectionPool +from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 521327bfe..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -3,10 +3,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy import signals from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -from scrapy.xlib.tx import ResponseFailed class HttpCacheMiddleware(object): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 74938067f..c9c512be8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -17,10 +17,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message -from scrapy.xlib.tx import ResponseFailed from scrapy.core.downloader.handlers.http11 import TunnelError logger = logging.getLogger(__name__) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 3de9399cf..eb17974bf 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -3,10 +3,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.xlib.tx import ResponseFailed from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler From 67cf64edbee99287bc8f663be033d1343e3a77ae Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 2 Dec 2016 20:53:06 +0000 Subject: [PATCH 0577/3444] Deprecate scrapy.xlib.tx --- scrapy/xlib/tx/__init__.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index 1c9bf09e5..0d94307b7 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -1,19 +1,10 @@ -from scrapy import twisted_version -if twisted_version > (13, 0, 0): - from twisted.web import client - from twisted.internet import endpoints -if twisted_version >= (11, 1, 0): - from . import client, endpoints -else: - from scrapy.exceptions import NotSupported - class _Mocked(object): - def __init__(self, *args, **kw): - raise NotSupported('HTTP1.1 not supported') - class _Mock(object): - def __getattr__(self, name): - return _Mocked - client = endpoints = _Mock() +from __future__ import absolute_import +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning + +from twisted.web import client +from twisted.internet import endpoints Agent = client.Agent # since < 11.1 ProxyAgent = client.ProxyAgent # since 11.1 @@ -21,3 +12,8 @@ ResponseDone = client.ResponseDone # since 11.1 ResponseFailed = client.ResponseFailed # since 11.1 HTTPConnectionPool = client.HTTPConnectionPool # since 12.1 TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1 + +warnings.warn("Importing from scrapy.xlib.tx is deprecated and will" + " no longer be supported in future Scrapy versions." + " Update your code to import from twisted proper.", + ScrapyDeprecationWarning, stacklevel=2) From 67bc2e0b18d990f08bfeb913be0c5bd7f299015a Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 2 Dec 2016 21:00:39 +0000 Subject: [PATCH 0578/3444] Wipe scrapy.xlib.tx --- scrapy/xlib/{tx/__init__.py => tx.py} | 0 scrapy/xlib/tx/LICENSE | 57 -------------------------- scrapy/xlib/tx/README | 2 - scrapy/xlib/tx/_newclient.py | 59 --------------------------- scrapy/xlib/tx/client.py | 58 -------------------------- scrapy/xlib/tx/endpoints.py | 26 ------------ scrapy/xlib/tx/interfaces.py | 28 ------------- scrapy/xlib/tx/iweb.py | 23 ----------- 8 files changed, 253 deletions(-) rename scrapy/xlib/{tx/__init__.py => tx.py} (100%) delete mode 100644 scrapy/xlib/tx/LICENSE delete mode 100644 scrapy/xlib/tx/README delete mode 100644 scrapy/xlib/tx/_newclient.py delete mode 100644 scrapy/xlib/tx/client.py delete mode 100644 scrapy/xlib/tx/endpoints.py delete mode 100644 scrapy/xlib/tx/interfaces.py delete mode 100644 scrapy/xlib/tx/iweb.py diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx.py similarity index 100% rename from scrapy/xlib/tx/__init__.py rename to scrapy/xlib/tx.py diff --git a/scrapy/xlib/tx/LICENSE b/scrapy/xlib/tx/LICENSE deleted file mode 100644 index 8529f6edf..000000000 --- a/scrapy/xlib/tx/LICENSE +++ /dev/null @@ -1,57 +0,0 @@ -Copyright (c) 2001-2013 -Allen Short -Andy Gayton -Andrew Bennetts -Antoine Pitrou -Apple Computer, Inc. -Benjamin Bruheim -Bob Ippolito -Canonical Limited -Christopher Armstrong -David Reid -Donovan Preston -Eric Mangold -Eyal Lotem -Itamar Turner-Trauring -James Knight -Jason A. Mobarak -Jean-Paul Calderone -Jessica McKellar -Jonathan Jacobs -Jonathan Lange -Jonathan D. Simms -Jürgen Hermann -Kevin Horn -Kevin Turner -Mary Gardiner -Matthew Lefkowitz -Massachusetts Institute of Technology -Moshe Zadka -Paul Swartz -Pavel Pergamenshchik -Ralph Meijer -Sean Riley -Software Freedom Conservancy -Travis B. Hartwell -Thijs Triemstra -Thomas Herve -Timothy Allen - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scrapy/xlib/tx/README b/scrapy/xlib/tx/README deleted file mode 100644 index 75ef485ce..000000000 --- a/scrapy/xlib/tx/README +++ /dev/null @@ -1,2 +0,0 @@ -This source files are adapted copies from Twisted trunk to support HTTP1.1 -handler under Twisted >= 11.1 and Twisted <= 13.0.0 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py deleted file mode 100644 index 39cd20f95..000000000 --- a/scrapy/xlib/tx/_newclient.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- test-case-name: twisted.web.test.test_newclient -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -An U{HTTP 1.1} client. - -The way to use the functionality provided by this module is to: - - - Connect a L{HTTP11ClientProtocol} to an HTTP server - - Create a L{Request} with the appropriate data - - Pass the request to L{HTTP11ClientProtocol.request} - - The returned Deferred will fire with a L{Response} object - - Create a L{IProtocol} provider which can handle the response body - - Connect it to the response with L{Response.deliverBody} - - When the protocol's C{connectionLost} method is called, the response is - complete. See L{Response.deliverBody} for details. - -Various other classes in this module support this usage: - - - HTTPParser is the basic HTTP parser. It can handle the parts of HTTP which - are symmetric between requests and responses. - - - HTTPClientParser extends HTTPParser to handle response-specific parts of - HTTP. One instance is created for each request to parse the corresponding - response. -""" - -__metaclass__ = type - -from zope.interface import implements - -from twisted.python import log -from twisted.python.reflect import fullyQualifiedName -from twisted.python.failure import Failure -from twisted.internet.interfaces import IConsumer, IPushProducer -from twisted.internet.error import ConnectionDone -from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred -from twisted.internet.defer import CancelledError -from twisted.internet.protocol import Protocol -from twisted.web.iweb import UNKNOWN_LENGTH, IResponse -from twisted.web.http import NO_CONTENT, NOT_MODIFIED -from twisted.web.http import _DataLoss, PotentialDataLoss -from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder - -from twisted.web._newclient import ( - BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, - RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted, - WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent, - ResponseNeverReceived, HTTPParser, HTTPClientParser, Request, - LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder, - TransportProxyProducer, HTTP11ClientProtocol -) - -# States HTTPParser can be in -STATUS = 'STATUS' -HEADER = 'HEADER' -BODY = 'BODY' -DONE = 'DONE' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py deleted file mode 100644 index c2d50648a..000000000 --- a/scrapy/xlib/tx/client.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -HTTP client. -""" - -from __future__ import division, absolute_import - -import os - -try: - from urlparse import urlunparse - from urllib import splithost, splittype -except ImportError: - from urllib.parse import splithost, splittype - from urllib.parse import urlunparse as _urlunparse - - def urlunparse(parts): - result = _urlunparse(tuple([p.decode("charmap") for p in parts])) - return result.encode("charmap") -import zlib - -from zope.interface import implementer - -from twisted.python import log -from twisted.python.failure import Failure -from twisted.web import http -from twisted.internet import defer, protocol, task, reactor -from twisted.internet.interfaces import IProtocol -from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from twisted.python import failure -from twisted.python.components import proxyForInterface -from twisted.web import error -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse -from twisted.web.http_headers import Headers - -from twisted.web.client import ( - PartialDownloadError, FileBodyProducer, - CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, - Agent, ProxyAgent, HTTPConnectionPool, readBody, -) - - -# The code which follows is based on the new HTTP client implementation. It -# should be significantly better than anything above, though it is not yet -# feature equivalent. - -from twisted.web._newclient import Response -from twisted.web._newclient import ResponseDone, ResponseFailed - - -__all__ = [ - 'PartialDownloadError', - 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', - 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', - 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py deleted file mode 100644 index 197e43ed3..000000000 --- a/scrapy/xlib/tx/endpoints.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- test-case-name: twisted.internet.test.test_endpoints -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that -wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces. - -This also implements an extensible mini-language for describing endpoints, -parsed by the L{clientFromString} and L{serverFromString} functions. - -@since: 10.1 -""" - -from __future__ import division, absolute_import - -from twisted.internet.endpoints import ( - clientFromString, serverFromString, quoteStringArgument, - TCP4ServerEndpoint, TCP6ServerEndpoint, - TCP4ClientEndpoint, TCP6ClientEndpoint, - UNIXServerEndpoint, UNIXClientEndpoint, - SSL4ServerEndpoint, SSL4ClientEndpoint, - AdoptedStreamServerEndpoint, connectProtocol, -) - -__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py deleted file mode 100644 index fdcbf3977..000000000 --- a/scrapy/xlib/tx/interfaces.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Interface documentation. - -Maintainer: Itamar Shtull-Trauring -""" - -from __future__ import division, absolute_import - -from twisted.internet.interfaces import ( - IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, - IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, - IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, - IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet, - IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, - IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, - ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, - IProtocol, IProcessProtocol, IHalfCloseableProtocol, - IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport, - IUNIXTransport, - ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection, - IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, - IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint, - IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, - IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver -) diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py deleted file mode 100644 index fd814dc22..000000000 --- a/scrapy/xlib/tx/iweb.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- test-case-name: twisted.web.test -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Interface definitions for L{twisted.web}. - -@var UNKNOWN_LENGTH: An opaque object which may be used as the value of - L{IBodyProducer.length} to indicate that the length of the entity - body is not known in advance. -""" - -from twisted.web.iweb import ( - IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, - IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, -) - -__all__ = [ - "ICredentialFactory", "IRequest", - "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", - "_IRequestEncoderFactory", - - "UNKNOWN_LENGTH"] From c58ea021b816f20fda1b522f80d8b6a1c22f2454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Sun, 4 Dec 2016 11:56:14 -0300 Subject: [PATCH 0579/3444] fixes docs --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 18d2f8084..2380a340c 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -87,13 +87,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: list + :type to: str or iterable :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: list + :type cc: str or iterable :param body: the e-mail body :type body: str From a4178f99daf69e930fcc635421cc3a9ba519e36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Mon, 5 Dec 2016 15:24:26 -0300 Subject: [PATCH 0580/3444] fixes params types in docs. --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 2380a340c..f68448276 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -87,13 +87,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: str or iterable + :type to: str or list of str :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: str or iterable + :type cc: str or list of str :param body: the e-mail body :type body: str From c08d278c0c7549b3377df995cdf724e3fbb8c02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Guimar=C3=A3es?= Date: Mon, 5 Dec 2016 16:47:24 -0300 Subject: [PATCH 0581/3444] removes note from docs. --- docs/topics/email.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index f68448276..aac93a91a 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -35,12 +35,6 @@ And here is how to use it to send an e-mail (without attachments):: mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"]) -.. note:: - As shown in the example above, ``to`` and ``cc`` need to be lists - of email addresses, not single addresses, and even for one recipient, - i.e. ``to="someone@example.com"`` will not work. - - MailSender class reference ========================== From 89d5f5acd3627cef040c421fce516bb87cfd4b5e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Dec 2016 22:42:16 +0100 Subject: [PATCH 0582/3444] Update changelog for upcoming 1.2.2 release --- docs/news.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c302d2e17..db5856d31 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,41 @@ Release notes ============= +Scrapy 1.2.2 (2016-12-XX) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Fix a cryptic traceback when a pipeline fails on ``open_spider()`` (:issue:`2011`) +- Fix embedded IPython shell variables (fixing :issue:`396` that re-appeared + in 1.2.0, fixed in :issue:`2418`) +- A couple of patches when dealing with robots.txt: + + - handle (non-standard) relative sitemap URLs (:issue:`2390`) + - handle non-ASCII URLs and User-Agents in Python 2 (:issue:`2373`) + +Documentation +~~~~~~~~~~~~~ + +- Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`) +- Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`) +- A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`) + and clarifications (:issue:`2354`, :issue:`2325`) + +Other changes +~~~~~~~~~~~~~ + +- Advertize `conda-forge`_ as Scrapy's official conda channel (:issue:`2387`) +- More helpful error messages when trying to use ``.css()`` or ``.xpath()`` + on non-Text Responses (:issue:`2264`) +- ``startproject`` command now generates a sample ``middlewares.py`` file (:issue:`2335`) +- Add more dependencies' version info in ``scrapy version`` verbose output (:issue:`2404`) +- Remove all ``*.pyc`` files from source distribution (:issue:`2386`) + +.. _conda-forge: https://anaconda.org/conda-forge/scrapy + + Scrapy 1.2.1 (2016-10-21) ------------------------- From aa2e1b030d717951475c3e4d06fcb447ab1e1181 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 14:44:19 +0100 Subject: [PATCH 0583/3444] Add reference to fixed scheduler settings documentation --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index db5856d31..3c75a0228 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -23,7 +23,7 @@ Documentation - Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`) - Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`) - A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`) - and clarifications (:issue:`2354`, :issue:`2325`) + and clarifications (:issue:`2354`, :issue:`2325`, :issue:`2414`) Other changes ~~~~~~~~~~~~~ From 09e310d0b7db40a244cf3508c722ab251ac452eb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 15:03:38 +0100 Subject: [PATCH 0584/3444] Set release date for 1.2.2 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 3c75a0228..5e28fb130 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.2.2 (2016-12-XX) +Scrapy 1.2.2 (2016-12-06) ------------------------- Bug fixes From f3d599532943ebf6a4639e9e99781c2f51cd24c5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 15:21:00 +0100 Subject: [PATCH 0585/3444] =?UTF-8?q?Bump=20version:=201.2.1=20=E2=86=92?= =?UTF-8?q?=201.2.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ffc933d13..ee039790d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.1 +current_version = 1.2.2 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 6085e9465..23aa83906 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.2.1 +1.2.2 From 5efd65255c88ebb156b07f5f09a0a0fdb66f8e7e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Dec 2016 18:49:53 +0100 Subject: [PATCH 0586/3444] TST: Randomize IMAGES_EXPIRES above 90 days --- tests/test_pipeline_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6c1976b63..342f25ea9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -244,7 +244,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { - "IMAGES_EXPIRES": random.randint(1, 1000), + "IMAGES_EXPIRES": random.randint(100, 1000), "IMAGES_STORE": self.tempdir, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), From 778bed07bf771dd3942ea8cd51b7944065f4e2cd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 17:56:13 +0100 Subject: [PATCH 0587/3444] Let framework handle only HTTP redirects by default for fetch and shell commands --- scrapy/commands/fetch.py | 11 ++++++++--- scrapy/commands/shell.py | 6 +++--- scrapy/shell.py | 12 ++++++++---- scrapy/utils/datatypes.py | 10 ++++++++++ tests/test_command_fetch.py | 2 +- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a157b19f8..6fe6d73b9 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -27,8 +28,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") - parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ - default=False, help="do not handle status codes like redirects and print response as-is") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -52,7 +53,11 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - if opts.no_status_aware: + # by default, let the framework handle redirects, + # i.e. command handles all codes expect 3xx + if not opts.no_redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400)) + else: request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index bc0203d89..40a58d94a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,8 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ - default=False, help="do not transparently handle status codes like redirects") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -70,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url, handle_statuses=opts.no_status_aware) + shell.start(url=url, redirect=not opts.no_redirect) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 966003f17..6c78722be 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,6 +7,7 @@ from __future__ import print_function import os import signal +from six.moves import range import warnings from twisted.internet import reactor, threads, defer @@ -20,6 +21,7 @@ from scrapy.item import BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.console import start_python_console +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.conf import get_config @@ -40,11 +42,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): + def start(self, url=None, request=None, response=None, spider=None, redirect=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider, handle_statuses=handle_statuses) + self.fetch(url, spider, redirect=redirect) elif request: self.fetch(request, spider) elif response: @@ -98,13 +100,15 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): + def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): if isinstance(request_or_url, Request): request = request_or_url else: url = any_to_uri(request_or_url) request = Request(url, dont_filter=True, **kwargs) - if handle_statuses: + if redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: request.meta['handle_httpstatus_all'] = True response = None try: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d04b43176..e516185bd 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -304,3 +304,13 @@ class LocalCache(OrderedDict): while len(self) >= self.limit: self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) + + +class SequenceExclude(object): + """Object to test if an item is NOT within some sequence.""" + + def __init__(self, seq): + self.seq = seq + + def __contains__(self, item): + return item not in self.seq diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 45d03a129..3fa3ed930 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -21,7 +21,7 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_redirect_disabled(self): - _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')]) + _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')]) err = err.strip() self.assertIn(b'downloader/response_status_count/302', err, err) self.assertNotIn(b'downloader/response_status_count/200', err, err) From 7e54de24550df658690277839cbee99c9afe4bc8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 18:41:24 +0100 Subject: [PATCH 0588/3444] Add tests for shell command with and without --no-redirect --- tests/test_command_shell.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7bb7439d6..ee6e8ad8e 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -49,6 +49,16 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) assert out.strip().endswith(b'/redirected') + @defer.inlineCallbacks + def test_redirect_follow_302(self): + _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'200') + + @defer.inlineCallbacks + def test_redirect_not_follow_302(self): + _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'302') + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') From 2cd579a7748d0c37eac557216b857c38fbcf80df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 19:07:32 +0100 Subject: [PATCH 0589/3444] Add test for fetch(url) within shell with and without redirect --- tests/test_command_shell.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index ee6e8ad8e..3e27d6abd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -59,6 +59,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) assert out.strip().endswith(b'302') + @defer.inlineCallbacks + def test_fetch_redirect_follow_302(self): + """Test that calling `fetch(url)` follows HTTP redirects by default.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}')" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Redirecting (302)' in errout + assert b'Crawled (200)' in errout + + @defer.inlineCallbacks + def test_fetch_redirect_not_follow_302(self): + """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Crawled (302)' in errout + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') From 948e3cd00328f9a23410b3a7197975f56efadd16 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 8 Dec 2016 12:50:26 +0100 Subject: [PATCH 0590/3444] Warn user instead of failing for wrong SPIDER_MODULES setting --- scrapy/spiderloader.py | 13 ++++++++++--- tests/test_spiderloader/__init__.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index fbf68cec4..265182329 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import warnings from zope.interface import implementer @@ -18,15 +19,21 @@ class SpiderLoader(object): self.spider_modules = settings.getlist('SPIDER_MODULES') self._spiders = {} self._load_all_spiders() - + def _load_spiders(self, module): for spcls in iter_spider_classes(module): self._spiders[spcls.name] = spcls def _load_all_spiders(self): for name in self.spider_modules: - for module in walk_modules(name): - self._load_spiders(module) + try: + for module in walk_modules(name): + self._load_spiders(module) + except ImportError as e: + msg = ("Could not load spiders from module '{}'; " + "Check SPIDER_MODULES setting " + "(exception: {})".format(name, str(e))) + warnings.warn(msg, RuntimeWarning) @classmethod def from_settings(cls, settings): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index fbd2c1669..b2ad93b3f 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,6 +1,7 @@ import sys import os import shutil +import warnings from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase): crawler = runner.create_crawler('spider1') self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) self.assertEqual(crawler.spidercls.name, 'spider1') + + def test_bad_spider_modules_warning(self): + + with warnings.catch_warnings(record=True) as w: + module = 'tests.test_spiderloader.test_spiders.doesnotexist' + settings = Settings({'SPIDER_MODULES': [module]}) + spider_loader = SpiderLoader.from_settings(settings) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) From 7d1783603251923bb549d34a64d43952fe03b3bc Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 8 Dec 2016 17:27:25 +0100 Subject: [PATCH 0591/3444] Update documentation about --no-redirect option --- docs/topics/commands.rst | 28 ++++++++++++++++++ docs/topics/shell.rst | 62 ++++++++++++++++++++++++++-------------- scrapy/shell.py | 9 ++++-- 3 files changed, 75 insertions(+), 24 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 32669104c..3a26b19ae 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page. If used outside a project, no particular per-spider behaviour would be applied and it will just use the default Scrapy downloader settings. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--headers``: print the response's HTTP headers instead of the response's body + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html @@ -368,11 +376,31 @@ given. Also supports UNIX-style local file paths, either relative with ``./`` or ``../`` prefixes or absolute file paths. See :ref:`topics-shell` for more info. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``-c code``: evaluate the code in the shell, print the result and exit + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy shell http://www.example.com/some/page.html [ ... scrapy shell starts ... ] + $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)' + (200, 'http://www.example.com/') + + # shell follows HTTP redirects by default + $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (200, 'http://example.com/') + + # you can disable this with --no-redirect + $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') + + .. command:: parse parse diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..6eb81a71f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -97,8 +97,12 @@ Available Shortcuts * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``fetch(request_or_url)`` - fetch a new response from the given request or - URL and update all related objects accordingly. + * ``fetch(url[, redirect=True])`` - fetch a new response from the given + URL and update all related objects accordingly. You can optionaly ask for + HTTP 3xx redirections to not be followed by passing ``redirect=False`` + + * ``fetch(request)`` - fetch a new response from the given request and + update all related objects accordingly. * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order @@ -157,36 +161,28 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available Scrapy objects: - [s] crawler + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler [s] item {} [s] request - [s] response <200 http://scrapy.org> - [s] settings - [s] spider + [s] response <200 https://scrapy.org/> + [s] settings + [s] spider [s] Useful shortcuts: + [s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed) + [s] fetch(req) Fetch a scrapy.Request and update local objects [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> + After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") - [s] Available Scrapy objects: - [s] crawler - [s] item {} - [s] request - [s] response <200 https://www.reddit.com/> - [s] settings - [s] spider - [s] Useful shortcuts: - [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects - [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() [u'reddit: the front page of the internet'] @@ -194,12 +190,36 @@ After that, we can start playing with the objects:: >>> request = request.replace(method="POST") >>> fetch(request) - [s] Available Scrapy objects: - [s] crawler - ... + >>> response.status + 404 + + >>> from pprint import pprint + + >>> pprint(response.headers) + {'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} >>> + .. _topics-shell-inspect-response: Invoking the shell from spiders to inspect responses diff --git a/scrapy/shell.py b/scrapy/shell.py index 6c78722be..babc267c7 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -148,10 +148,13 @@ class Shell(object): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) b.append("Useful shortcuts:") - b.append(" shelp() Shell help (print this help)") if self.inthread: - b.append(" fetch(req_or_url) Fetch request (or URL) and " - "update local objects") + b.append(" fetch(url[, redirect=True]) " + "Fetch URL and update local objects " + "(by default, redirects are followed)") + b.append(" fetch(req) " + "Fetch a scrapy.Request and update local objects ") + b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") return "\n".join("[s] %s" % l for l in b) From a75ad2bbc63e2fd351c43069a034461c1ab673cf Mon Sep 17 00:00:00 2001 From: Akhil Lb Date: Wed, 4 Nov 2015 01:59:57 +0530 Subject: [PATCH 0592/3444] LOG_SHORT_NAMES option --- docs/topics/logging.rst | 5 +++++ docs/topics/settings.rst | 10 ++++++++++ scrapy/settings/default_settings.py | 1 + scrapy/utils/log.py | 3 ++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index b7aa6d985..231f5186b 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -150,6 +150,7 @@ These settings can be used to configure the logging: * :setting:`LOG_FORMAT` * :setting:`LOG_DATEFORMAT` * :setting:`LOG_STDOUT` +* :setting:`LOG_SHORT_NAMES` The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be @@ -170,6 +171,10 @@ listed in `logging's logrecord attributes docs `_ respectively. +If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy +component that prints the log. It is unset by default, hence logs contain the +scrapy component responsible for that log output. + Command-line options -------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a17472564..c528987ec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -788,6 +788,16 @@ If ``True``, all standard output (and error) of your process will be redirected to the log. For example if you ``print 'hello'`` it will appear in the Scrapy log. +.. setting:: LOG_SHORT_NAMES + +LOG_SHORT_NAMES +____________ + +Default: ``False`` + +If ``True``, the logs will just contain the root path. If it is set to ``False`` +then it displays the component responsible for the log output + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 61f4bd567..24714a7a8 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -191,6 +191,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51f303216..f33ce7017 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -118,7 +118,8 @@ def _get_handler(settings): ) handler.setFormatter(formatter) handler.setLevel(settings.get('LOG_LEVEL')) - handler.addFilter(TopLevelFormatter(['scrapy'])) + if settings.getbool('LOG_SHORT_NAMES'): + handler.addFilter(TopLevelFormatter(['scrapy'])) return handler From 05cec0f2f348345e3d32242968450fb523d25dff Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jan 2016 15:21:05 +0500 Subject: [PATCH 0593/3444] fixed ReST syntax --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c528987ec..503f4afb1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -791,7 +791,7 @@ log. .. setting:: LOG_SHORT_NAMES LOG_SHORT_NAMES -____________ +--------------- Default: ``False`` From 6eab59cbac6a35e7d92a7391541ad2f16493338b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Dec 2016 02:14:12 +0500 Subject: [PATCH 0594/3444] TST cleanup runspider tests --- tests/test_commands.py | 47 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index b507c46bc..bcd7215a0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -38,11 +38,11 @@ class ProjectTest(unittest.TestCase): return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs) - def proc(self, *new_args, **kwargs): + def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args p = subprocess.Popen(args, cwd=self.cwd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs) + **popen_kwargs) waited = 0 interval = 0.2 @@ -182,6 +182,17 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug("It Works!") + return [] +""" + @contextmanager def _create_file(self, content, name): tmpdir = self.mktemp() @@ -194,32 +205,23 @@ class RunSpiderCommandTest(CommandTest): finally: rmtree(tmpdir) - def runspider(self, code, name='myspider.py'): + def runspider(self, code, name='myspider.py', args=()): with self._create_file(code, name) as fname: - return self.proc('runspider', fname) + return self.proc('runspider', fname, *args) + + def get_log(self, code, name='myspider.py', args=()): + p = self.runspider(code, name=name, args=args) + return to_native_str(p.stderr.read()) def test_runspider(self): - spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug("It Works!") - return [] -""" - p = self.runspider(spider) - log = to_native_str(p.stderr.read()) - + log = self.get_log(self.debug_log_spider) self.assertIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) def test_runspider_no_spider_found(self): - p = self.runspider("from scrapy.spiders import Spider\n") - log = to_native_str(p.stderr.read()) + log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): @@ -228,12 +230,11 @@ class MySpider(scrapy.Spider): self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): - p = self.runspider('', 'myspider.txt') - log = to_native_str(p.stderr.read()) + log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) def test_start_requests_errors(self): - p = self.runspider(""" + log = self.get_log(""" import scrapy class BadSpider(scrapy.Spider): @@ -241,11 +242,11 @@ class BadSpider(scrapy.Spider): def start_requests(self): raise Exception("oops!") """, name="badspider.py") - log = to_native_str(p.stderr.read()) print(log) self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + class BenchCommandTest(CommandTest): def test_run(self): From e46572d6f2de1533b1df2ab206971351c22bbbbe Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Dec 2016 02:19:33 +0500 Subject: [PATCH 0595/3444] TST end-to-end test for LOG_LEVEL option there were no end-to-end tests for this option --- tests/test_commands.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index bcd7215a0..1dd88f342 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -220,6 +220,12 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_runspider_log_level(self): + log = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_LEVEL=INFO')) + self.assertNotIn("DEBUG: It Works!", log) + self.assertIn("INFO: Spider opened", log) + def test_runspider_no_spider_found(self): log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) From 05b4555f3932afb04ab3b15893a5858fca9dec04 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 9 Dec 2016 02:19:51 +0500 Subject: [PATCH 0596/3444] TST tests for LOG_SHORT_NAMES --- tests/test_commands.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1dd88f342..922098668 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -226,6 +226,21 @@ class MySpider(scrapy.Spider): self.assertNotIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) + def test_runspider_log_short_names(self): + log1 = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_SHORT_NAMES=1')) + print(log1) + self.assertIn("[myspider] DEBUG: It Works!", log1) + self.assertIn("[scrapy]", log1) + self.assertNotIn("[scrapy.core.engine]", log1) + + log2 = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_SHORT_NAMES=0')) + print(log2) + self.assertIn("[myspider] DEBUG: It Works!", log2) + self.assertNotIn("[scrapy]", log2) + self.assertIn("[scrapy.core.engine]", log2) + def test_runspider_no_spider_found(self): log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) From f457379a54cdbad60bcf24947e053340abc7e16b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 9 Dec 2016 16:56:26 +0100 Subject: [PATCH 0597/3444] Add stacktrace in warning message --- scrapy/spiderloader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 265182329..d4f0f663f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import traceback import warnings from zope.interface import implementer @@ -30,9 +31,9 @@ class SpiderLoader(object): for module in walk_modules(name): self._load_spiders(module) except ImportError as e: - msg = ("Could not load spiders from module '{}'; " - "Check SPIDER_MODULES setting " - "(exception: {})".format(name, str(e))) + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "Check SPIDER_MODULES setting".format( + modname=name, tb=traceback.format_exc())) warnings.warn(msg, RuntimeWarning) @classmethod From f7e4081414d318ddb5297fb98a120e57044dc622 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Dec 2016 22:37:53 +0100 Subject: [PATCH 0598/3444] Add tests for SequenceExclude container --- tests/test_utils_datatypes.py | 63 ++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b31d2179c..80f797227 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,7 @@ import copy import unittest -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaselessDict, SequenceExclude __doctests__ = ['scrapy.utils.datatypes'] @@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase): assert isinstance(h2, CaselessDict) +class SequenceExcludeTest(unittest.TestCase): + + def test_list(self): + seq = [1, 2, 3] + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn(4, d) + self.assertNotIn(2, d) + + def test_range(self): + seq = range(10, 20) + d = SequenceExclude(seq) + self.assertIn(5, d) + self.assertIn(20, d) + self.assertNotIn(15, d) + + def test_six_range(self): + import six.moves + seq = six.moves.range(10**3, 10**6) + d = SequenceExclude(seq) + self.assertIn(10**2, d) + self.assertIn(10**7, d) + self.assertNotIn(10**4, d) + + def test_range_step(self): + seq = range(10, 20, 3) + d = SequenceExclude(seq) + are_not_in = [v for v in range(10, 20, 3) if v in d] + self.assertEquals([], are_not_in) + + are_not_in = [v for v in range(10, 20) if v in d] + self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + + def test_string_seq(self): + seq = "cde" + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_stringset_seq(self): + seq = set("cde") + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_set(self): + """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" + seq = set([-3, "test", 1.1]) + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn("foo", d) + self.assertIn(3.14, d) + self.assertIn(set("bar"), d) + + # supplied sequence is a set, so checking for list (non)inclusion fails + self.assertRaises(TypeError, (0, 1, 2) in d) + self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c']) + + for v in [-3, "test", 1.1]: + self.assertNotIn(v, d) + if __name__ == "__main__": unittest.main() From 70a69d2199c3c08a7e16f33ea5d35fd4066eb14b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Dec 2016 22:40:48 +0100 Subject: [PATCH 0599/3444] Use built-in range() --- scrapy/commands/fetch.py | 2 +- scrapy/shell.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 6fe6d73b9..7d4840529 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -56,7 +56,7 @@ class Command(ScrapyCommand): # by default, let the framework handle redirects, # i.e. command handles all codes expect 3xx if not opts.no_redirect: - request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400)) + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) else: request.meta['handle_httpstatus_all'] = True diff --git a/scrapy/shell.py b/scrapy/shell.py index babc267c7..6f94635a1 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,7 +7,6 @@ from __future__ import print_function import os import signal -from six.moves import range import warnings from twisted.internet import reactor, threads, defer From 0fc73a9d558158f1686f9cc9c289fe364b5df536 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 16 Dec 2016 21:47:58 +0500 Subject: [PATCH 0600/3444] DOC update examples with long longger names --- docs/intro/tutorial.rst | 24 +++---- docs/topics/benchmarking.rst | 92 +++++++++++++++++---------- docs/topics/downloader-middleware.rst | 8 +-- docs/topics/settings.rst | 2 +- docs/topics/shell.rst | 10 +-- 5 files changed, 81 insertions(+), 55 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 0941eb1e5..8e14d1b7c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -130,15 +130,15 @@ will send some requests for the ``quotes.toscrape.com`` domain. You will get an similar to this:: ... (omitted for brevity) - 2016-09-20 14:48:00 [scrapy] INFO: Spider opened - 2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) - 2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) (referer: None) - 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) (referer: None) - 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html - 2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) (referer: None) - 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html - 2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished) + 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened + 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html + 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html + 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) ... Now, check the files in the current directory. You should notice that two new @@ -212,7 +212,7 @@ using the shell :ref:`Scrapy shell `. Run:: You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler @@ -429,9 +429,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} - 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 632190067..99469ebf1 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -18,40 +18,66 @@ To run it use:: You should see an output like this:: - 2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot) - 2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened - 2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats: - {'downloader/request_bytes': 231508, - 'downloader/request_count': 682, - 'downloader/request_method_count/GET': 682, - 'downloader/response_bytes': 1172802, - 'downloader/response_count': 682, - 'downloader/response_status_count/200': 682, - 'finish_reason': 'closespider_timeout', - 'finish_time': datetime.datetime(2013, 5, 16, 16, 8, 57, 985539), - 'log_count/INFO': 14, - 'request_depth_max': 34, - 'response_received_count': 682, - 'scheduler/dequeued': 682, - 'scheduler/dequeued/memory': 682, - 'scheduler/enqueued': 12767, - 'scheduler/enqueued/memory': 12767, - 'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)} - 2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout) + 2016-12-16 21:18:48 [scrapy.utils.log] INFO: Scrapy 1.2.2 started (bot: quotesbot) + 2016-12-16 21:18:48 [scrapy.utils.log] INFO: Overridden settings: {'CLOSESPIDER_TIMEOUT': 10, 'ROBOTSTXT_OBEY': True, 'SPIDER_MODULES': ['quotesbot.spiders'], 'LOGSTATS_INTERVAL': 1, 'BOT_NAME': 'quotesbot', 'LOG_LEVEL': 'INFO', 'NEWSPIDER_MODULE': 'quotesbot.spiders'} + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled extensions: + ['scrapy.extensions.closespider.CloseSpider', + 'scrapy.extensions.logstats.LogStats', + 'scrapy.extensions.telnet.TelnetConsole', + 'scrapy.extensions.corestats.CoreStats'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: + ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', + 'scrapy.downloadermiddlewares.retry.RetryMiddleware', + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', + 'scrapy.downloadermiddlewares.stats.DownloaderStats'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: + ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.spidermiddlewares.referer.RefererMiddleware', + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', + 'scrapy.spidermiddlewares.depth.DepthMiddleware'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled item pipelines: + [] + 2016-12-16 21:18:49 [scrapy.core.engine] INFO: Spider opened + 2016-12-16 21:18:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:50 [scrapy.extensions.logstats] INFO: Crawled 70 pages (at 4200 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:51 [scrapy.extensions.logstats] INFO: Crawled 134 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:52 [scrapy.extensions.logstats] INFO: Crawled 198 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:53 [scrapy.extensions.logstats] INFO: Crawled 254 pages (at 3360 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:54 [scrapy.extensions.logstats] INFO: Crawled 302 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:55 [scrapy.extensions.logstats] INFO: Crawled 358 pages (at 3360 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:56 [scrapy.extensions.logstats] INFO: Crawled 406 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:57 [scrapy.extensions.logstats] INFO: Crawled 438 pages (at 1920 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:58 [scrapy.extensions.logstats] INFO: Crawled 470 pages (at 1920 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:59 [scrapy.core.engine] INFO: Closing spider (closespider_timeout) + 2016-12-16 21:18:59 [scrapy.extensions.logstats] INFO: Crawled 518 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:19:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats: + {'downloader/request_bytes': 229995, + 'downloader/request_count': 534, + 'downloader/request_method_count/GET': 534, + 'downloader/response_bytes': 1565504, + 'downloader/response_count': 534, + 'downloader/response_status_count/200': 534, + 'finish_reason': 'closespider_timeout', + 'finish_time': datetime.datetime(2016, 12, 16, 16, 19, 0, 647725), + 'log_count/INFO': 17, + 'request_depth_max': 19, + 'response_received_count': 534, + 'scheduler/dequeued': 533, + 'scheduler/dequeued/memory': 533, + 'scheduler/enqueued': 10661, + 'scheduler/enqueued/memory': 10661, + 'start_time': datetime.datetime(2016, 12, 16, 16, 18, 49, 799869)} + 2016-12-16 21:19:00 [scrapy.core.engine] INFO: Spider closed (closespider_timeout) -That tells you that Scrapy is able to crawl about 3900 pages per minute in the +That tells you that Scrapy is able to crawl about 3000 pages per minute in the hardware where you run it. Note that this is a very simple spider intended to follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 29d9b0298..3b9a5335a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -238,14 +238,14 @@ header) and all cookies received in responses (ie. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: - 2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ Set-Cookie: ip_isocode=US Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) (referer: None) + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [...] diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 503f4afb1..0515a9e0d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1037,7 +1037,7 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap Example entry in logs:: - 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + 1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request: - reason: cannot serialize (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..da91108b2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -173,7 +173,7 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() - u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' + 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") [s] Available Scrapy objects: @@ -189,7 +189,7 @@ After that, we can start playing with the objects:: [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() - [u'reddit: the front page of the internet'] + ['reddit: the front page of the internet'] >>> request = request.replace(method="POST") @@ -234,8 +234,8 @@ Here's an example of how you would call it from your spider:: When you run the spider, you will get something similar to this:: - 2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] crawler ... @@ -258,7 +258,7 @@ Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the crawling:: >>> ^D - 2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is From da19f0b7b73ca4fd78d828e710e111c60bc658e3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 16 Dec 2016 22:14:54 +0500 Subject: [PATCH 0601/3444] DOC how to override log level for a specific Scrapy component --- docs/topics/logging.rst | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 231f5186b..ac3b614fc 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -10,7 +10,7 @@ Logging about the new logging system. Scrapy uses `Python's builtin logging system -`_ for event logging. We'll +`_ for event logging. We'll provide some simple examples to get you started, but for more advanced use-cases it's strongly suggested to read thoroughly its documentation. @@ -193,6 +193,43 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +Advanced customization +---------------------- + +Because Scrapy uses stdlib logging module, you can customize logging using +all features of stdlib logging. + +For example, let's say you're scraping a website which returns many +HTTP 404 and 500 responses, and you want to hide all messages like this:: + + 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring + response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + is not handled or not allowed + +The first thing to note is a logger name - it is in brackets: +``[scrapy.spidermiddlewares.httperror]``. If you get just ``[scrapy]`` then +:setting:`LOG_SHORT_NAMES` is likely set to True; set it to False and re-run +the crawl. + +Next, we can see that the message has INFO level. To hide it +we should set logging level for ``scrapy.spidermiddlewares.httperror`` +higher than INFO; next level after INFO is WARNING. It could be done +e.g. in the spider's ``__init__`` method:: + + import logging + import scrapy + + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('scrapy.spidermiddlewares.httperror') + logger.setLevel(logging.WARNING) + super().__init__(*args, **kwargs) + +If you run this spider again then INFO messages from +``scrapy.spidermiddlewares.httperror`` logger will be gone. + scrapy.utils.log module ======================= From 2b3abdb7006a2875744a89d45f46e858c981d751 Mon Sep 17 00:00:00 2001 From: zhouyc Date: Mon, 19 Dec 2016 11:27:57 +0800 Subject: [PATCH 0602/3444] update parsel version which will cause attribute error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4eb8d2318..3ae7915b4 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.3', + 'parsel>=0.9.5', 'PyDispatcher>=2.0.5', 'service_identity', ], From 140a57d7b00f0b044598715ab640bd7e96e32318 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 19 Dec 2016 17:51:30 +0100 Subject: [PATCH 0603/3444] Amend note on --no-redirect option for shell tool --- docs/topics/commands.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3a26b19ae..6636c30cb 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -382,7 +382,9 @@ Supported options: * ``-c code``: evaluate the code in the shell, print the result and exit -* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them); + this only affects the URL you may pass as argument on the command line; + once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default. Usage example:: @@ -397,6 +399,7 @@ Usage example:: (200, 'http://example.com/') # you can disable this with --no-redirect + # (only for the URL passed as command line argument) $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') From 6dec4a3ccb455511decc4c1414b9042c57a205ea Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 20 Dec 2016 20:02:31 -0400 Subject: [PATCH 0604/3444] ENH Pass arguments to logger rather than formatted message. This not only use the standard form but helps error aggregation libraries (i.e.: Sentry) to avoid duplicating the message. --- scrapy/core/downloader/handlers/http11.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 54aa359fb..ecd7f90d3 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,14 +319,13 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ("Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize})." - ).format(url=request.url, size=expected_size, maxsize=maxsize) + error_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} - logger.error(error_message) + logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_message) + raise defer.CancelledError(error_msg % error_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From e9b3cf01f4393c50b4a10f2032db0fecae5aa5f4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 9 Dec 2016 17:23:59 +0100 Subject: [PATCH 0605/3444] Update changelog for upcoming 1.3.0 release --- docs/news.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5e28fb130..5f759f0fc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,26 @@ Release notes ============= +Scrapy 1.3.0 (2016-12-XX) +------------------------- + +New Features +~~~~~~~~~~~~ + +- ``MailSender`` now accepts single strings as values for ``to`` and ``cc`` + arguments (:issue:`2272`) + +Dependencies & Cleanups +~~~~~~~~~~~~~~~~~~~~~~~ + +- Scrapy now requires Twisted >= 13.1 which is the case for many Linux + distributions already. +- As a consequence, we got rid of ``scrapy.xlib.tx.*`` modules, which + copied some of Twisted code for users stuck with an "old" Twisted version +- ``ChunkedTransferMiddleware`` is deprecated and removed from the default + downloader middlewares. + + Scrapy 1.2.2 (2016-12-06) ------------------------- From 9d5afd8c35410e2874a97a17b1a2b5f4deef2b38 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 16 Dec 2016 18:46:49 +0100 Subject: [PATCH 0606/3444] Add note on HttpErrorMiddleware new logging level --- docs/news.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5f759f0fc..a7cc30b0b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -22,6 +22,12 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. +Logging +~~~~~~~ + +- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; + this is technically **backwards incompatible** so please check your log parsers. + Scrapy 1.2.2 (2016-12-06) ------------------------- From 9098001888a57ac4db030fc6563d541174f47d47 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 19 Dec 2016 17:09:47 +0100 Subject: [PATCH 0607/3444] Add note on LOG_SHORT_NAMES --- docs/news.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index a7cc30b0b..02227aa8e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -27,6 +27,11 @@ Logging - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; this is technically **backwards incompatible** so please check your log parsers. +- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, + instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); + this is **backwards incompatible** if you have log parsers expecting the short + logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` + set to ``True``. Scrapy 1.2.2 (2016-12-06) From 49a84c2d414386b27a3f9e4439c5a967ce6cb9bc Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 14:32:21 +0100 Subject: [PATCH 0608/3444] Add note on HTTP redirects with shell and fetch --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 02227aa8e..02976fedf 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,6 +11,9 @@ New Features - ``MailSender`` now accepts single strings as values for ``to`` and ``cc`` arguments (:issue:`2272`) +- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside + scrapy shell now follow HTTP redirections by default (:issue:`2290`); + See :command:`fetch` and :command:`shell` for details. Dependencies & Cleanups ~~~~~~~~~~~~~~~~~~~~~~~ From 4eeec3e42cf4a48dd3481e43ca064576a4a825d7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 15:59:18 +0100 Subject: [PATCH 0609/3444] Add preamble on why 1.3.0 comes so soon after 1.2.2 --- docs/news.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 02976fedf..09c47dc30 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,14 @@ Release notes Scrapy 1.3.0 (2016-12-XX) ------------------------- +This release comes rather soon after 1.2.2 for one main reason: +it was found out that releases since 0.18 up to 1.2.2 (included) use +some backported code from Twisted, even if newer Twisted modules are available. +Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly. + +As it is a major change, we wanted to get the bug fix out quickly +while not breaking any projects using the 1.2 series. + New Features ~~~~~~~~~~~~ From d60a6899dfe0fa5ac08298fa466b71e4f221b93e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:01:36 +0100 Subject: [PATCH 0610/3444] Merge logging changes into "New features" section --- docs/news.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 09c47dc30..f42edba30 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -22,6 +22,13 @@ New Features - ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside scrapy shell now follow HTTP redirections by default (:issue:`2290`); See :command:`fetch` and :command:`shell` for details. +- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; + this is technically **backwards incompatible** so please check your log parsers. +- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, + instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); + this is **backwards incompatible** if you have log parsers expecting the short + logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` + set to ``True``. Dependencies & Cleanups ~~~~~~~~~~~~~~~~~~~~~~~ @@ -33,17 +40,6 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. -Logging -~~~~~~~ - -- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; - this is technically **backwards incompatible** so please check your log parsers. -- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, - instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); - this is **backwards incompatible** if you have log parsers expecting the short - logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` - set to ``True``. - Scrapy 1.2.2 (2016-12-06) ------------------------- From b9e7ca044cfb82fc9592cef7e2fdcfa422fe8d44 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:11:19 +0100 Subject: [PATCH 0611/3444] Reword things a tiny bit --- docs/news.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index f42edba30..6690ca558 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,8 +8,10 @@ Scrapy 1.3.0 (2016-12-XX) This release comes rather soon after 1.2.2 for one main reason: it was found out that releases since 0.18 up to 1.2.2 (included) use -some backported code from Twisted, even if newer Twisted modules are available. +some backported code from Twisted (``scrapy.xlib.tx.*``), +even if newer Twisted modules are available. Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly. +(See also cleanups below.) As it is a major change, we wanted to get the bug fix out quickly while not breaking any projects using the 1.2 series. From f8793e2460977a0c218900687cd7c6d592a5c7d5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:27:01 +0100 Subject: [PATCH 0612/3444] Set release date for 1.3.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 6690ca558..cce46599b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.3.0 (2016-12-XX) +Scrapy 1.3.0 (2016-12-21) ------------------------- This release comes rather soon after 1.2.2 for one main reason: From ac74d5a467908a8c3db05f80e9d533be2c9deb64 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 16:28:44 +0100 Subject: [PATCH 0613/3444] =?UTF-8?q?Bump=20version:=201.2.2=20=E2=86=92?= =?UTF-8?q?=201.3.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ee039790d..57ff603fa 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.2 +current_version = 1.3.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 23aa83906..f0bb29e76 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.2.2 +1.3.0 From 07f9985a941d2fce4d7115a35dc983bc21ded4be Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 21 Dec 2016 17:03:11 +0100 Subject: [PATCH 0614/3444] TST: Randomize FILES_EXPIRES above 90 days --- tests/test_pipeline_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 157c21a89..e3ec04b8d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -208,7 +208,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { - "FILES_EXPIRES": random.randint(1, 1000), + "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), "FILES_STORE": self.tempdir From e7c7e055ff3c661cd60b15fe5ad910a2765db707 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sat, 24 Dec 2016 11:55:04 -0500 Subject: [PATCH 0615/3444] settings: fixing name of the pipeline template --- scrapy/templates/project/module/settings.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 72f25ebef..486df6b71 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { -# '$project_name.pipelines.SomePipeline': 300, +# '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) From 2240f00a136e264cc1856860fddba85ec97432c1 Mon Sep 17 00:00:00 2001 From: nyov Date: Tue, 1 Mar 2016 07:12:19 +0000 Subject: [PATCH 0616/3444] Remove dependency on os.environ from default settings Avoid loading settings from environment in scrapy core. Instead it's better to populate them from the starting shell or an embedding script. --- docs/topics/commands.rst | 6 +++--- docs/topics/settings.rst | 8 ++++---- scrapy/cmdline.py | 8 +++++++- scrapy/commands/edit.py | 4 +++- scrapy/settings/default_settings.py | 11 +++-------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6636c30cb..935e3281e 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -291,12 +291,12 @@ edit * Syntax: ``scrapy edit `` * Requires project: *yes* -Edit the given spider using the editor defined in the :setting:`EDITOR` -setting. +Edit the given spider using the editor defined in the ``EDITOR`` environment +variable or (if unset) the :setting:`EDITOR` setting. This command is provided only as a convenience shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and -debug his spiders. +debug spiders. Usage example:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0515a9e0d..1bb7a9d2b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -635,11 +635,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r EDITOR ------ -Default: `depends on the environment` +Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows) -The editor to use for editing spiders with the :command:`edit` command. It -defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults -to ``vi`` (on Unix systems) or the IDLE editor (on Windows). +The editor to use for editing spiders with the :command:`edit` command. +Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit` +command will prefer it over the default setting. .. setting:: EXTENSIONS diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cb7bbd64d..dca931e99 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,5 +1,5 @@ from __future__ import print_function -import sys +import sys, os import optparse import cProfile import inspect @@ -106,6 +106,12 @@ def execute(argv=None, settings=None): if settings is None: settings = get_project_settings() + # set EDITOR from environment if available + try: + editor = os.environ['EDITOR'] + except KeyError: pass + else: + settings['EDITOR'] = editor check_deprecated_settings(settings) # --- backwards compatibility for scrapy.conf.settings singleton --- diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 2df6a730c..a7f8983b4 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -3,6 +3,7 @@ import sys, os from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError + class Command(ScrapyCommand): requires_project = True @@ -15,7 +16,8 @@ class Command(ScrapyCommand): return "Edit spider" def long_desc(self): - return "Edit a spider using the editor defined in EDITOR setting" + return ("Edit a spider using the editor defined in the EDITOR environment" + " variable or else the EDITOR setting") def _err(self, msg): sys.stderr.write(msg + os.linesep) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..f687ef6b1 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -13,7 +13,6 @@ Scrapy developers, if you add a setting here remember to: """ -import os import sys from importlib import import_module from os.path import join, abspath, dirname @@ -111,13 +110,9 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter' -try: - EDITOR = os.environ['EDITOR'] -except KeyError: - if sys.platform == 'win32': - EDITOR = '%s -m idlelib.idle' - else: - EDITOR = 'vi' +EDITOR = 'vi' +if sys.platform == 'win32': + EDITOR = '%s -m idlelib.idle' EXTENSIONS = {} From 9922ec15d7a79ac109bf02ccd9ac154bca5d5167 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 27 Dec 2016 09:52:35 -0200 Subject: [PATCH 0617/3444] mention contributing document before CoC in the README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b72ebf53d..38ad43eba 100644 --- a/README.rst +++ b/README.rst @@ -73,14 +73,14 @@ See http://scrapy.org/community/ Contributing ============ +See http://doc.scrapy.org/en/master/contributing.html + Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@scrapinghub.com. -See http://doc.scrapy.org/en/master/contributing.html - Companies using Scrapy ====================== From dabcb17d72e5bf0ab3a0d7cfe50f70aa2529cd09 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 27 Dec 2016 11:32:15 -0200 Subject: [PATCH 0618/3444] update code of conduct http://contributor-covenant.org/version/1/4 --- CODE_OF_CONDUCT.md | 80 ++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 95b4a7e3c..162602248 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,24 +1,41 @@ -# Contributor Code of Conduct +# Contributor Covenant Code of Conduct -As contributors and maintainers of this project, and in the interest of -fostering an open and welcoming community, we pledge to respect all people who -contribute through reporting issues, posting feature requests, updating -documentation, submitting pull requests or patches, and other activities. +## Our Pledge -We are committed to making participation in this project a harassment-free -experience for everyone, regardless of level of experience, gender, gender -identity and expression, sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, religion, or nationality. +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery -* Personal attacks -* Trolling or insulting/derogatory comments +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment -* Publishing other's private information, such as physical or electronic - addresses, without explicit permission -* Other unethical or unprofessional conduct +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions @@ -26,25 +43,32 @@ that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. -By adopting this Code of Conduct, project maintainers commit themselves to -fairly and consistently applying these principles to every aspect of managing -this project. Project maintainers who do not follow or enforce the Code of -Conduct may be permanently removed from the project team. +## Scope This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting a project maintainer at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@scrapinghub.com. All complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. Maintainers are -obligated to maintain confidentiality with regard to the reporter of an -incident. +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 1.3.0, available at -[http://contributor-covenant.org/version/1/3/0/][version] +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/3/0/ +[version]: http://contributor-covenant.org/version/1/4/ From b7dd089bf9d041218012ce1da7bce5045e76d37a Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 27 Dec 2016 11:33:23 -0200 Subject: [PATCH 0619/3444] show CoC in its own section --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 38ad43eba..38dda62e5 100644 --- a/README.rst +++ b/README.rst @@ -75,6 +75,9 @@ Contributing See http://doc.scrapy.org/en/master/contributing.html +Code of Conduct +--------------- + Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). From 8a86574394172a8221a95d9e2fcc229a26be2103 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Dec 2016 14:10:50 +0000 Subject: [PATCH 0620/3444] .devN release suffix must be preceded with a dot https://packaging.python.org/distributing/#standards-compliance-for-interoperability --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 506f3779b..ac79a0425 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ branches: only: - master - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ + - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ env: - TOXENV=py27 - TOXENV=jessie @@ -35,4 +35,4 @@ deploy: on: tags: true repo: scrapy/scrapy - condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$" + condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" From b6ab1ae9c3ffa7aeee0e7ee77e2dd55d7b663a30 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 3 Jan 2017 15:14:59 -0200 Subject: [PATCH 0621/3444] docs: installation instructions, mention conda in the beginning (closes #2475) --- docs/intro/install.rst | 74 +++++++++++------------------------------- 1 file changed, 19 insertions(+), 55 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 767749ec5..86387ef5e 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,14 +7,25 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.3 or above -(except on Windows where Python 3 is not supported yet). +Scrapy runs on Python 2.7 and Python 3.3 or above. -If you’re already familiar with installation of Python packages, +If you're using `Anaconda`_ or `Miniconda`_, you can install the package from +the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows +and OS X. + +To install Scrapy using ``conda``, run:: + + conda install -c conda-forge scrapy + +Alternatively, if you’re already familiar with installation of Python packages, you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +Note that sometimes this may require solving compilation issues for some Scrapy +dependencies depending on your operating system, so be sure to check the +:ref:`intro-install-platform-notes`. + We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. @@ -108,42 +119,14 @@ Platform specific installation notes Windows ------- -* Install Python 2.7 from https://www.python.org/downloads/ +Though it's possible to install Scrapy on Windows using pip, we recommend you +to install `Anaconda`_ or `Miniconda`_ and use the package from the +`conda-forge`_ channel, which will avoid most installation issues. - You need to adjust ``PATH`` environment variable to include paths to - the Python executable and additional scripts. The following paths need to be - added to ``PATH``:: +Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: - C:\Python27\;C:\Python27\Scripts\; + conda install -c conda-forge scrapy - To update the ``PATH`` open a Command prompt and run:: - - c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py - - Close the command prompt window and reopen it so changes take effect, run the - following command and check it shows the expected Python version:: - - python --version - -* Install `pywin32` from http://sourceforge.net/projects/pywin32/ - - Be sure you download the architecture (win32 or amd64) that matches your system - -* *(Only required for Python<2.7.9)* Install `pip`_ from - https://pip.pypa.io/en/latest/installing/ - - Now open a Command prompt to check ``pip`` is installed correctly:: - - pip --version - -* At this point Python 2.7 and ``pip`` package manager must be working, let's - install Scrapy:: - - pip install Scrapy - -.. note:: - Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support - Python 3 on Windows. Ubuntu 12.04 or above --------------------- @@ -234,27 +217,8 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy -Anaconda --------- - - -Using Anaconda is an alternative to using a virtualenv and installing with ``pip``. - -.. note:: - - For Windows users, or if you have issues installing through ``pip``, this is - the recommended way to install Scrapy. - -If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_ -community have up-to-date packages for Linux, Windows and OS X. - -To install Scrapy using ``conda``, run:: - - conda install -c conda-forge scrapy - .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ -.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx .. _lxml: http://lxml.de/ .. _parsel: https://pypi.python.org/pypi/parsel .. _w3lib: https://pypi.python.org/pypi/w3lib From a21473147160e4b176fedb32c92e1b68b4af04f6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 15:38:35 +0100 Subject: [PATCH 0622/3444] Add Python 3.6 tox env + Travis CI build for it --- .travis.yml | 1 + tox.ini | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 506f3779b..b0ac3afde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 + - TOXENV=py36 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index 812302b4c..bdc14a128 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,10 @@ deps = {[testenv:py33]deps} basepython = python3.5 deps = {[testenv:py33]deps} +[testenv:py36] +basepython = python3.6 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps = From 40851a3d4cde6b44cc92e03b7272b3acd33ed7f6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 15:44:32 +0100 Subject: [PATCH 0623/3444] Use Python 3.6-dev on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0ac3afde..5c9eb8cc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 3.5 +python: 3.6-dev sudo: false branches: only: From 53769245f553355ad1907ad0b47a05093ec65cbe Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 16:02:14 +0100 Subject: [PATCH 0624/3444] Use python 3.6 directly --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5c9eb8cc7..4c4adb948 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 3.6-dev +python: 3.6 sudo: false branches: only: From 6b838b02966902311ab869bb6a35e023265ed274 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 16:10:14 +0100 Subject: [PATCH 0625/3444] Use matrix build config --- .travis.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c4adb948..52fbf02ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,24 @@ language: python -python: 3.6 sudo: false branches: only: - master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ -env: - - TOXENV=py27 - - TOXENV=jessie - - TOXENV=py33 - - TOXENV=py35 - - TOXENV=py36 - - TOXENV=docs +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 2.7 + env: TOXENV=jessie + - python: 3.3 + env: TOXENV=py33 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - python: 3.6 + env: TOXENV=docs install: - pip install -U tox twine wheel codecov script: tox From b3406677b980656751af9a12ddf2d33ad884fbc1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 9 Jan 2017 14:40:02 +0100 Subject: [PATCH 0626/3444] Update classifiers in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f673b1dc4..388cf0dec 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From 900b6710d3c9d70c740707d72c611c50d909616b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Thu, 5 Jan 2017 12:53:00 +0200 Subject: [PATCH 0627/3444] Document copying of spider arguments to attributes --- docs/topics/spiders.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0179e9284..29106e87a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -297,6 +297,37 @@ Spiders can access arguments in their `__init__` methods:: self.start_urls = ['http://www.example.com/categories/%s' % category] # ... +The default `__init__` method will take any spider arguments +and copy them to the spider as attributes. +The above example can also be written as follows:: + + import scrapy + + class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + +Keep in mind that spider arguments are only strings. +The spider will not do any parsing on its own. +If you were to set the `start_urls` attribute from the command line, +you would have to parse it on you own into a list +using something like +`ast.literal_eval `_ +or `json.loads `_ +and then set it as an attribute. +Otherwise, you would cause iteration over a `start_urls` string +(a very common python pitfall) +resulting in each character being seen as a separate url. + +A valid use case is to set the http auth credentials +used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` +or the user agent +used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: + + scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot + Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. From bf2277a028aee0feaecd90a3a64c53bba4e5fbdf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 10 Jan 2017 10:27:38 +0100 Subject: [PATCH 0628/3444] Update spiders.rst --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 29106e87a..c123c2635 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -312,7 +312,7 @@ The above example can also be written as follows:: Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. If you were to set the `start_urls` attribute from the command line, -you would have to parse it on you own into a list +you would have to parse it on your own into a list using something like `ast.literal_eval `_ or `json.loads `_ From 5586fc7e3876d5a5e8d1797c73cb9b743c8097d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paulius=20Aleksi=C5=ABnas?= Date: Tue, 10 Jan 2017 11:12:42 +0200 Subject: [PATCH 0629/3444] Update architecture.rst In the data flow image arrows are red. --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ea0cb0ea7..4ac39ad2d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -12,7 +12,7 @@ Overview The following diagram shows an overview of the Scrapy architecture with its components and an outline of the data flow that takes place inside the system -(shown by the green arrows). A brief description of the components is included +(shown by the red arrows). A brief description of the components is included below with links for more detailed information about them. The data flow is also described below. From df1a42419f8bce48b605087937320af1ec968116 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sat, 14 Jan 2017 20:45:20 -0500 Subject: [PATCH 0630/3444] adding formid to FormRequest documentation --- docs/topics/request-response.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 664a7239f..a1bd1e146 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -358,7 +358,7 @@ fields with form data from :class:`Response` objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ``
`` element contained @@ -383,6 +383,9 @@ fields with form data from :class:`Response` objects. :param formname: if given, the form with name attribute set to this value will be used. :type formname: string + :param formid: if given, the form with id attribute set to this value will be used. + :type formid: string + :param formxpath: if given, the first form that matches the xpath will be used. :type formxpath: string @@ -421,6 +424,9 @@ fields with form data from :class:`Response` objects. .. versionadded:: 1.1.0 The ``formcss`` parameter. + .. versionadded:: 1.1.0 + The ``formid`` parameter. + Request usage examples ---------------------- From b279bc8546994f5610836e63490b4a7a262b3d77 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 19 Jan 2017 16:28:52 +0100 Subject: [PATCH 0631/3444] Fix view command against new --no-redirect option --- scrapy/commands/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 4eb44f77d..59592d08b 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): "contents in a browser" def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + super(Command, self).add_options(parser) parser.add_option("--spider", dest="spider", help="use this spider") From 299544416a05c126838db3c75b6fc154b6a08de1 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sun, 22 Jan 2017 19:23:44 -0500 Subject: [PATCH 0632/3444] changing README to README.rst --- artwork/{README => README.rst} | 2 ++ docs/{README => README.rst} | 2 ++ sep/{README => README.rst} | 2 ++ 3 files changed, 6 insertions(+) rename artwork/{README => README.rst} (97%) rename docs/{README => README.rst} (99%) rename sep/{README => README.rst} (95%) diff --git a/artwork/README b/artwork/README.rst similarity index 97% rename from artwork/README rename to artwork/README.rst index c185d57da..016462f2c 100644 --- a/artwork/README +++ b/artwork/README.rst @@ -1,3 +1,5 @@ +:orphan: + Scrapy artwork ============== diff --git a/docs/README b/docs/README.rst similarity index 99% rename from docs/README rename to docs/README.rst index cf04965ac..733af2af4 100644 --- a/docs/README +++ b/docs/README.rst @@ -1,3 +1,5 @@ +:orphan: + ====================================== Scrapy documentation quick start guide ====================================== diff --git a/sep/README b/sep/README.rst similarity index 95% rename from sep/README rename to sep/README.rst index 668772492..e2d2e6274 100644 --- a/sep/README +++ b/sep/README.rst @@ -1,3 +1,5 @@ +:orphan: + Scrapy Enhancement Proposals ============================ From 53757e51e56ace34bef8616451cca921da16791c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 24 Jan 2017 11:29:11 -0300 Subject: [PATCH 0633/3444] Preserve request class when converting to/from dicts (utils.reqser) --- scrapy/utils/reqser.py | 6 +++++- tests/test_utils_reqser.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 7e1e99e48..2fceb0d94 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -5,6 +5,7 @@ import six from scrapy.http import Request from scrapy.utils.python import to_unicode, to_native_str +from scrapy.utils.misc import load_object def request_to_dict(request, spider=None): @@ -32,6 +33,8 @@ def request_to_dict(request, spider=None): 'priority': request.priority, 'dont_filter': request.dont_filter, } + if type(request) is not Request: + d['_class'] = request.__module__ + '.' + request.__class__.__name__ return d @@ -47,7 +50,8 @@ def request_from_dict(d, spider=None): eb = d['errback'] if eb and spider: eb = _get_method(spider, eb) - return Request( + request_cls = load_object(d['_class']) if '_class' in d else Request + return request_cls( url=to_native_str(d['url']), callback=cb, errback=eb, diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index a62f13e21..5b889ab5d 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import unittest -from scrapy.http import Request +from scrapy.http import Request, FormRequest from scrapy.spiders import Spider from scrapy.utils.reqser import request_to_dict, request_from_dict @@ -42,6 +42,7 @@ class RequestSerializationTest(unittest.TestCase): self._assert_same_request(request, request2) def _assert_same_request(self, r1, r2): + self.assertEqual(r1.__class__, r2.__class__) self.assertEqual(r1.url, r2.url) self.assertEqual(r1.callback, r2.callback) self.assertEqual(r1.errback, r2.errback) @@ -54,6 +55,12 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) + def test_request_class(self): + r = FormRequest("http://www.example.com") + self._assert_serializes_ok(r, spider=self.spider) + r = CustomRequest("http://www.example.com") + self._assert_serializes_ok(r, spider=self.spider) + def test_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider.parse_item, errback=self.spider.handle_error) @@ -77,3 +84,7 @@ class TestSpider(Spider): def handle_error(self, failure): pass + + +class CustomRequest(Request): + pass From bae12870bbb4e6f589887ad6036e50e5db64369f Mon Sep 17 00:00:00 2001 From: Michael Fladischer Date: Tue, 24 Jan 2017 22:20:37 +0100 Subject: [PATCH 0634/3444] Fix spelling error in scrapy.1 manpage. The word "intepreted" should be "interpreted". --- extras/scrapy.1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extras/scrapy.1 b/extras/scrapy.1 index a4f29569b..84f693fc3 100644 --- a/extras/scrapy.1 +++ b/extras/scrapy.1 @@ -28,16 +28,16 @@ Query Scrapy settings Print raw setting value .TP .I --getbool=SETTING -Print setting value, intepreted as a boolean +Print setting value, interpreted as a boolean .TP .I --getint=SETTING -Print setting value, intepreted as an integer +Print setting value, interpreted as an integer .TP .I --getfloat=SETTING -Print setting value, intepreted as an float +Print setting value, interpreted as an float .TP .I --getlist=SETTING -Print setting value, intepreted as an float +Print setting value, interpreted as an float .TP .I --init Print initial setting value (before loading extensions and spiders) From 87472346df6ce0ca63842e50d548a423b606767d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 25 Jan 2017 11:28:20 +0100 Subject: [PATCH 0635/3444] Update scrapy.1 --- extras/scrapy.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extras/scrapy.1 b/extras/scrapy.1 index 84f693fc3..2fa8d8231 100644 --- a/extras/scrapy.1 +++ b/extras/scrapy.1 @@ -34,10 +34,10 @@ Print setting value, interpreted as a boolean Print setting value, interpreted as an integer .TP .I --getfloat=SETTING -Print setting value, interpreted as an float +Print setting value, interpreted as a float .TP .I --getlist=SETTING -Print setting value, interpreted as an float +Print setting value, interpreted as a float .TP .I --init Print initial setting value (before loading extensions and spiders) From fc07711614549ce8f0464b3fca5b0f4acd746681 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:54:28 +0100 Subject: [PATCH 0636/3444] Remove unused --headers option for view command --- scrapy/commands/view.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 59592d08b..59e665016 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -12,8 +12,7 @@ class Command(fetch.Command): def add_options(self, parser): super(Command, self).add_options(parser) - parser.add_option("--spider", dest="spider", - help="use this spider") + parser.remove_option("--headers") def _print_response(self, response, opts): open_in_browser(response) From 4156a86148f07c108d3ea4248a2d4a7e2a58ffa9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:57:37 +0100 Subject: [PATCH 0637/3444] Update docs on view command --- docs/topics/commands.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6636c30cb..eaeeee113 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -358,6 +358,12 @@ Opens the given URL in a browser, as your Scrapy spider would "see" it. Sometimes spiders see pages differently from regular users, so this can be used to check what the spider "sees" and confirm it's what you expect. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy view http://www.example.com/some/page.html From ae6d8d728e12e8efd704ba529de1db2eacfb494f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 16:33:08 +0100 Subject: [PATCH 0638/3444] Support 'True' and 'False' strings as boolean settings values --- scrapy/settings/__init__.py | 16 +++++++++++++--- tests/test_settings/__init__.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 7b7808959..28446a372 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -114,8 +114,8 @@ class BaseSettings(MutableMapping): """ Get a setting value as a boolean. - ``1``, ``'1'``, and ``True`` return ``True``, while ``0``, ``'0'``, - ``False`` and ``None`` return ``False``. + ``1``, ``'1'``, `True`` and ``'True'`` return ``True``, + while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``. For example, settings populated through environment variables set to ``'0'`` will return ``False`` when using this method. @@ -126,7 +126,17 @@ class BaseSettings(MutableMapping): :param default: the value to return if no setting is found :type default: any """ - return bool(int(self.get(name, default))) + got = self.get(name, default) + try: + return bool(int(got)) + except ValueError: + if got in ("True", "true"): + return True + if got in ("False", "false"): + return False + raise ValueError("Supported values for boolean settings " + "are 0/1, True/False, '0'/'1', " + "'True'/'False' and 'true'/'false'") def getint(self, name, default=0): """ diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4acf22cba..863684075 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -211,9 +211,15 @@ class BaseSettingsTest(unittest.TestCase): 'TEST_ENABLED1': '1', 'TEST_ENABLED2': True, 'TEST_ENABLED3': 1, + 'TEST_ENABLED4': 'True', + 'TEST_ENABLED5': 'true', + 'TEST_ENABLED_WRONG': 'on', 'TEST_DISABLED1': '0', 'TEST_DISABLED2': False, 'TEST_DISABLED3': 0, + 'TEST_DISABLED4': 'False', + 'TEST_DISABLED5': 'false', + 'TEST_DISABLED_WRONG': 'off', 'TEST_INT1': 123, 'TEST_INT2': '123', 'TEST_FLOAT1': 123.45, @@ -231,11 +237,15 @@ class BaseSettingsTest(unittest.TestCase): self.assertTrue(settings.getbool('TEST_ENABLED1')) self.assertTrue(settings.getbool('TEST_ENABLED2')) self.assertTrue(settings.getbool('TEST_ENABLED3')) + self.assertTrue(settings.getbool('TEST_ENABLED4')) + self.assertTrue(settings.getbool('TEST_ENABLED5')) self.assertFalse(settings.getbool('TEST_ENABLEDx')) self.assertTrue(settings.getbool('TEST_ENABLEDx', True)) self.assertFalse(settings.getbool('TEST_DISABLED1')) self.assertFalse(settings.getbool('TEST_DISABLED2')) self.assertFalse(settings.getbool('TEST_DISABLED3')) + self.assertFalse(settings.getbool('TEST_DISABLED4')) + self.assertFalse(settings.getbool('TEST_DISABLED5')) self.assertEqual(settings.getint('TEST_INT1'), 123) self.assertEqual(settings.getint('TEST_INT2'), 123) self.assertEqual(settings.getint('TEST_INTx'), 0) @@ -258,6 +268,8 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getdict('TEST_DICT3'), {}) self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5}) self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1') + self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG') + self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG') def test_getpriority(self): settings = BaseSettings({'key': 'value'}, priority=99) From d2e9ea0c88b7578c5fc8d4d37e5df9d078e9b884 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 13 Jan 2017 16:17:51 +0100 Subject: [PATCH 0639/3444] Enforce DNS resolution timeout --- scrapy/resolver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 3954fd977..4f4f0b04f 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -16,8 +16,11 @@ class CachingThreadedResolver(ThreadedResolver): def getHostByName(self, name, timeout=None): if name in dnscache: return defer.succeed(dnscache[name]) - if not timeout: - timeout = self.timeout + # in Twisted<=16.6, getHostByName() is always called with + # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), + # so the input argument above is simply overridden + # to enforce Scrapy's DNS_TIMEOUT setting's value + timeout = (self.timeout,) d = super(CachingThreadedResolver, self).getHostByName(name, timeout) d.addCallback(self._cache_result, name) return d From a58624375824d821f0bdadb7bc04fad53171ca70 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 13 Jan 2017 15:51:44 +0100 Subject: [PATCH 0640/3444] Buffer CONNECT response bytes from proxy until all HTTP headers are received --- scrapy/core/downloader/handlers/http11.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index ecd7f90d3..4b02cb16f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -105,6 +105,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._tunneledHost = host self._tunneledPort = port self._contextFactory = contextFactory + self._connectBuffer = b'' def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" @@ -121,8 +122,16 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ + self._connectBuffer += rcvd_bytes + # make sure that enough (all) bytes are consumed + # and that we've got all HTTP headers (ending with a blank line) + # from the proxy so that we don't send those bytes to the TLS layer + # + # see https://github.com/scrapy/scrapy/issues/2491 + if b'\r\n\r\n' not in self._connectBuffer: + return self._protocol.dataReceived = self._protocolDataReceived - respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes) + respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group('status')) == 200: try: # this sets proper Server Name Indication extension From 8c4f614d2101c21d3560fe68dcec6d3910417cb7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Sep 2016 15:25:38 +0200 Subject: [PATCH 0641/3444] Enable PyPy tests on Travis --- .travis.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6f3ab511f..2df02ea43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,30 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 + - python: 2.7 + env: TOXENV=pypy - python: 3.6 env: TOXENV=docs + allow_failures: + - python: 2.7 + env: TOXENV=pypy install: - - pip install -U tox twine wheel codecov + - | + if [ "$TOXENV" = "pypy" ]; then + export PYENV_ROOT="$HOME/.pyenv" + if [ -f "$PYENV_ROOT/bin/pyenv" ]; then + pushd "$PYENV_ROOT" && git pull && popd + else + rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" + fi + # get latest PyPy from pyenv directly (thanks to natural version sort option -V) + export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1` + "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" + virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + - pip install -U tox twine wheel codecov + script: tox after_success: - codecov From 70f260d3537d788e4efaf4a060fce7f0fc183ed9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Feb 2017 15:14:32 +0100 Subject: [PATCH 0642/3444] Don't run coverage stats when on PyPy --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index bdc14a128..0fdea11bb 100644 --- a/tox.ini +++ b/tox.ini @@ -54,6 +54,11 @@ commands = pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib py.test --cov=scrapy --cov-report= {posargs:scrapy tests} +[testenv:pypy] +basepython = pypy +commands = + py.test {posargs:scrapy tests} + [testenv:py33] basepython = python3.3 deps = From 55742c0392e8c582ed622f0ea28d8d62ece2c401 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 1 Feb 2017 22:43:28 +0500 Subject: [PATCH 0643/3444] DOC mention LevelDB cache storage backend --- docs/topics/downloader-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3b9a5335a..912671d19 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -318,10 +318,11 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with two HTTP cache storage backends: + Scrapy ships with three HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` + * :ref:`httpcache-storage-leveldb` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own storage backend. From 0cf6344cc22a193079ea1fd2fbe28187cc212c7a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 9 Dec 2016 17:58:38 +0100 Subject: [PATCH 0644/3444] Support kwargs for response.xpath() --- scrapy/http/response/text.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index afa430329..5a6507aa8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -111,8 +111,8 @@ class TextResponse(Response): self._cached_selector = Selector(self) return self._cached_selector - def xpath(self, query): - return self.selector.xpath(query) + def xpath(self, query, **kwargs): + return self.selector.xpath(query, **kwargs) def css(self, query): return self.selector.css(query) From 803d8c4b5709d792f1c314bbde124e69903ff23f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Dec 2016 11:26:42 +0100 Subject: [PATCH 0645/3444] Add tests for passing kwargs on response .xpath() shortcut --- tests/test_http_response.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 7624aa4c4..9df3bf6e7 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -320,6 +320,20 @@ class TextResponseTest(BaseResponseTest): response.selector.css("title::text").extract(), ) + def test_selector_shortcuts_kwargs(self): + body = b"Some page

A nice paragraph.

" + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), + response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + ) + self.assertEqual( + response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", pcount=1).extract(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + ) + def test_urljoin_with_base_url(self): """Test urljoin shortcut which also evaluates base-url through get_base_url().""" body = b'' @@ -428,3 +442,21 @@ class XmlResponseTest(TextResponseTest): response.xpath("//elem/text()").extract(), response.selector.xpath("//elem/text()").extract(), ) + + def test_selector_shortcuts_kwargs(self): + body = b''' + + value + ''' + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + ) + + response.selector.register_namespace('s2', 'http://scrapy.org') + self.assertEqual( + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), + response.selector.xpath("//s2:elem/text()").extract(), + ) From 1c0b8053579e9680e4bd4c788a0a2bdf88a8f175 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 11 Jan 2017 19:44:21 +0100 Subject: [PATCH 0646/3444] DOC Mention XPath variables in Selectors section --- docs/topics/selectors.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 39ec9b73c..43370d479 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -283,6 +283,40 @@ XPath specification. .. _Location Paths: https://www.w3.org/TR/xpath#location-paths +.. _topics-selectors-xpath-variables: + +Variables in XPath expressions +------------------------------ + +XPath allows you to reference variables in your XPath expressions, using +the ``$somevariable`` syntax. This is somewhat similar to parameterized +queries or prepared statements in the SQL world where you replace +some arguments in your queries with placeholders like ``?``, +which are then substituted with values passed with the query. + +Here's an example to match an element based on its "id" attribute value, +without hard-coding it (that was shown previously):: + + >>> # `$val` used in the expression, a `val` argument needs to be passed + >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first() + u'Name: My image 1 ' + +Here's another example, to find the "id" attribute of a ``
`` tag containing +five ```` children (here we pass the value ``5`` as an integer):: + + >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first() + u'images' + +All variable references must have a binding value when calling ``.xpath()`` +(otherwise you'll get a ``ValueError: XPath error:`` exception). +This is done by passing as many named arguments as necessary. + +`parsel`_, the library powering Scrapy selectors, has more details and examples +on `XPath variables`_. + +.. _parsel: https://parsel.readthedocs.io/ +.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions + Using EXSLT extensions ---------------------- From 1295c17a26544f451261f4c6558b344162f3c4d4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 17:47:22 +0100 Subject: [PATCH 0647/3444] Bump parsel requirement to at least parsel v1.1 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 64b6e771c..f92603d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=0.9.5 +parsel>=1.1 diff --git a/setup.py b/setup.py index 388cf0dec..a6e6f9615 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.5', + 'parsel>=1.1', 'PyDispatcher>=2.0.5', 'service_identity', ], From 3358254c5c78a1b8de1d02cb3b1db85fe824c798 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 17:53:28 +0100 Subject: [PATCH 0648/3444] Make DNS retry test compatible with Twisted 17+ --- tests/test_crawl.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1b4a4b3b0..c7f4c0e35 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -91,12 +91,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_dns_error(self): - with mock.patch('socket.gethostbyname', - side_effect=socket.gaierror(-5, 'No address associated with hostname')): - crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl("http://example.com/") - self._assert_retried(l) + crawler = self.runner.create_crawler(SimpleSpider) + with LogCapture() as l: + # try to fetch the homepage of a non-existent domain + yield crawler.crawl("http://dns.resolution.invalid/") + self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): From e604c0f3abeafdd896606960170ab5bad699d79d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 18:26:41 +0100 Subject: [PATCH 0649/3444] Remove unused imports --- tests/test_crawl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c7f4c0e35..0c64948fa 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,4 @@ import json -import socket import logging from testfixtures import LogCapture @@ -9,7 +8,6 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.crawler import CrawlerRunner from scrapy.utils.python import to_unicode -from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer From 02e1d2b1fd0257a926f438a7758a9e65a0114325 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 22:28:37 +0100 Subject: [PATCH 0650/3444] Add trailing dot --- tests/test_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 0c64948fa..d5babdded 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid/") + yield crawler.crawl("http://dns.resolution.invalid./") self._assert_retried(l) @defer.inlineCallbacks From 09643796b4d6ef25b23c1d7366d22ca55bd091d8 Mon Sep 17 00:00:00 2001 From: Lukas Anzinger Date: Fri, 3 Feb 2017 20:05:17 +0100 Subject: [PATCH 0651/3444] Fix typo in downloader-middleware.rst. --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3b9a5335a..2e32eb280 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -748,7 +748,7 @@ REDIRECT_MAX_TIMES Default: ``20`` -The maximum number of redirections that will be follow for a single request. +The maximum number of redirections that will be followed for a single request. MetaRefreshMiddleware --------------------- From 3021084f376a643f129388f76f039ab1301abb11 Mon Sep 17 00:00:00 2001 From: djrobust Date: Sat, 4 Feb 2017 20:07:05 -0800 Subject: [PATCH 0652/3444] Use 'yield' when parsing multiple responses Use 'yield' consistently across examples of parse functions. --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 664a7239f..3853f5935 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -207,12 +207,12 @@ different fields from different pages:: request = scrapy.Request("http://www.example.com/some_page.html", callback=self.parse_page2) request.meta['item'] = item - return request + yield request def parse_page2(self, response): item = response.meta['item'] item['other_url'] = response.url - return item + yield item .. _topics-request-response-ref-errbacks: From fcb3daf4fa3e4d97e9e66462395a021b3bc4363d Mon Sep 17 00:00:00 2001 From: Takehiro Shiozaki Date: Mon, 6 Feb 2017 14:03:41 +0900 Subject: [PATCH 0653/3444] fix typo --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 604f1864f..8360827e8 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -112,7 +112,7 @@ following methods: .. method:: process_spider_exception(response, exception, spider) - This method is called when when a spider or :meth:`process_spider_input` + This method is called when a spider or :meth:`process_spider_input` method (from other spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an From 16c4b4e184c1b213eafacecca0097ff44effa696 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 6 Feb 2017 11:41:08 +0100 Subject: [PATCH 0654/3444] [httpcompression] add support for br - brotli content encoding --- requirements.txt | 1 + scrapy/downloadermiddlewares/httpcompression.py | 6 +++++- tests/sample_data/compressed/html-br.bin | Bin 0 -> 4027 bytes .../test_downloadermiddleware_httpcompression.py | 13 ++++++++++++- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 tests/sample_data/compressed/html-br.bin diff --git a/requirements.txt b/requirements.txt index f92603d3d..362d05013 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ six>=1.5.2 PyDispatcher>=2.0.5 service_identity parsel>=1.1 +brotlipy==0.6 diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index bcf20f10c..9202fd8da 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,7 @@ import zlib +import brotli + from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes @@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br') def process_response(self, request, response, spider): @@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) + if encoding == b"br": + body = brotli.decompress(body) return body diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7eea4bb826a1a1851d047e64dd98423138cf996 GIT binary patch literal 4027 zcmV;s4@B@=R3tGHVT)KTxseDvm_{W4YMH#TqFGI-3`~9)H9xksFwjQ_FSnd3&0Y8= z@_ThUt$zy9SegPwSgY7y5IE>Uo9&E0Ix0^mRs=<3xGiyIT}Yr9fO2kCiRXNW-0{i+Lu3MI!yIoUSf-Z226AD}r0^ zdA#v**@y{+8$OqPhc=zMEOz;5{^K$+BPXfRJ1|wzaHbBHwx3!Jh1BbRnT%?EOzxB` zc@o=4lizjLI=g3zEFMR~-@xbc=^ZH>n1uLgXP=Rip&%H2-eg z$?YM9bB=&I5l()^leA~+n16V4g_^DL-W4w{9=Ts$j!|s*p|9)8b6}Tq?$r33#pv6QdYC{6y7(EBcDcrg)1%XlDFWLE zIxI2y!eR*(8>O@}u3Hx&$4JYjr`~J*l_%PUM_=!22LDWBb0`w>VTQ3!mL%lo=0kh> z;=f_ZiDy6Qi&t|`{$gtJAKjVrTHwK@8XlMYrk*v45K zR(OuASJjF_HuS)^Z3_;}-x*kbam!jCxmOBSZRDe}&Ao{=EGz4)%9_~X?Reh+%sac7 zGuk9hsij`&OiZ72ftz#WrA9ZS6`R$ubDx3drEzSlcoi3GIyjXkI?`smByM(WtKW-H zWt7nbG9EvTV{l{PV!0a@LvYsd{#$Oc7=PHO*Tvu0%JJ>8p$>ryS1;SQzlXd;_<=@yK5Wg8buXuZIS?L!lfDPB6Z^`*eg zutTl2T*u?5VH;d&i+*2*Ep~8PI;ShzaWl4DOzYKVhmHR_p&Q&md=hBvsdBnWMe#f`~|qjj-zg?1*K|BSawH>RT%cD#arQ7^TI z6KvdkI?!mmV|~lieEkzilzWsWq#SbmLLQWoW|ul_2z*TXMh3@&sY}_3+cF$~=~WaRuLRH zUEPy7hC`>~;lwtyrg)YdG@M|=iOVCKn3mX%);O^=MJtD6aba>hXdREsw_u5NhA+4< z^sKWQ6|@y+i+U>g{@V(_mUeYdG6iL&*_d)M3>B2UhLP}Q%IjQ-WWddTE~_uq2D4R*$|#Mo;&1hp{C*{ib`vZ;l|QC2kzlp zcW8qz7Ow>GBmI5eG5r=Ij!~P~x`Wh@iT^RIF&vn>rIiM8`^4oaS=lALY>4}Rd9{?x z2~pZ28W!2T#&BcGbi5XgW9h~am$ddO1B-hYldX6)DLps$m>P*^fkBPe_i$Q=Q0L+p zOUieVrI$zYIdT_{Z#*_}8RPXWf?%)XCZqB)4+m?*aBFFV!egD_UB)01g%>2+HtLY+JuQ7OeW5{mY#yF<9sft#; zm%%GGhDL{_4vFSs8EZt&f=bTu_)ucGPgpfh2}P@v-36Ci;&yf@H5?~v3Z8`Zj(6eG zjSk}BrDb}So@WuVuC8=js+hJX*q5eTw&5(OwWr;~oDPn5VsJ{uu7xQSKm$c*Z({v2&xOGa_`jLDebo8PVxh;rI z2UgQBA(pbl-m4reT$bA~mr(motI4NglL8WtC&D?=w#mzb8)5KR1N}Ykemgi!- z3hdhf_2FiLf%Ql}hp$6dS8>!O%UY!A9R`oCJ6Jks!6_NlBmI5QXm&re)=b=%c&$oo zvx-AU(;S);+M$Ei>6-SN1vad4`F?b$H7Xd}F3~E3_3QWlz>ediMS+aksu44b-GoQ) z_&Q)MV!KOdT4mwF(zEQ2%TJ&mUDTA{EHF4Q_NzmuL(_ro5-$?Arx)GPNe8L3<839- zdL-WmUT$2P)9h1}+xS^HyHySzOzlc=0=?-@fxP-Kv%toX^4065!>~oPTM_!b1ZeB- z;e;NwQvhF$YSuK1-NKDqR~;hW0_t^vs}xL3%}HHQes@Ud%o7b~feS<9!x?UK58t-5 zSC{MkX>8-n*BAU+-r&+iR~?29txo6-4RlD~c>J{JU};j)>^G(5!G7VwrHMu{Y-(iw zsbJ`xoInkJa=s}Cx9;kkOioxY z9R%4f##`(GFWl~~E}$ul`~eUzbZ=NMu<51=K(I^Fdd;(|*!Khv9lLfQIJ&2LFDE&? z)~qCzEv7b`BAX`>Uc%H}_i=W!ho#^j20V6#$7;*ba;7oQZ!n*1cd1l&a4}Tb_O0hT zT%VKq=MOB;ZtceUZb|pcSC4UwJ5Pvze0Ak9i`Z1ZCA-|C{*PIiO2tBNT!#C4Np@;! zlWnIr#`Pw-uM&~yvo21Luh=1OOKh@t-LC5~S*)BW^o$9e`+u%i$?RI&WS@1B|35Cd zrc$ACKL1O#xqi7`l5w@P$(}gwD^6UMh(^D4dyLCDC%(RK{hw_a?7nk7C1)!)4n9}2 zUNMV#MaHNd+ijA)cfCqpRXQ(T9UhF=;$`!{+5eGeYej!9r=E}L9BZSJIWS$^KJgyO z?5-{OOOAIQHs?8axl@UJa9g(NK6c34F1BG%J!c)MtSQ{n23odvO^y#?e^HmQeB(eFigv0KW8rG_HO78cg@E%tYcSQJK1o1Sbbw2>9n`k zXS*!Jb{$K*5^>NQR==~1V6~^ewmy{XuR*ib+Bo|Dt;c^*OW7L*yOfBC|Czkr9rW1Fk z6^QfxUoq#j8RM;@w718$4HtZ#JG%YXIrYc?t7+D1(&&&Twr~8$A|sXz0u_EqyI~jtXW;#@LC?1Q?(w~+C-vH zG%sUQ%NbAANSU*0+v7GZmtEC{Ypt_jOSwnPJMJ1KC)cdXwp=d5WmT2!T4QdwgsADf zJmac@lrgJi6MbViF5*^;%e97R^hBF!P4x6!&4$kD+&ndBdp(A2I#uH_*OCXeI+xYC z*NZ;y|5NAe>e_D0uq=<|byX@aOjU+eS+QMm(&f$S*e=5`+?HW^-D?iPKcoNK9I}nR1?6ZQJp>9NXnJOnXiF@OqZ=HLuIy^o!wIrES}$9BR+`f3`L;*}GKd3%s7D z{hHf!+_uMDg>9LZlo79GsdsjA;q?^F*G#v^Wt!}#a<*Z(U50HKhOw4NeEQCV0i4eb zyY;$Ws$|XT+OKD=cs`cpcs-TMg2&@ha?bSH>dadm+m>^$*LK-%(_V9KxZRe=Ww=e3 zYqf22)^=0hYx4H&5AP~Vm2f*N*>IV*+wGW!*S#v+aJj51uzF(~Ym9@wH>2|YglgW( zYj~xMT`imGIm=~n`kZZ6Dla^aVHmdKHjGu+_PpEFJ@ZxbRBkWy=e(SL^Wyxt9JoEE zV_6=Tr&hKlXT)omwqYA<$p_PI&r`py!R~WX&aAHOvRt<1vb<)Y&I1@Tx2ZdGGYr#X z7^_3~o~x##w(q&*l;qtt=YnInEXy%0Z>?;W^uy8!P|j4f_LbK_s`0D? Date: Mon, 6 Feb 2017 12:17:45 +0100 Subject: [PATCH 0655/3444] [httpcompression] import brotli when available --- requirements.txt | 1 - scrapy/downloadermiddlewares/httpcompression.py | 17 ++++++++++++----- ...test_downloadermiddleware_httpcompression.py | 3 ++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 362d05013..f92603d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,3 @@ six>=1.5.2 PyDispatcher>=2.0.5 service_identity parsel>=1.1 -brotlipy==0.6 diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 9202fd8da..04c9e355d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ import zlib -import brotli from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse @@ -8,10 +7,18 @@ from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured +ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] + +try: + import brotli + ACCEPTED_ENCODINGS.append(b'br') +except ImportError: + pass + + class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): @@ -19,7 +26,8 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br') + request.headers.setdefault('Accept-Encoding', + ",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): @@ -57,7 +65,6 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) - if encoding == b"br": + if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) return body - diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c47de54ed..b47b267e2 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,6 +1,6 @@ from io import BytesIO from unittest import TestCase -from os.path import join, abspath, dirname +from os.path import join from gzip import GzipFile from scrapy.spiders import Spider @@ -20,6 +20,7 @@ FORMAT = { 'br': ('html-br.bin', 'br') } + class HttpCompressionTest(TestCase): def setUp(self): From 3daf473686aab89aa03ebd0ebc59a73b4b6e00f1 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 6 Feb 2017 12:29:33 +0100 Subject: [PATCH 0656/3444] [httpcompression] skip test if no brotli --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- tests/requirements.txt | 1 + tests/test_downloadermiddleware_httpcompression.py | 11 ++++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 04c9e355d..2fc1bb8eb 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -27,7 +27,7 @@ class HttpCompressionMiddleware(object): def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', - ",".join(ACCEPTED_ENCODINGS)) + b",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): diff --git a/tests/requirements.txt b/tests/requirements.txt index 9d0c3c996..9baa4be21 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,6 +6,7 @@ pytest==2.9.2 pytest-twisted pytest-cov==2.2.1 jmespath +brotlipy==0.6 testfixtures # optional for shell wrapper tests bpython diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b47b267e2..7924fb3b5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,11 +1,12 @@ from io import BytesIO -from unittest import TestCase +from unittest import TestCase, SkipTest from os.path import join from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ + ACCEPTED_ENCODINGS from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -53,7 +54,7 @@ class HttpCompressionTest(TestCase): assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) self.assertEqual(request.headers.get('Accept-Encoding'), - b'gzip,deflate,br') + b','.join(ACCEPTED_ENCODINGS)) def test_process_response_gzip(self): response = self._getresponse('gzip') @@ -66,6 +67,10 @@ class HttpCompressionTest(TestCase): assert 'Content-Encoding' not in newresponse.headers def test_process_response_br(self): + try: + import brotli + except ImportError: + raise SkipTest("no brotli") response = self._getresponse('br') request = response.request self.assertEqual(response.headers['Content-Encoding'], b'br') From af802bad14f833178bc4e03e3db3a86c44dcb735 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 6 Feb 2017 15:45:21 +0100 Subject: [PATCH 0657/3444] [httpcompression] add brotlipy for python 3 --- tests/requirements-py3.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index ed189c66c..d73a2300f 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -8,3 +8,4 @@ botocore # optional for shell wrapper tests bpython ipython +brotlipy==0.6 From f73eb715ac4374b1eafb1da3c1311c5a7c153b5e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 02:16:06 +0500 Subject: [PATCH 0658/3444] =?UTF-8?q?LinkExtractor:=20don=E2=80=99t=20chec?= =?UTF-8?q?k=20all=20regexes=20if=20one=20of=20them=20matches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/linkextractors/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index f51934b00..e5d21e174 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -40,7 +40,7 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) -_matches = lambda url, regexs: any((r.search(url) for r in regexs)) +_matches = lambda url, regexs: any(r.search(url) for r in regexs) _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} @@ -93,8 +93,8 @@ class FilteringLinkExtractor(object): if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): return False - allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] - denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] + allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) def _process_links(self, links): From 85a124970ad406c2680402256853dd3ff86d191f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 03:32:54 +0500 Subject: [PATCH 0659/3444] Enable memusage extension by default. Fixes GH-2187. --- docs/topics/settings.rst | 10 ++++++---- scrapy/settings/default_settings.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0515a9e0d..c1f488f73 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -827,13 +827,15 @@ Example:: MEMUSAGE_ENABLED ---------------- -Default: ``False`` +Default: ``True`` Scope: ``scrapy.extensions.memusage`` -Whether to enable the memory usage extension that will shutdown the Scrapy -process when it exceeds a memory limit, and also notify by email when that -happened. +Whether to enable the memory usage extension. This extension keeps track of +a peak memory used by the process (it writes it to stats). It can also +optionally shutdown the Scrapy process when it exceeds a memory limit +(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened +(see :setting:`MEMUSAGE_NOTIFY_MAIL`). See :ref:`topics-extensions-ref-memusage`. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..1cc169c9a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -207,7 +207,7 @@ MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 -MEMUSAGE_ENABLED = False +MEMUSAGE_ENABLED = True MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_REPORT = False From fb4ef21a1dcdc0c70fa80443e7379150639c42f6 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 7 Feb 2017 10:22:42 +0100 Subject: [PATCH 0660/3444] [httpcompression] minor style edits --- scrapy/downloadermiddlewares/httpcompression.py | 1 - tests/requirements-py3.txt | 2 +- tests/requirements.txt | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 2fc1bb8eb..19d6345e4 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ import zlib - from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d73a2300f..51a25f5e5 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -8,4 +8,4 @@ botocore # optional for shell wrapper tests bpython ipython -brotlipy==0.6 +brotlipy diff --git a/tests/requirements.txt b/tests/requirements.txt index 9baa4be21..c1576a2e7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,7 +6,7 @@ pytest==2.9.2 pytest-twisted pytest-cov==2.2.1 jmespath -brotlipy==0.6 +brotlipy testfixtures # optional for shell wrapper tests bpython From 24e82bfe75ad1a7b5ceb7a2840ecbae82d05268d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 23 Dec 2016 12:58:29 -0300 Subject: [PATCH 0661/3444] Validate values for components order --- scrapy/utils/conf.py | 9 +++++++++ tests/test_utils_conf.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e8af90f11..435e9a6b3 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,5 +1,6 @@ import os import sys +import numbers from operator import itemgetter import six @@ -34,6 +35,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in six.iteritems(compdict)} + def _validate_values(compdict): + """Fail if a value in the components dict is not a real number or None.""" + for name, value in six.iteritems(compdict): + if value is not None and not isinstance(value, numbers.Real): + raise ValueError('Invalid value {} for component {}, please provide ' \ + 'a real number or None instead'.format(value, name)) + # BEGIN Backwards compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) @@ -43,6 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): compdict.update(custom) # END Backwards compatibility + _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dab41ac8d..f203c32ef 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -62,6 +62,27 @@ class BuildComponentListTest(unittest.TestCase): self.assertRaises(ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower()) + def test_valid_numbers(self): + # work well with None and numeric values + d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0} + self.assertEqual(build_component_list(d, convert=lambda x: x), + ['d', 'a', 'c']) + d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222} + self.assertEqual(build_component_list(d, convert=lambda x: x), + ['b', 'c', 'a']) + # raise exception for invalid values + d = {'one': '5'} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': '1.0'} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': [1, 2, 3]} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': {'a': 'a', 'b': 2}} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': 'lorem ipsum',} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + + class UtilsConfTestCase(unittest.TestCase): From eaf62ab69c23459fa36dafe298bcfabc5952b7f7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 17:56:43 +0500 Subject: [PATCH 0662/3444] cleanup MetaRefreshMiddleware: remove redundant check --- scrapy/downloadermiddlewares/redirect.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index db276eefb..26677e527 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -53,8 +53,10 @@ class BaseRedirectMiddleware(object): class RedirectMiddleware(BaseRedirectMiddleware): - """Handle redirection of requests based on response status and meta-refresh html tag""" - + """ + Handle redirection of requests based on response status + and meta-refresh html tag. + """ def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or response.status in getattr(spider, 'handle_httpstatus_list', []) or @@ -92,10 +94,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): not isinstance(response, HtmlResponse): return response - if isinstance(response, HtmlResponse): - interval, url = get_meta_refresh(response) - if url and interval < self._maxdelay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + interval, url = get_meta_refresh(response) + if url and interval < self._maxdelay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response From 04b2f79e7a1316d29b2cb94c8fb2623e040b64ec Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 22:30:58 +0500 Subject: [PATCH 0663/3444] Remove code required to support ancient twisted versions. See GH-1887. --- scrapy/core/downloader/handlers/http.py | 8 +--- tests/mockserver.py | 39 +++++-------------- tests/test_downloader_handlers.py | 49 +++++++++--------------- tests/test_downloadermiddleware_retry.py | 5 +-- 4 files changed, 30 insertions(+), 71 deletions(-) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 81da2615a..e4a7d8564 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,10 +1,6 @@ -from scrapy import twisted_version +from __future__ import absolute_import from .http10 import HTTP10DownloadHandler - -if twisted_version >= (11, 1, 0): - from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler -else: - HTTPDownloadHandler = HTTP10DownloadHandler +from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler # backwards compatibility diff --git a/tests/mockserver.py b/tests/mockserver.py index a40e2e501..e611cc3ec 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -2,34 +2,18 @@ from __future__ import print_function import sys, time, random, os, json from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE + from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource -from twisted.internet import reactor, defer, ssl -from scrapy import twisted_version +from twisted.web.test.test_webclient import PayloadResource +from twisted.web.server import GzipEncoderFactory +from twisted.web.resource import EncodingResourceWrapper +from twisted.internet import reactor, ssl +from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode -if twisted_version < (11, 0, 0): - def deferLater(clock, delay, func, *args, **kw): - def _cancel_method(): - _cancel_cb(None) - d.errback(Exception()) - - def _cancel_cb(result): - if cl.active(): - cl.cancel() - return result - - d = defer.Deferred() - d.cancel = _cancel_method - d.addCallback(lambda ignored: func(*args, **kw)) - d.addBoth(_cancel_cb) - cl = clock.callLater(delay, d.callback, None) - return d -else: - from twisted.internet.task import deferLater - - def getarg(request, name, default=None, type=None): if name in request.args: value = request.args[name][0] @@ -174,13 +158,8 @@ class Root(Resource): self.putChild(b"drop", Drop()) self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) - - if twisted_version > (12, 3, 0): - from twisted.web.test.test_webclient import PayloadResource - from twisted.web.server import GzipEncoderFactory - from twisted.web.resource import EncodingResourceWrapper - self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"payload", PayloadResource()) + self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d3b49d6a..6333efceb 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -17,7 +17,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri -from scrapy import twisted_version from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler @@ -281,8 +280,6 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -366,8 +363,6 @@ class Https11InvalidDNSId(Https11TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' def setUp(self): self.mockserver = MockServer() @@ -396,31 +391,27 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): + crawler = get_crawler(SingleRequestSpider) + body = b'1' * 100 # PayloadResource requires body length to be 100 + request = Request('http://localhost:8998/payload', method='POST', + body=body, meta={'download_maxsize': 50}) + yield crawler.crawl(seed=request) + failure = crawler.spider.meta['failure'] + # download_maxsize < 100, hence the CancelledError + self.assertIsInstance(failure.value, defer.CancelledError) - if twisted_version > (12, 3, 0): - - crawler = get_crawler(SingleRequestSpider) - body = b'1'*100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) + if six.PY2: + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) - failure = crawler.spider.meta['failure'] - # download_maxsize < 100, hence the CancelledError - self.assertIsInstance(failure.value, defer.CancelledError) - - if six.PY2: - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') - else: - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') else: - raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload only enabled for PY2") class UriResource(resource.Resource): @@ -500,8 +491,6 @@ class Http10ProxyTestCase(HttpProxyTestCase): class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -692,8 +681,6 @@ class FTPTestCase(unittest.TestCase): username = "scrapy" password = "passwd" - if twisted_version < (10, 2, 0): - skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" if six.PY3: skip = "Twisted missing ftp support for PY3" diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index eb17974bf..e129b71f8 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionLost, TCPTimedOutError from twisted.web.client import ResponseFailed -from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.spiders import Spider from scrapy.http import Request, Response @@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase): def test_twistederrors(self): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, - ConnectError, ConnectionLost] - if twisted_version >= (11, 1, 0): # http11 available - exceptions.append(ResponseFailed) + ConnectError, ConnectionLost, ResponseFailed] for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) From 4e765acaed7a914630ee5320fa6f6523890a2b9d Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 24 Jan 2017 10:30:47 -0400 Subject: [PATCH 0664/3444] BUG: Fix __classcell__ propagation. Python 3.6 added simpler customization of class creation but this requires to propagate correctly the __classcell__ attribute in custom __new__ methods. See https://docs.python.org/3.6/whatsnew/3.6.html#pep-487-simpler- customization-of-class-creation --- scrapy/item.py | 3 +++ tests/test_item.py | 52 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scrapy/item.py b/scrapy/item.py index 138728a9a..aa05e9c69 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -25,6 +25,7 @@ class Field(dict): class ItemMeta(ABCMeta): def __new__(mcs, class_name, bases, attrs): + classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) @@ -39,6 +40,8 @@ class ItemMeta(ABCMeta): new_attrs['fields'] = fields new_attrs['_class'] = _class + if classcell is not None: + new_attrs['__classcell__'] = classcell return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) diff --git a/tests/test_item.py b/tests/test_item.py index dcb169c3a..85a554de0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,8 +1,14 @@ +import sys import unittest -from scrapy.item import Item, Field import six +from scrapy.item import ABCMeta, Item, ItemMeta, Field +from tests import mock + + +PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) + class ItemTest(unittest.TestCase): @@ -244,5 +250,49 @@ class ItemTest(unittest.TestCase): self.assertNotEqual(item['name'], copied_item['name']) +class ItemMetaTest(unittest.TestCase): + + def test_new_method_propagates_classcell(self): + new_mock = mock.Mock(side_effect=ABCMeta.__new__) + base = ItemMeta.__bases__[0] + + with mock.patch.object(base, '__new__', new_mock): + + class MyItem(Item): + if not PY36_PLUS: + # This attribute is an internal attribute in Python 3.6+ + # and must be propagated properly. See + # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object + # In <3.6, we add a dummy attribute just to ensure the + # __new__ method propagates it correctly. + __classcell__ = object() + + def f(self): + # For rationale of this see: + # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 + return __class__ + + MyItem() + + (first_call, second_call) = new_mock.call_args_list[-2:] + + mcs, class_name, bases, attrs = first_call[0] + assert '__classcell__' not in attrs + mcs, class_name, bases, attrs = second_call[0] + assert '__classcell__' in attrs + + +class ItemMetaClassCellRegression(unittest.TestCase): + + def test_item_meta_classcell_regression(self): + class MyItem(six.with_metaclass(ItemMeta, Item)): + def __init__(self, *args, **kwargs): + # This call to super() trigger the __classcell__ propagation + # requirement. When not done properly raises an error: + # TypeError: __class__ set to + # defining 'MyItem' as + super(MyItem, self).__init__(*args, **kwargs) + + if __name__ == "__main__": unittest.main() From 48c8c679de72da295aa753ffd9ed68b3958d3cbb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Feb 2017 18:03:48 +0100 Subject: [PATCH 0665/3444] Update changelog for upcoming 1.3.1 release --- docs/news.rst | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cce46599b..41374f970 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,54 @@ Release notes ============= +Scrapy 1.3.1 (2017-02-XX) +------------------------- + +New features +~~~~~~~~~~~~ + +- Support ``'True'`` and ``'False'`` string values for boolean settings (:issue:`2519`); + you can now do something like ``scrapy crawl myspider -s REDIRECT_ENABLED=False``. +- Support kwargs with ``response.xpath()`` to use :ref:`XPath variables ` + and ad-hoc namespaces declarations ; + this requires at least Parsel v1.1 (:issue:`2457`). +- Add support for Python 3.6 (:issue:`2485`). +- Run tests on PyPy (warning: some tests still fail, so PyPy is not supported yet). + +Bug fixes +~~~~~~~~~ + +- Enforce ``DNS_TIMEOUT`` setting (:issue:`2496`). +- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`). +- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`). +- Fix name of generated pipeline class when using basic project template (:issue:`2466`). +- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`). +- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). +- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, + ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + +Documentation +~~~~~~~~~~~~~ + +- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4 + (:issue:`2469`). +- Clarify that passing spider arguments converts them to spider attributes + (:issue:`2483`). +- Document ``formid`` argument on ``FormRequest.from_response()`` (:issue:`2497`). +- Add .rst extension to README files (:issue:`2507`). +- Mention LevelDB cache storage backend (:issue:`2525`). +- Use ``yield`` in sample callback code (:issue:`2533`). +- Add note about HTML entities decoding with ``.re()/.re_first()`` (:issue:`1704`). +- Typos (:issue:`2512`, :issue:`2534`, :issue:`2531`). + +Cleanups +~~~~~~~~ + +- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`). +- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`). +- Remove dead code supporting old Twisted versions (:issue:`2544`). + + Scrapy 1.3.0 (2016-12-21) ------------------------- From ff8a564b1a775ec16143b7c00199d40875868337 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 8 Feb 2017 17:05:06 +0100 Subject: [PATCH 0666/3444] Set date for 1.3.1 release --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 41374f970..3c1d24561 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.3.1 (2017-02-XX) +Scrapy 1.3.1 (2017-02-08) ------------------------- New features From af55a8713e561015b4e50eb5dd7061709770c67a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 8 Feb 2017 17:08:19 +0100 Subject: [PATCH 0667/3444] =?UTF-8?q?Bump=20version:=201.3.0=20=E2=86=92?= =?UTF-8?q?=201.3.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57ff603fa..0a8a71e8e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.0 +current_version = 1.3.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index f0bb29e76..3a3cd8cc8 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1 From 9c0aae724ed821fd954a14db83902a86f7fe7731 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Feb 2017 10:32:36 -0300 Subject: [PATCH 0668/3444] Use credentials from request.meta['proxy'] if present --- docs/topics/downloader-middleware.rst | 5 ++- scrapy/downloadermiddlewares/httpproxy.py | 24 +++++++---- tests/test_downloadermiddleware_httpproxy.py | 43 +++++++++++++++++--- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ca78ccc6..f0ff3c77c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,7 +681,10 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://username:password@some_proxy_server:port``. Keep in mind + this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment + variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 98c87aa9c..edc1c52ed 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,7 +8,6 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached -from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -20,23 +19,23 @@ class HttpProxyMiddleware(object): for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) - if not self.proxies: - raise NotConfigured - @classmethod def from_crawler(cls, crawler): auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) + def _basic_auth_header(self, username, password): + user_pass = to_bytes( + '%s:%s' % (unquote(username), unquote(password)), + encoding=self.auth_encoding) + return base64.b64encode(user_pass).strip() + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes( - '%s:%s' % (unquote(user), unquote(password)), - encoding=self.auth_encoding) - creds = base64.b64encode(user_pass).strip() + creds = self._basic_auth_header(user, password) else: creds = None @@ -45,6 +44,15 @@ class HttpProxyMiddleware(object): def process_request(self, request, spider): # ignore if proxy is already set if 'proxy' in request.meta: + if request.meta['proxy'] is None: + return + # extract credentials if present + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + request.meta['proxy'] = proxy_url + if creds and not request.headers.get('Proxy-Authorization'): + request.headers['Proxy-Authorization'] = b'Basic ' + creds + return + elif not self.proxies: return parsed = urlparse_cached(request) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 2b26431a4..dd09e4dd0 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -20,10 +20,6 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv - def test_no_proxies(self): - os.environ = {} - self.assertRaises(NotConfigured, HttpProxyMiddleware) - def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() @@ -47,6 +43,13 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.url, url) self.assertEquals(req.meta.get('proxy'), proxy) + def test_proxy_precedence_meta(self): + os.environ['http_proxy'] = 'https://proxy.com' + mw = HttpProxyMiddleware() + req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() @@ -54,6 +57,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -62,6 +70,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -72,6 +85,12 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') @@ -79,15 +98,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + # proxy from request.meta, latin-1 encoding + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + def test_proxy_already_seted(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() req = Request('http://noproxy.com', meta={'proxy': None}) assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None def test_no_proxy(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' @@ -104,3 +129,9 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta + + # proxy from meta['proxy'] takes precedence + os.environ['no_proxy'] = '*' + req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'http://proxy.com'}) From 29e60213db19030907dc8afd50173aa4421132df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Feb 2017 10:41:21 +0100 Subject: [PATCH 0669/3444] Use consistent selectors for author field in tutorial --- docs/intro/tutorial.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8e14d1b7c..3dc5ad2ed 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -225,7 +225,7 @@ You will see something like:: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> + >>> Using the shell, you can try selecting elements using `CSS`_ with the response object:: @@ -423,7 +423,7 @@ in the callback, as you can see below:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -522,7 +522,7 @@ page, extracting data from it:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -568,7 +568,7 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author+a::attr(href)').extract(): + for href in response.css('.author + a::attr(href)').extract(): yield scrapy.Request(response.urljoin(href), callback=self.parse_author) @@ -624,7 +624,7 @@ option when running them:: scrapy crawl quotes -o quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become -spider attributes by default. +spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes @@ -647,7 +647,7 @@ with a specific tag, building the URL based on the argument:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small a::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), } next_page = response.css('li.next a::attr(href)').extract_first() From 9956f198db54fa181856ad788e2ad8ee76ae5437 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 18:40:53 +0500 Subject: [PATCH 0670/3444] add a couple more lines to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b116640b4..406146e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ dist .idea htmlcov/ .coverage +.coverage.* +.cache/ # Windows Thumbs.db From de65ad3fb1e90f6fcffbcc74bfa6aff8ff65e14f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 18:44:39 +0500 Subject: [PATCH 0671/3444] TST replace Ubuntu 12.04 tox environment with 14.04 --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 0fdea11bb..bbf50b733 100644 --- a/tox.ini +++ b/tox.ini @@ -21,16 +21,16 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} -[testenv:precise] +[testenv:trusty] basepython = python2.7 deps = pyOpenSSL==0.13 - lxml==2.3.2 - Twisted==11.1.0 - boto==2.2.2 - Pillow<2.0 + lxml==3.3.3 + Twisted==13.2.0 + boto==2.20.1 + Pillow==2.3.0 cssselect==0.9.1 - zope.interface==3.6.1 + zope.interface==4.0.5 -rtests/requirements.txt [testenv:jessie] From 1bc4d8b6b6cf84c1785a6ad69abf37b4f0114bb3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 20:03:53 +0500 Subject: [PATCH 0672/3444] fixed tls in Twisted 17+ --- scrapy/core/downloader/tls.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 955b7630c..498e3d60f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,6 +1,8 @@ import logging from OpenSSL import SSL +from scrapy import twisted_version + logger = logging.getLogger(__name__) @@ -18,11 +20,17 @@ openssl_methods = { METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only } -# ClientTLSOptions requires a recent-enough version of Twisted -try: +if twisted_version >= (14, 0, 0): + # ClientTLSOptions requires a recent-enough version of Twisted. + # Not having ScrapyClientTLSOptions should not matter for older + # Twisted versions because it is not used in the fallback + # ScrapyClientContextFactory. # taken from twisted/twisted/internet/_sslverify.py + try: + # XXX: this try-except is not needed in Twisted 17.0.0+ because + # it requires pyOpenSSL 0.16+. from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START except ImportError: SSL_CB_HANDSHAKE_START = 0x10 @@ -30,10 +38,17 @@ try: from twisted.internet.ssl import AcceptableCiphers from twisted.internet._sslverify import (ClientTLSOptions, - _maybeSetHostNameIndication, verifyHostname, VerificationError) + if twisted_version < (17, 0, 0): + from twisted.internet._sslverify import _maybeSetHostNameIndication + set_tlsext_host_name = _maybeSetHostNameIndication + else: + def set_tlsext_host_name(connection, hostNameBytes): + connection.set_tlsext_host_name(hostNameBytes) + + class ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors @@ -46,7 +61,7 @@ try: def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL_CB_HANDSHAKE_START: - _maybeSetHostNameIndication(connection, self._hostnameBytes) + set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) @@ -62,8 +77,3 @@ try: self._hostnameASCII, repr(e))) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') - -except ImportError: - # ImportError should not matter for older Twisted versions - # as the above is not used in the fallback ScrapyClientContextFactory - pass From 9315e944a225a4010870333ff751fb2ed512f489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 13 Feb 2017 14:56:29 -0300 Subject: [PATCH 0673/3444] Release notes for 1.3.2 --- docs/news.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 3c1d24561..ff1e4ce03 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,16 @@ Release notes ============= +Scrapy 1.3.2 (2017-02-13) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`). +- Use consistent selectors for author field in tutorial (:issue:`2551`). +- Fix TLS compatibility in Twisted 17+ (:issue:`2558`) + Scrapy 1.3.1 (2017-02-08) ------------------------- From 7dd7646e6563798d408b8062059e826f08256b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 13 Feb 2017 14:57:55 -0300 Subject: [PATCH 0674/3444] =?UTF-8?q?Bump=20version:=201.3.1=20=E2=86=92?= =?UTF-8?q?=201.3.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0a8a71e8e..b95e0bad5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.1 +current_version = 1.3.2 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 3a3cd8cc8..1892b9267 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.3.1 +1.3.2 From 5b31dfe3c970f7e16dc911bb8b028a0fff54a7f9 Mon Sep 17 00:00:00 2001 From: terut Date: Mon, 13 Feb 2017 23:51:43 -0800 Subject: [PATCH 0675/3444] Separate building request from _requests_to_follow in CrawlSpider You just overwrite buiding request if you can use another request class because of something like splash-plugin. --- scrapy/spiders/crawl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 031f649d6..e5ac72e18 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -48,6 +48,11 @@ class CrawlSpider(Spider): def process_results(self, response, results): return results + def _build_request(self, rule, link): + r = Request(url=link.url, callback=self._response_downloaded) + r.meta.update(rule=rule, link_text=link.text) + return r + def _requests_to_follow(self, response): if not isinstance(response, HtmlResponse): return @@ -59,8 +64,7 @@ class CrawlSpider(Spider): links = rule.process_links(links) for link in links: seen.add(link) - r = Request(url=link.url, callback=self._response_downloaded) - r.meta.update(rule=n, link_text=link.text) + r = self._build_request(n, link) yield rule.process_request(r) def _response_downloaded(self, response): From ae0ea31abd4e65f7decb4482d1669d7af17bca0a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 8 Feb 2017 13:21:10 -0300 Subject: [PATCH 0676/3444] Add HTTPPROXY_ENABLED setting (default True) --- docs/topics/downloader-middleware.rst | 15 +++++++++++---- scrapy/downloadermiddlewares/httpproxy.py | 3 +++ scrapy/settings/default_settings.py | 1 + tests/test_downloadermiddleware_httpproxy.py | 8 ++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f0ff3c77c..0ef3fb071 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,10 +681,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://username:password@some_proxy_server:port``. Keep in mind - this value will take precedence over ``http_proxy``/``https_proxy`` - environment variables, and it will also ignore ``no_proxy`` environment - variable. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -952,8 +951,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index edc1c52ed..0d5320bf8 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,6 +8,7 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached +from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -21,6 +22,8 @@ class HttpProxyMiddleware(object): @classmethod def from_crawler(cls, crawler): + if not crawler.settings.getbool('HTTPPROXY_ENABLED'): + raise NotConfigured auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..cb88bc2bf 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index dd09e4dd0..c77179ceb 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,14 @@ import os import sys +from functools import partial from twisted.trial.unittest import TestCase, SkipTest from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.crawler import Crawler +from scrapy.settings import Settings spider = Spider('foo') @@ -20,6 +23,11 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv + def test_not_enabled(self): + settings = Settings({'HTTPPROXY_ENABLED': False}) + crawler = Crawler(spider, settings) + self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) + def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() From e285b1d6c2aaa1fdfe788f1894b0196bc64d1be1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 18:17:07 +0500 Subject: [PATCH 0677/3444] retry stats --- scrapy/downloadermiddlewares/retry.py | 14 ++++++++++++-- scrapy/downloadermiddlewares/stats.py | 4 +++- scrapy/utils/misc.py | 2 +- scrapy/utils/python.py | 11 +++++++++++ scrapy/utils/response.py | 3 ++- tests/test_downloadermiddleware_retry.py | 15 ++++++++++++--- tests/test_proxy_connect.py | 4 +++- 7 files changed, 44 insertions(+), 9 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c9c512be8..d84697b14 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -22,6 +22,7 @@ from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) @@ -35,16 +36,18 @@ class RetryMiddleware(object): ConnectionLost, TCPTimedOutError, ResponseFailed, IOError, TunnelError) - def __init__(self, settings): + def __init__(self, crawler): + settings = crawler.settings if not settings.getbool('RETRY_ENABLED'): raise NotConfigured self.max_retry_times = settings.getint('RETRY_TIMES') self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + self.stats = crawler.stats @classmethod def from_crawler(cls, crawler): - return cls(crawler.settings) + return cls(crawler) def process_response(self, request, response, spider): if request.meta.get('dont_retry', False): @@ -70,8 +73,15 @@ class RetryMiddleware(object): retryreq.meta['retry_times'] = retries retryreq.dont_filter = True retryreq.priority = request.priority + self.priority_adjust + + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + self.stats.inc_value('retry/count') + self.stats.inc_value('retry/reason_count/%s' % reason) return retryreq else: + self.stats.inc_value('retry/max_reached') logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 9c0ad90a5..ef0aafce0 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,8 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.request import request_httprepr from scrapy.utils.response import response_httprepr +from scrapy.utils.python import global_object_name + class DownloaderStats(object): @@ -27,6 +29,6 @@ class DownloaderStats(object): return response def process_exception(self, request, exception, spider): - ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) + ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 30c9e5058..35f855007 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -113,7 +113,7 @@ def md5sum(file): m.update(d) return m.hexdigest() + def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False - diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 42fbbda7f..4c500abf4 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -344,3 +344,14 @@ def without_none_values(iterable): return {k: v for k, v in six.iteritems(iterable) if v is not None} except AttributeError: return type(iterable)((v for v in iterable if v is not None)) + + +def global_object_name(obj): + """ + Return full name of a global object. + + >>> from scrapy import Request + >>> global_object_name(Request) + 'scrapy.http.request.Request' + """ + return "%s.%s" % (obj.__module__, obj.__name__) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index deb5741be..bf276b5ca 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -43,7 +43,8 @@ def get_meta_refresh(response): def response_status_message(status): """Return status code plus status text descriptive message """ - return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status"))) + message = http.RESPONSES.get(int(status), "Unknown Status") + return '%s %s' % (status, to_native_str(message)) def response_httprepr(response): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index e129b71f8..b833cb448 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -13,9 +13,9 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): - crawler = get_crawler(Spider) - self.spider = crawler._create_spider('foo') - self.mw = RetryMiddleware.from_crawler(crawler) + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): @@ -70,6 +70,10 @@ class RetryTest(unittest.TestCase): # discard it assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.crawler.stats.get_value('retry/max_reached') == 1 + assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2 + assert self.crawler.stats.get_value('retry/count') == 2 + def test_twistederrors(self): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, @@ -79,6 +83,11 @@ class RetryTest(unittest.TestCase): req = Request('http://www.scrapytest.org/%s' % exc.__name__) self._test_retry_exception(req, exc('foo')) + stats = self.crawler.stats + assert stats.get_value('retry/max_reached') == len(exceptions) + assert stats.get_value('retry/count') == len(exceptions) * 2 + assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2 + def _test_retry_exception(self, req, exception): # first retry req = self.mw.process_exception(req, exception, self.spider) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 0f06fd53d..6213a51e8 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -101,7 +101,9 @@ class ProxyConnectTestCase(TestCase): self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): + print(log) self.assertEqual(str(log).count('Crawled (%d)' % code), 1) def _assert_got_tunnel_error(self, log): - self.assertEqual(str(log).count('TunnelError'), 1) + print(log) + self.assertIn('TunnelError', str(log)) From 922d3fec54f40b0349c8c98dd8441aee3075cd65 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 14 Feb 2017 12:11:06 -0300 Subject: [PATCH 0678/3444] Doc: binary mode is required for exporters --- docs/topics/exporters.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index af469eb7b..85c73222d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -225,7 +225,8 @@ XmlItemExporter Exports Items in XML format to the specified file object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param root_element: The name of root element in the exported XML. :type root_element: str @@ -281,7 +282,8 @@ CsvItemExporter CSV columns and their order. The :attr:`export_empty_fields` attribute has no effect on this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from @@ -312,7 +314,8 @@ PickleItemExporter Exports Items in pickle format to the given file-like object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param protocol: The pickle protocol to use. :type protocol: int @@ -333,7 +336,8 @@ PprintItemExporter Exports Items in pretty print format to the specified file object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor. @@ -356,7 +360,8 @@ JsonItemExporter arguments to the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ constructor argument to customize this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) A typical output of this exporter would be:: @@ -386,7 +391,8 @@ JsonLinesItemExporter the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ constructor argument to customize this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) A typical output of this exporter would be:: From 39df675f091cf904dc904acd9538a0bebe2e55cd Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 14 Feb 2017 23:28:50 +0500 Subject: [PATCH 0679/3444] make retry middleware changes backwards compatible --- scrapy/downloadermiddlewares/retry.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index d84697b14..549d74f46 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -36,18 +36,16 @@ class RetryMiddleware(object): ConnectionLost, TCPTimedOutError, ResponseFailed, IOError, TunnelError) - def __init__(self, crawler): - settings = crawler.settings + def __init__(self, settings): if not settings.getbool('RETRY_ENABLED'): raise NotConfigured self.max_retry_times = settings.getint('RETRY_TIMES') self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') - self.stats = crawler.stats @classmethod def from_crawler(cls, crawler): - return cls(crawler) + return cls(crawler.settings) def process_response(self, request, response, spider): if request.meta.get('dont_retry', False): @@ -65,6 +63,7 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + stats = spider.crawler.stats if retries <= self.max_retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, @@ -77,11 +76,11 @@ class RetryMiddleware(object): if isinstance(reason, Exception): reason = global_object_name(reason.__class__) - self.stats.inc_value('retry/count') - self.stats.inc_value('retry/reason_count/%s' % reason) + stats.inc_value('retry/count') + stats.inc_value('retry/reason_count/%s' % reason) return retryreq else: - self.stats.inc_value('retry/max_reached') + stats.inc_value('retry/max_reached') logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) From e1ceaf3b5fa29326f032c4ed3f50943384b9e63d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 21:06:05 +0500 Subject: [PATCH 0680/3444] require w3lib 1.17+ --- requirements-py3.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 08ccf1958..cc0a7f644 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -3,5 +3,5 @@ lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 queuelib>=1.1.1 -w3lib>=1.14.2 +w3lib>=1.17.0 service_identity diff --git a/requirements.txt b/requirements.txt index f92603d3d..392f83dd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 -w3lib>=1.15.0 +w3lib>=1.17.0 queuelib six>=1.5.2 PyDispatcher>=2.0.5 diff --git a/setup.py b/setup.py index a6e6f9615..086ab8142 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ setup( ], install_requires=[ 'Twisted>=13.1.0', - 'w3lib>=1.15.0', + 'w3lib>=1.17.0', 'queuelib', 'lxml', 'pyOpenSSL', From 877057fac0d47e5ece95a55594706c91c8855883 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 06:00:09 +0500 Subject: [PATCH 0681/3444] initial response.follow implementation --- docs/intro/overview.rst | 3 +- docs/intro/tutorial.rst | 46 +++++++++++++++++++------ docs/topics/request-response.rst | 4 +++ scrapy/http/response/text.py | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 12 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7195017ff..1da1a4059 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination:: next_page = response.css('li.next a::attr("href")').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + yield response.follow(next_page, self.parse) Put this in a text file, name it to something like ``quotes_spider.py`` diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3dc5ad2ed..d47bf69e5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. +As a shortcut for creating Request objects you can use +:meth:`response.follow ` method:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + ] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +Unlike scrapy.Request, ``response.follow`` supports +relative URLs directly; you can also pass a selector to it instead of +a string. Note that ``response.follow`` just returns a Request instance; +you still have to yield this Request. + More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: - import scrapy @@ -568,15 +595,12 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author + a::attr(href)').extract(): - yield scrapy.Request(response.urljoin(href), - callback=self.parse_author) + for href in response.css('.author + a::attr(href)'): + yield response.follow(href, self.parse_author) # follow pagination links - next_page = response.css('li.next a::attr(href)').extract_first() - if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, self.parse) def parse_author(self, response): def extract_with_css(query): @@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also the pagination links with the ``parse`` callback as we saw before. +Here we're passing callbacks to ``response.follow`` as positional arguments +to make the code shorter; it also works for ``scrapy.Request``. + The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument:: next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, self.parse) + yield response.follow(next_page, self.parse) If you pass the ``tag=humor`` argument to this spider, you'll notice that it diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1fdd26043..71050fddd 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -683,6 +683,10 @@ TextResponse objects response.css('p') + .. method:: TextResponse.follow(url, ...) + + Return a scrapy.Request instance to follow a link ``url``. + .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5a6507aa8..1718b1f3b 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst import six from six.moves.urllib.parse import urljoin +import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding + +from scrapy.link import Link +from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url from scrapy.utils.python import memoizemethod_noargs, to_native_str @@ -116,3 +120,58 @@ class TextResponse(Response): def css(self, query): return self.selector.css(query) + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding=None, priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object (e.g. a link extractor result); + * attribute Selector (not SelectorList) - e.g. + ``response.css('a::attr(href)')[0]`` or + ``response.xpath('//img/@src')[0]``. + * a Selector for ```` element, e.g. + ``response.css('a.my_link')[0]``. + """ + if isinstance(url, Link): + url = url.url + elif isinstance(url, parsel.Selector): + url = _url_from_selector(url) + elif isinstance(url, parsel.SelectorList): + raise ValueError("Please pass either string") + + + encoding = self.encoding if encoding is None else encoding + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) + + +def _url_from_selector(sel): + # type: (parsel.Selector) -> str + if isinstance(sel.root, six.string_types): + # e.g. ::attr(href) result + return sel.root + if not hasattr(sel.root, 'tag'): + raise ValueError("Unsupported selector: %s" % sel) + if sel.root.tag != 'a': + raise ValueError("Only elements are supported; got <%s>" % + sel.root.tag) + href = sel.root.get('href') + if href is None: + raise ValueError(" element has no href attribute: %s" % sel) + return href From 71dd5d0bf9d0c41d70e72b0fd0a89528ef246065 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 06:11:08 +0500 Subject: [PATCH 0682/3444] strip URL extracted from selectors (as per html5 standard) --- scrapy/http/response/text.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1718b1f3b..5bfd2debb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -142,10 +142,9 @@ class TextResponse(Response): if isinstance(url, Link): url = url.url elif isinstance(url, parsel.Selector): - url = _url_from_selector(url) + url = _url_from_selector(url).strip() elif isinstance(url, parsel.SelectorList): - raise ValueError("Please pass either string") - + raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding url = self.urljoin(url) From 608c3f0c452bd508aa24bc9d4bf759e0b11e1683 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:17:41 +0500 Subject: [PATCH 0683/3444] handle whitespace in response.follow; add tests --- scrapy/http/response/text.py | 7 ++- tests/__init__.py | 7 ++- tests/test_http_response.py | 105 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5bfd2debb..6eacfbd35 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoin import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding +from w3lib.html import strip_html5_whitespace from scrapy.link import Link from scrapy.http.request import Request @@ -142,7 +143,7 @@ class TextResponse(Response): if isinstance(url, Link): url = url.url elif isinstance(url, parsel.Selector): - url = _url_from_selector(url).strip() + url = _url_from_selector(url) elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") @@ -164,7 +165,7 @@ def _url_from_selector(sel): # type: (parsel.Selector) -> str if isinstance(sel.root, six.string_types): # e.g. ::attr(href) result - return sel.root + return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): raise ValueError("Unsupported selector: %s" % sel) if sel.root.tag != 'a': @@ -173,4 +174,4 @@ def _url_from_selector(sel): href = sel.root.get('href') if href is None: raise ValueError(" element has no href attribute: %s" % sel) - return href + return strip_html5_whitespace(href) diff --git a/tests/__init__.py b/tests/__init__.py index d940f28ea..c2e4fd2bf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,9 +26,12 @@ try: except ImportError: import mock -tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') +tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'sample_data') + def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - return open(path, 'rb').read() + with open(path, 'rb') as f: + return f.read() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 9df3bf6e7..2a9baf5ed 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import unittest import six @@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse, from scrapy.selector import Selector from scrapy.utils.python import to_native_str from scrapy.exceptions import NotSupported +from scrapy.link import Link +from tests import get_testdata class BaseResponseTest(unittest.TestCase): @@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse + def _links_response(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + def test_html_encoding(self): body = b"""Some page @@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest): r1 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r1, 'gb2312', body) + def assert_followed_url(self, follow_obj, target_url, response=None): + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + self.assertEqual(req.url, target_url) + return req + + def test_follow_url_absolute(self): + self.assert_followed_url('http://foo.example.com', + 'http://foo.example.com') + + def test_follow_url_relative(self): + self.assert_followed_url('foo', + 'http://example.com/foo') + + def test_follow_link(self): + self.assert_followed_url(Link('http://example.com/foo'), + 'http://example.com/foo') + + def test_follow_selector(self): + resp = self._links_response() + urls = [ + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample3.html', + 'http://www.google.com/something', + 'http://example.com/innertag.html' + ] + + # select elements + for sellist in [resp.css('a'), resp.xpath('//a')]: + for sel, url in zip(sellist, urls): + self.assert_followed_url(sel, url, response=resp) + + # href attributes should work + for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sel, url in zip(sellist, urls): + self.assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css('img::attr(src)'): + self.assert_followed_url(src, 'http://example.com/sample2.jpg') + + def test_follow_whitespace_url(self): + self.assert_followed_url('foo ', + 'http://example.com/foo%20') + + def test_follow_whitespace_link(self): + self.assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo%20') + + def test_follow_whitespace_selector(self): + resp = self.response_class( + 'http://example.com', + body=b'''click me''' + ) + self.assert_followed_url(resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self.assert_followed_url(resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) + + def test_follow_encoding(self): + resp1 = self.response_class( + 'http://example.com', + encoding='utf8', + body='click me'.encode('utf8') + ) + req = self.assert_followed_url( + resp1.css('a')[0], + 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + response=resp1, + ) + self.assertEqual(req.encoding, 'utf8') + + resp2 = self.response_class( + 'http://example.com', + encoding='cp1251', + body='click me'.encode('cp1251') + ) + req = self.assert_followed_url( + resp2.css('a')[0], + 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + response=resp2, + ) + self.assertEqual(req.encoding, 'cp1251') + class XmlResponseTest(TextResponseTest): From 2674f317df9e4970f5953db3a6df04331246d8c9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:39:47 +0500 Subject: [PATCH 0684/3444] Response.follow --- scrapy/http/response/__init__.py | 29 +++++ scrapy/http/response/text.py | 27 ++-- tests/test_http_response.py | 204 +++++++++++++++---------------- 3 files changed, 143 insertions(+), 117 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 58ad414f1..e5fb4eef8 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -6,7 +6,9 @@ See documentation in docs/topics/request-response.rst """ from six.moves.urllib.parse import urljoin +from scrapy.http.request import Request from scrapy.http.headers import Headers +from scrapy.link import Link from scrapy.utils.trackref import object_ref from scrapy.http.common import obsolete_setter from scrapy.exceptions import NotSupported @@ -101,3 +103,30 @@ class Response(object_ref): is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding='utf-8', priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object. + """ + if isinstance(url, Link): + url = url.url + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6eacfbd35..3c360bcf9 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -140,25 +140,22 @@ class TextResponse(Response): * a Selector for ```` element, e.g. ``response.css('a.my_link')[0]``. """ - if isinstance(url, Link): - url = url.url - elif isinstance(url, parsel.Selector): + if isinstance(url, parsel.Selector): url = _url_from_selector(url) elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") - encoding = self.encoding if encoding is None else encoding - url = self.urljoin(url) - return Request(url, callback, - method=method, - headers=headers, - body=body, - cookies=cookies, - meta=meta, - encoding=encoding, - priority=priority, - dont_filter=dont_filter, - errback=errback) + return super(TextResponse, self).follow(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback + ) def _url_from_selector(sel): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2a9baf5ed..e64d0eeba 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -143,6 +143,38 @@ class BaseResponseTest(unittest.TestCase): r.css('body') r.xpath('//body') + def test_follow_url_absolute(self): + self._assert_followed_url('http://foo.example.com', + 'http://foo.example.com') + + def test_follow_url_relative(self): + self._assert_followed_url('foo', + 'http://example.com/foo') + + def test_follow_link(self): + self._assert_followed_url(Link('http://example.com/foo'), + 'http://example.com/foo') + + def test_follow_whitespace_url(self): + self._assert_followed_url('foo ', + 'http://example.com/foo%20') + + def test_follow_whitespace_link(self): + self._assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo%20') + + def _assert_followed_url(self, follow_obj, target_url, response=None): + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + self.assertEqual(req.url, target_url) + return req + + def _links_response(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + class TextResponseTest(BaseResponseTest): @@ -354,16 +386,81 @@ class TextResponseTest(BaseResponseTest): absolute = 'http://www.example.com/elsewhere/test' self.assertEqual(joined, absolute) + def test_follow_selector(self): + resp = self._links_response() + urls = [ + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample3.html', + 'http://www.google.com/something', + 'http://example.com/innertag.html' + ] + + # select elements + for sellist in [resp.css('a'), resp.xpath('//a')]: + for sel, url in zip(sellist, urls): + self._assert_followed_url(sel, url, response=resp) + + # href attributes should work + for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sel, url in zip(sellist, urls): + self._assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css('img::attr(src)'): + self._assert_followed_url(src, 'http://example.com/sample2.jpg') + + def test_follow_whitespace_selector(self): + resp = self.response_class( + 'http://example.com', + body=b'''click me''' + ) + self._assert_followed_url(resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self._assert_followed_url(resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) + + def test_follow_encoding(self): + resp1 = self.response_class( + 'http://example.com', + encoding='utf8', + body='click me'.encode('utf8') + ) + req = self._assert_followed_url( + resp1.css('a')[0], + 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + response=resp1, + ) + self.assertEqual(req.encoding, 'utf8') + + resp2 = self.response_class( + 'http://example.com', + encoding='cp1251', + body='click me'.encode('cp1251') + ) + req = self._assert_followed_url( + resp2.css('a')[0], + 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + response=resp2, + ) + self.assertEqual(req.encoding, 'cp1251') + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse - def _links_response(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - resp = self.response_class('http://example.com/index', body=body) - return resp - def test_html_encoding(self): body = b"""Some page @@ -396,103 +493,6 @@ class HtmlResponseTest(TextResponseTest): r1 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r1, 'gb2312', body) - def assert_followed_url(self, follow_obj, target_url, response=None): - if response is None: - response = self._links_response() - req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) - return req - - def test_follow_url_absolute(self): - self.assert_followed_url('http://foo.example.com', - 'http://foo.example.com') - - def test_follow_url_relative(self): - self.assert_followed_url('foo', - 'http://example.com/foo') - - def test_follow_link(self): - self.assert_followed_url(Link('http://example.com/foo'), - 'http://example.com/foo') - - def test_follow_selector(self): - resp = self._links_response() - urls = [ - 'http://example.com/sample2.html', - 'http://example.com/sample3.html', - 'http://example.com/sample3.html', - 'http://www.google.com/something', - 'http://example.com/innertag.html' - ] - - # select elements - for sellist in [resp.css('a'), resp.xpath('//a')]: - for sel, url in zip(sellist, urls): - self.assert_followed_url(sel, url, response=resp) - - # href attributes should work - for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: - for sel, url in zip(sellist, urls): - self.assert_followed_url(sel, url, response=resp) - - # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) - - def test_follow_selector_list(self): - resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) - - def test_follow_selector_attribute(self): - resp = self._links_response() - for src in resp.css('img::attr(src)'): - self.assert_followed_url(src, 'http://example.com/sample2.jpg') - - def test_follow_whitespace_url(self): - self.assert_followed_url('foo ', - 'http://example.com/foo%20') - - def test_follow_whitespace_link(self): - self.assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') - - def test_follow_whitespace_selector(self): - resp = self.response_class( - 'http://example.com', - body=b'''click me''' - ) - self.assert_followed_url(resp.css('a')[0], - 'http://example.com/foo', - response=resp) - self.assert_followed_url(resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) - - def test_follow_encoding(self): - resp1 = self.response_class( - 'http://example.com', - encoding='utf8', - body='click me'.encode('utf8') - ) - req = self.assert_followed_url( - resp1.css('a')[0], - 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', - response=resp1, - ) - self.assertEqual(req.encoding, 'utf8') - - resp2 = self.response_class( - 'http://example.com', - encoding='cp1251', - body='click me'.encode('cp1251') - ) - req = self.assert_followed_url( - resp2.css('a')[0], - 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', - response=resp2, - ) - self.assertEqual(req.encoding, 'cp1251') - class XmlResponseTest(TextResponseTest): From 160da6abab8954906181ce69593ff6e84d950ac1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:41:53 +0500 Subject: [PATCH 0685/3444] fixed tests in Python 2 --- tests/test_http_response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e64d0eeba..fa74b468b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -411,8 +411,8 @@ class TextResponseTest(BaseResponseTest): def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) + self.assertRaisesRegexp(ValueError, 'SelectorList', + resp.follow, resp.css('a')) def test_follow_selector_attribute(self): resp = self._links_response() @@ -435,7 +435,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body='click me'.encode('utf8') + body=u'click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -447,7 +447,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body='click me'.encode('cp1251') + body=u'click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], From 5b79c6a679b66868c89302a1693e5dedc62b6f61 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 00:06:52 +0500 Subject: [PATCH 0686/3444] DOC document response.follow methods; expand the tutorial --- docs/intro/tutorial.rst | 41 +++++++++++++++++++++++++------- docs/topics/request-response.rst | 7 +++--- scrapy/http/response/__init__.py | 15 ++++++------ scrapy/http/response/text.py | 18 +++++++------- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index d47bf69e5..3b3bd8d21 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary:: >>> Extracting data in our spider ------------------------------- +----------------------------- Let's get back to our spider. Until now, it doesn't extract any data in particular, just saves the whole HTML page to a local file. Let's integrate the @@ -551,8 +551,14 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. + +.. _response-follow-example: + +A shortcut for creating Requests +-------------------------------- + As a shortcut for creating Request objects you can use -:meth:`response.follow ` method:: +:meth:`response.follow `:: import scrapy @@ -571,13 +577,32 @@ As a shortcut for creating Request objects you can use 'tags': quote.css('div.tags a.tag::text').extract(), } - for href in response.css('li.next a::attr(href)'): - yield response.follow(href, callback=self.parse) + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) -Unlike scrapy.Request, ``response.follow`` supports -relative URLs directly; you can also pass a selector to it instead of -a string. Note that ``response.follow`` just returns a Request instance; -you still have to yield this Request. +Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no +need to call urljoin. Note that ``response.follow`` just returns a Request +instance; you still have to yield this Request. + +You can also pass a selector to ``response.follow`` instead of a string; +this selector should extract necessary attributes:: + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +For ```` elements there is a shortcut: ``response.follow`` uses their href +attribute automatically. So the code can be shortened further:: + + for a in response.css('li.next a'): + yield response.follow(a, callback=self.parse) + +.. note:: + + ``response.follow(response.css('li.next a'))`` is not valid because + ``response.css`` returns a list-like object with selectors for all results, + not a single selector. A ``for`` loop like in the example above, or + ``response.follow(response.css('li.next a')[0])`` is fine. More examples and patterns -------------------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 71050fddd..3e80f18b5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -597,6 +597,9 @@ Response objects urlparse.urljoin(response.url, url) + .. automethod:: Response.follow + + .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin .. _topics-request-response-ref-response-subclasses: @@ -683,9 +686,7 @@ TextResponse objects response.css('p') - .. method:: TextResponse.follow(url, ...) - - Return a scrapy.Request instance to follow a link ``url``. + .. automethod:: TextResponse.follow .. method:: TextResponse.body_as_unicode() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index e5fb4eef8..434d87eab 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -109,13 +109,14 @@ class Response(object_ref): dont_filter=False, errback=None): # type: (...) -> Request """ - Return a scrapy.Request instance to follow a link ``url``. - - ``url`` can be: - - * absolute URL; - * relative URL; - * scrapy.link.Link object. + Return a :class:`~.Request` instance to follow a link ``url``. + It accepts the same arguments as ``Request.__init__`` method, + but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, + not only an absolute URL. + + :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` + method which supports selectors in addition to absolute/relative URLs + and Link objects. """ if isinstance(url, Link): url = url.url diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 3c360bcf9..6415e191a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace -from scrapy.link import Link from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url @@ -127,18 +126,19 @@ class TextResponse(Response): dont_filter=False, errback=None): # type: (...) -> Request """ - Return a scrapy.Request instance to follow a link ``url``. - - ``url`` can be: - - * absolute URL; - * relative URL; - * scrapy.link.Link object (e.g. a link extractor result); - * attribute Selector (not SelectorList) - e.g. + Return a :class:`~.Request` instance to follow a link ``url``. + It accepts the same arguments as ``Request.__init__`` method, + but ``url`` can be not only an absolute URL, but also + + * a relative URL; + * a scrapy.link.Link object (e.g. a link extractor result); + * an attribute Selector (not SelectorList) - e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]``. * a Selector for ```` element, e.g. ``response.css('a.my_link')[0]``. + + See :ref:`response-follow-example` for usage examples. """ if isinstance(url, parsel.Selector): url = _url_from_selector(url) From fade5763af3d03f076f3317589038201bbdeccaf Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 02:02:50 +0500 Subject: [PATCH 0687/3444] TST more response.follow tests --- tests/test_http_response.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index fa74b468b..924bb7979 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -414,11 +414,24 @@ class TextResponseTest(BaseResponseTest): self.assertRaisesRegexp(ValueError, 'SelectorList', resp.follow, resp.css('a')) + def test_follow_selector_invalid(self): + resp = self._links_response() + self.assertRaisesRegexp(ValueError, 'Unsupported', + resp.follow, resp.xpath('count(//div)')[0]) + def test_follow_selector_attribute(self): resp = self._links_response() for src in resp.css('img::attr(src)'): self._assert_followed_url(src, 'http://example.com/sample2.jpg') + def test_follow_selector_no_href(self): + resp = self.response_class( + url='http://example.com', + body=b'click me', + ) + self.assertRaisesRegexp(ValueError, 'no href', + resp.follow, resp.css('a')[0]) + def test_follow_whitespace_selector(self): resp = self.response_class( 'http://example.com', From 074caf434e255bc96f106e57e3e288028f372485 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 9 Feb 2017 00:17:56 +0500 Subject: [PATCH 0688/3444] FormRequest: handle whitespaces in action attribute properly --- scrapy/http/request/form.py | 10 ++++++++-- tests/test_http_request.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2862dc096..905d8412f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +import six from six.moves.urllib.parse import urljoin, urlencode + import lxml.html from parsel.selector import create_root_node -import six +from w3lib.html import strip_html5_whitespace + from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url @@ -51,7 +54,10 @@ class FormRequest(Request): def _get_form_url(form, url): if url is None: - return urljoin(form.base_url, form.action) + action = form.get('action') + if action is None: + return form.base_url + return urljoin(form.base_url, strip_html5_whitespace(action)) return urljoin(form.base_url, url) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d7216e1d2..7eadb874f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -556,7 +556,6 @@ class FormRequestTest(RequestTest): fs = _qs(req, to_unicode=True, encoding='latin1') self.assertTrue(fs[u'price in \u00a5']) - def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( """ @@ -989,7 +988,7 @@ class FormRequestTest(RequestTest): """ - + @@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response) self.assertEqual(req.url, 'http://b.com/test_form') + def test_spaces_in_action(self): + resp = _buildresponse('') + req = self.request_class.from_response(resp) + self.assertEqual(req.url, 'http://example.com/path') + def test_from_response_css(self): response = _buildresponse( """
@@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formcss="input[name='abc']") + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) kwargs.setdefault('url', 'http://example.com') kwargs.setdefault('encoding', 'utf-8') return HtmlResponse(**kwargs) + def _qs(req, encoding='utf-8', to_unicode=False): if req.method == 'POST': qs = req.body From ad36a4a6ae8376a779f9feb08adfb2ca4a59dbb4 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 8 Feb 2017 16:58:38 +0500 Subject: [PATCH 0689/3444] RegexLinkExtractor: add \x0c to whitespace characters, as per html5 standard --- scrapy/linkextractors/regex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index 0fc7b079f..e689b4727 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -10,9 +10,10 @@ linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", re.DOTALL | re.IGNORECASE) + def clean_link(link_text): """Remove leading and trailing whitespace and punctuation""" - return link_text.strip("\t\r\n '\"") + return link_text.strip("\t\r\n '\"\x0c") class RegexLinkExtractor(SgmlLinkExtractor): From d079e15fe2fcf269d040ac435e2eb414d2b4c334 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 8 Feb 2017 17:03:11 +0500 Subject: [PATCH 0690/3444] Strip leading/trailing whitespaces in link extractors. Fixes GH-838. --- docs/topics/link-extractors.rst | 10 +++++++++- scrapy/linkextractors/htmlparser.py | 8 ++++++-- scrapy/linkextractors/lxmlhtml.py | 16 ++++++++++----- scrapy/linkextractors/sgml.py | 13 ++++++++---- scrapy/utils/url.py | 13 ++++++++++++ .../link_extractor/sgml_linkextractor.html | 1 + tests/test_linkextractors.py | 3 +++ tests/test_linkextractors_deprecated.py | 20 +++++++++++++------ 8 files changed, 66 insertions(+), 18 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 4636ddb18..2486e0982 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -51,7 +51,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None) +.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True) LxmlLinkExtractor is the recommended link extractor with handy filtering options. It is implemented using lxml's robust HTMLParser. @@ -132,4 +132,12 @@ LxmlLinkExtractor :type process_value: callable + :param strip: whether to strip whitespaces from extracted attributes. + According to HTML5 standard, leading and trailing whitespaces + must be stripped from ``href`` attributes of ```` and ```` + elements, so LinkExtractor strips them by default. Set ``strip=False`` + to turn it off (e.g. if you're extracting urls from elements or + attributes which allow leading/trailing whitespaces). + :type strip: boolean + .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 9867e1179..4841e4a54 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -1,7 +1,6 @@ """ HTMLParser-based link extractor """ - import warnings import six from six.moves.html_parser import HTMLParser @@ -11,12 +10,14 @@ from w3lib.url import safe_url_string from scrapy.link import Link from scrapy.utils.python import unique as unique_list +from scrapy.utils.url import trim_href_attribute from scrapy.exceptions import ScrapyDeprecationWarning class HtmlParserLinkExtractor(HTMLParser): - def __init__(self, tag="a", attr="href", process=None, unique=False): + def __init__(self, tag="a", attr="href", process=None, unique=False, + strip=True): HTMLParser.__init__(self) warnings.warn( @@ -29,6 +30,7 @@ class HtmlParserLinkExtractor(HTMLParser): self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique + self.strip = strip def _extract_links(self, response_text, response_url, response_encoding): self.reset() @@ -70,6 +72,8 @@ class HtmlParserLinkExtractor(HTMLParser): for attr, value in attrs: if self.scan_attr(attr): url = self.process_attr(value) + if self.strip: + url = trim_href_attribute(url) link = Link(url=url) self.links.append(link) self.current_link = link diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 71d57b392..f753033ab 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -9,8 +9,9 @@ import lxml.etree as etree from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list, to_native_str -from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.response import get_base_url +from scrapy.utils.url import trim_href_attribute +from scrapy.linkextractors import FilteringLinkExtractor # from lxml/src/lxml/html/__init__.py @@ -27,11 +28,13 @@ def _nons(tag): class LxmlParserLinkExtractor(object): - def __init__(self, tag="a", attr="href", process=None, unique=False): + def __init__(self, tag="a", attr="href", process=None, unique=False, + strip=True): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique + self.strip = strip def _iter_links(self, document): for el in document.iter(etree.Element): @@ -49,9 +52,11 @@ class LxmlParserLinkExtractor(object): for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) try: + if self.strip: + attr_val = trim_href_attribute(attr_val) attr_val = urljoin(base_url, attr_val) except ValueError: - continue # skipping bogus links + continue # skipping bogus links else: url = self.process_attr(attr_val) if url is None: @@ -85,12 +90,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, - unique=True, process_value=None, deny_extensions=None, restrict_css=()): + unique=True, process_value=None, deny_extensions=None, restrict_css=(), + strip=True): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process=process_value) + unique=unique, process=process_value, strip=strip) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index c68dae4c8..6ecfd52aa 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -7,18 +7,19 @@ import warnings from sgmllib import SGMLParser from w3lib.url import safe_url_string -from scrapy.selector import Selector from scrapy.link import Link from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list, to_unicode from scrapy.utils.response import get_base_url +from scrapy.utils.url import trim_href_attribute from scrapy.exceptions import ScrapyDeprecationWarning class BaseSgmlLinkExtractor(SGMLParser): - def __init__(self, tag="a", attr="href", unique=False, process_value=None): + def __init__(self, tag="a", attr="href", unique=False, process_value=None, + strip=True): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -30,6 +31,7 @@ class BaseSgmlLinkExtractor(SGMLParser): self.process_value = (lambda v: v) if process_value is None else process_value self.current_link = None self.unique = unique + self.strip = strip def _extract_links(self, response_text, response_url, response_encoding, base_url=None): """ Do the real extraction work """ @@ -81,6 +83,8 @@ class BaseSgmlLinkExtractor(SGMLParser): if self.scan_attr(attr): url = self.process_value(value) if url is not None: + if self.strip: + url = trim_href_attribute(url) link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel'))) self.links.append(link) self.current_link = link @@ -103,7 +107,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, - process_value=None, deny_extensions=None, restrict_css=()): + process_value=None, deny_extensions=None, restrict_css=(), + strip=True): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " @@ -118,7 +123,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value) + unique=unique, process_value=process_value, strip=strip) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..090f65f80 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -103,3 +103,16 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def trim_href_attribute(href): + """ + Process href attribute of ``a`` or ``area`` elements according to HTML5 + standards (strip all leading and trailing whitespaces). References: + + * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements + * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces + * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace + * https://www.w3.org/TR/html5/infrastructure.html#space-character + """ + return href.strip(' \t\n\r\x0c') diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html index 35aa457ee..fbb803f2d 100644 --- a/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -13,6 +13,7 @@ sample 3 repetition inner tag +href with whitespaces
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 129336d14..340c64f35 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -32,6 +32,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), ]) def test_extract_filter_allow(self): @@ -281,6 +282,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) @@ -291,6 +293,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 36dfe174f..fef227aa1 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -117,12 +117,14 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): def test_extraction(self): # Default arguments lx = HtmlParserLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) + self.assertEqual(lx.extract_links(self.response), [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + ]) def test_link_wrong_href(self): html = """ @@ -220,3 +222,9 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual([link for link in lx.extract_links(response)], [ Link(url='http://b.com/test.html', text=u'', nofollow=False), ]) + + @unittest.expectedFailure + def test_extraction(self): + # RegexLinkExtractor doesn't parse URLs with leading/trailing + # whitespaces correctly. + super(RegexLinkExtractorTestCase, self).test_extraction() From d09eed7674b5df2a1883a7c7abad40fdfd062c74 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 8 Feb 2017 23:44:55 +0500 Subject: [PATCH 0691/3444] use w3lib.html.strip_html5_whitespace function; expand docs; strip consistently before calling process_value --- docs/topics/link-extractors.rst | 9 +++++---- scrapy/linkextractors/htmlparser.py | 6 +++--- scrapy/linkextractors/lxmlhtml.py | 6 +++--- scrapy/linkextractors/sgml.py | 7 ++++--- scrapy/utils/url.py | 13 ------------- 5 files changed, 15 insertions(+), 26 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 2486e0982..75bdb4142 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -134,10 +134,11 @@ LxmlLinkExtractor :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces - must be stripped from ``href`` attributes of ```` and ```` - elements, so LinkExtractor strips them by default. Set ``strip=False`` - to turn it off (e.g. if you're extracting urls from elements or - attributes which allow leading/trailing whitespaces). + must be stripped from ``href`` attributes of ````, ```` + and many other elements, ``src`` attribute of ````, ``
`` elements denoting the individual +the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader From bcce8d3d80462946483a403a1fb25b3e12229136 Mon Sep 17 00:00:00 2001 From: orangain Date: Thu, 17 Dec 2015 14:48:37 +0900 Subject: [PATCH 0031/3444] DOC: Update MetaRefreshMiddlware's setting variables * `REDIRECT_MAX_METAREFRESH_DELAY` has been deprecated and was renamed to `METAREFRESH_MAXDELAY`. * Merge duplicate documents about `METAREFRESH_MAXDELAY` appeared both in the settings page and the downloader-middlewares page. --- docs/topics/downloader-middleware.rst | 8 +++++--- docs/topics/settings.rst | 10 ---------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index cc0254d29..3641da231 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -787,14 +787,16 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. -.. setting:: REDIRECT_MAX_METAREFRESH_DELAY +.. setting:: METAREFRESH_MAXDELAY -REDIRECT_MAX_METAREFRESH_DELAY -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +METAREFRESH_MAXDELAY +^^^^^^^^^^^^^^^^^^^^ Default: ``100`` The maximum meta-refresh delay (in seconds) to follow the redirection. +Some sites use meta-refresh for redirecting to a session expired page, so we +restrict automatic redirection to the maximum delay. RetryMiddleware --------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 60b5f4585..cc070d8c0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -857,16 +857,6 @@ Defines the maximum times a request can be redirected. After this maximum the request's response is returned as is. We used Firefox default value for the same task. -.. setting:: REDIRECT_MAX_METAREFRESH_DELAY - -REDIRECT_MAX_METAREFRESH_DELAY ------------------------------- - -Default: ``100`` - -Some sites use meta-refresh for redirecting to a session expired page, so we -restrict automatic redirection to a maximum delay (in seconds) - .. setting:: REDIRECT_PRIORITY_ADJUST REDIRECT_PRIORITY_ADJUST From 4f49aab7c068c553c5dcb2935ee3e4df6c0b71aa Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 18 Dec 2015 16:16:05 -0500 Subject: [PATCH 0032/3444] BF: robustify _monkeypatches check for twisted - str() name first (Closes: #1634) In my case, while running datalad tests using nose, scrapy was failing since v was None --- scrapy/_monkeypatches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 782891326..60e0de1f2 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -20,6 +20,6 @@ if sys.version_info[0] == 2: import twisted.persisted.styles # NOQA # Remove only entries with twisted serializers for non-twisted types. for k, v in frozenset(copyreg.dispatch_table.items()): - if not getattr(k, '__module__', '').startswith('twisted') \ - and getattr(v, '__module__', '').startswith('twisted'): + if not str(getattr(k, '__module__', '')).startswith('twisted') \ + and str(getattr(v, '__module__', '')).startswith('twisted'): copyreg.dispatch_table.pop(k) From f57121c77be82ad7521c3f4ed5cda17ada118ef4 Mon Sep 17 00:00:00 2001 From: Aron Bordin Date: Wed, 30 Dec 2015 13:10:13 -0200 Subject: [PATCH 0033/3444] show download warnsize once --- scrapy/core/downloader/handlers/http11.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..403accef8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -292,6 +292,7 @@ class _ResponseReader(protocol.Protocol): self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize + self._reached_warnsize = False self._bytes_received = 0 def dataReceived(self, bodyBytes): @@ -305,11 +306,12 @@ class _ResponseReader(protocol.Protocol): 'maxsize': self._maxsize}) self._finished.cancel() - if self._warnsize and self._bytes_received > self._warnsize: - logger.warning("Received (%(bytes)s) bytes larger than download " - "warn size (%(warnsize)s).", - {'bytes': self._bytes_received, - 'warnsize': self._warnsize}) + if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: + self._reached_warnsize = True + logger.warning("Received more bytes than download " + "warn size (%(warnsize)s) in request %(request)s.", + {'warnsize': self._warnsize, + 'request': self._request}) def connectionLost(self, reason): if self._finished.called: From 1b435b2887ac4e19159cd4416f9396e022d0382a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 30 Dec 2015 15:43:04 -0300 Subject: [PATCH 0034/3444] Add 1.0.4 release notes --- docs/news.rst | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5df1b1a6a..4d7dc4d41 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,58 @@ Release notes ============= +1.0.4 (2015-12-30) +------------------ + +- Ignoring xlib/tx folder, depending on Twisted version. (:commit:`7dfa979`) +- Run on new travis-ci infra (:commit:`6e42f0b`) +- Spelling fixes (:commit:`823a1cc`) +- escape nodename in xmliter regex (:commit:`da3c155`) +- test xml nodename with dots (:commit:`4418fc3`) +- TST don't use broken Pillow version in tests (:commit:`a55078c`) +- disable log on version command. closes #1426 (:commit:`86fc330`) +- disable log on startproject command (:commit:`db4c9fe`) +- Add PyPI download stats badge (:commit:`df2b944`) +- don't run tests twice on Travis if a PR is made from a scrapy/scrapy branch (:commit:`a83ab41`) +- Add Python 3 porting status badge to the README (:commit:`73ac80d`) +- fixed RFPDupeFilter persistence (:commit:`97d080e`) +- TST a test to show that dupefilter persistence is not working (:commit:`97f2fb3`) +- explicit close file on file:// scheme handler (:commit:`d9b4850`) +- Disable dupefilter in shell (:commit:`c0d0734`) +- DOC: Add captions to toctrees which appear in sidebar (:commit:`aa239ad`) +- DOC Removed pywin32 from install instructions as it's already declared as dependency. (:commit:`10eb400`) +- Added installation notes about using Conda for Windows and other OSes. (:commit:`1c3600a`) +- Fixed minor grammar issues. (:commit:`7f4ddd5`) +- fixed a typo in the documentation. (:commit:`b71f677`) +- Version 1 now exists (:commit:`5456c0e`) +- fix another invalid xpath error (:commit:`0a1366e`) +- fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`) +- Typos corrections (:commit:`7067117`) +- fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`) +- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`) +- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`) +- Reference Homebrew's homepage for installation instructions (:commit:`1925db1`) +- Add oldest supported tox version to contributing docs (:commit:`5d10d6d`) +- Note in install docs about pip being already included in python>=2.7.9 (:commit:`85c980e`) +- Add non-python dependencies to Ubuntu install section in the docs (:commit:`fbd010d`) +- Add OS X installation section to docs (:commit:`d8f4cba`) +- DOC(ENH): specify path to rtd theme explicitly (:commit:`de73b1a`) +- minor: scrapy.Spider docs grammar (:commit:`1ddcc7b`) +- Make common practices sample code match the comments (:commit:`1b85bcf`) +- nextcall repetitive calls (heartbeats). (:commit:`55f7104`) +- Backport fix compatibility with Twisted 15.4.0 (:commit:`b262411`) +- pin pytest to 2.7.3 (:commit:`a6535c2`) +- Merge pull request #1512 from mgedmin/patch-1 (:commit:`8876111`) +- Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`) +- Typo (:commit:`f8d0682`) +- Fix list formatting (:commit:`5f83a93`) +- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`) +- Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`) +- Update tutorial.rst (:commit:`fbc1f25`) +- Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`) +- Small grammatical change (:commit:`8752294`) +- Add openssl version to version command (:commit:`13c45ac`) + 1.0.3 (2015-08-11) ------------------ From c702c5301523e57ce276d80c8106f9a52853948d Mon Sep 17 00:00:00 2001 From: palego Date: Sun, 3 Jan 2016 14:33:42 +0100 Subject: [PATCH 0035/3444] change os.mknod() for open() os.mknod() is a privileged command on OS X, making the test fail --- tests/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 5755b3881..8edccd4bd 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -83,7 +83,8 @@ class StartprojectTemplatesTest(ProjectTest): def test_startproject_template_override(self): copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) - os.mknod(join(self.tmpl_proj, 'root_template')) + with open(join(self.tmpl_proj, 'root_template'), 'w'): + pass assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] From d03d262f5290d7a439ac797947413894f158f425 Mon Sep 17 00:00:00 2001 From: palego Date: Mon, 4 Jan 2016 10:00:13 +0100 Subject: [PATCH 0036/3444] indentation --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 803ed5fc0..3324aa21a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -56,7 +56,7 @@ class RetryMiddleware(object): def process_exception(self, request, exception, spider): if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \ and not request.meta.get('dont_retry', False): - return self._retry(request, exception, spider) + return self._retry(request, exception, spider) def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 From 2abc9bc901491b24ea1a35058ae2b86e44492c88 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Jr Date: Wed, 6 Jan 2016 10:29:45 -0200 Subject: [PATCH 0037/3444] Update deprecated examples * update the scrapy.org example to deal with the new layout. * replaced slashdot.org by reddit.com, because it seems that slashdot is blocking requests. --- docs/topics/shell.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 2b118bfbd..3569cbf37 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -106,10 +106,10 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -http://scrapy.org page, and then proceed to scrape the http://slashdot.org -page. Finally, we modify the (Slashdot) request method to POST and re-fetch it -getting a HTTP 405 (method not allowed) error. We end the session by typing -Ctrl-D (in Unix systems) or Ctrl-Z in Windows. +http://scrapy.org page, and then proceed to scrape the http://reddit.com +page. Finally, we modify the (Reddit) request method to POST and re-fetch it +getting an error. We end the session by typing Ctrl-D (in Unix systems) or +Ctrl-Z in Windows. Keep in mind that the data extracted here may not be the same when you try it, as those pages are not static and could have changed by the time you test this. @@ -140,24 +140,24 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: - >>> response.xpath("//h1/text()").extract()[0] - u'Meet Scrapy' + >>> response.xpath('//title/text()').extract_first() + u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' - >>> fetch("http://slashdot.org") + >>> fetch("http://reddit.com") [s] Available Scrapy objects: - [s] crawler + [s] crawler [s] item {} - [s] request - [s] response <200 http://slashdot.org> - [s] settings - [s] spider + [s] request + [s] response <200 https://www.reddit.com/> + [s] settings + [s] spider [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() - [u'Slashdot: News for nerds, stuff that matters'] + [u'reddit: the front page of the internet'] >>> request = request.replace(method="POST") From d4872940dbb450fcd0fa688766af0aa6dfccc861 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 6 Jan 2016 21:21:21 +0100 Subject: [PATCH 0038/3444] PY3: port utils/iterators --- scrapy/utils/iterators.py | 14 ++++++++++---- tests/py3-ignores.txt | 1 - tests/test_utils_iterators.py | 21 +++++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c0d93f7a9..ed286f5c5 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -8,6 +8,8 @@ except ImportError: from io import BytesIO import six +if six.PY3: + from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector @@ -65,7 +67,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, unicode) + self._is_unicode = isinstance(self._text, six.text_type) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -94,7 +96,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. - + quotechar is the character used to enclosure fields on the given obj. """ @@ -102,7 +104,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _getrow(csv_r): return [to_unicode(field, encoding) for field in next(csv_r)] - lines = BytesIO(_body_or_str(obj, unicode=False)) + # Python 3 csv reader input object needs to return strings + if six.PY3: + lines = StringIO(_body_or_str(obj, unicode=True)) + else: + lines = BytesIO(_body_or_str(obj, unicode=False)) kwargs = {} if delimiter: kwargs["delimiter"] = delimiter @@ -125,7 +131,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types)), \ + assert isinstance(obj, (Response, six.string_types, six.binary_type)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 55ed75c92..015578e1e 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -16,7 +16,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_iterators.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 590c53302..d42ed2c91 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,5 @@ import os +import six from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml @@ -99,7 +100,7 @@ class XmliterTestCase(unittest.TestCase): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( - self.xmliter(response, 'item').next().extract(), + next(self.xmliter(response, 'item')).extract(), u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -189,11 +190,11 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assert_(all((isinstance(k, unicode) for k in result_row.keys()))) - self.assert_(all((isinstance(v, unicode) for v in result_row.values()))) + self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys()))) + self.assert_(all((isinstance(v, six.text_type) for v in result_row.values()))) def test_csviter_delimiter(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -205,8 +206,8 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') - body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|') - + body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|') + response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") @@ -237,7 +238,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') response = Response(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -249,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() - headers, body = sample[0].split(','), '\n'.join(sample[1:]) + headers, body = sample[0].split(b','), b'\n'.join(sample[1:]) response = TextResponse(url="http://example.com/", body=body) - csv = csviter(response, headers=headers) + csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, @@ -262,7 +263,7 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') - body = '\n'.join((body, 'a,b', 'a,b,c,d')) + body = b'\n'.join((body, b'a,b', b'a,b,c,d')) response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) From 6ddd8147382348f4796ac905f0357925dfa81da1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jan 2016 10:48:45 +0100 Subject: [PATCH 0039/3444] Support unicode tags in xml iterators (fixes #1665) --- scrapy/utils/iterators.py | 10 +++---- tests/test_utils_iterators.py | 51 ++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c0d93f7a9..bec985062 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def xmliter(obj, nodename): """Return a iterator of Selector's over all nodes of a XML document, - given tha name of the node to iterate. Useful for parsing XML feeds. + given the name of the node to iterate. Useful for parsing XML feeds. obj can be: - a Response object @@ -36,7 +36,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL) + r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -49,7 +49,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node) + nodetext = etree.tostring(node, encoding='unicode') node.clear() xs = Selector(text=nodetext, type='xml') if namespace: @@ -94,7 +94,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. - + quotechar is the character used to enclosure fields on the given obj. """ @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types)), \ + assert isinstance(obj, (Response, six.string_types, bytes)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 590c53302..8dceed7cc 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os from twisted.trial import unittest @@ -45,6 +46,54 @@ class XmliterTestCase(unittest.TestCase): for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) + def test_xmliter_unicode(self): + # example taken from https://github.com/scrapy/scrapy/issues/1665 + body = """ + <þingflokkar> + <þingflokkur id="26"> + + + - + + + + 80 + + + <þingflokkur id="21"> + Alþýðubandalag + + Ab + Alþb. + + + 76 + 123 + + + <þingflokkur id="27"> + Alþýðuflokkur + + A + Alþfl. + + + 27 + 120 + + + """ + response = XmlResponse(url="http://example.com", body=body) + attrs = [] + for x in self.xmliter(response, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" @@ -206,7 +255,7 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|') - + response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") From d7d4ef67a697243143df969e32b8ed956394f4fb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jan 2016 11:08:49 +0100 Subject: [PATCH 0040/3444] Changes following comments --- scrapy/utils/iterators.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index ed286f5c5..ce59c9719 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,15 +1,12 @@ import re import csv import logging - try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import StringIO import six -if six.PY3: - from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector @@ -131,7 +128,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types, six.binary_type)), \ + assert isinstance(obj, (Response, six.string_types, bytes)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: From 9fad25f3d14091d250cc4b1d668befca00c30ef0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 11:42:41 +0100 Subject: [PATCH 0041/3444] Use explicit Unicode and bytes for XML body in tests --- scrapy/utils/iterators.py | 9 ++++++--- tests/test_utils_iterators.py | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c215a0bdd..69c7f2c23 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding='unicode') + nodetext = etree.tostring(node, encoding=six.text_type) node.clear() xs = Selector(text=nodetext, type='xml') if namespace: @@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types, bytes)), \ - "obj must be Response or basestring, not %s" % type(obj).__name__ + expected_types = (Response, six.text_type, six.binary_type) + assert isinstance(obj, expected_types), \ + "obj must be %s, not %s" % ( + " or ".join(t.__name__ for t in expected_types), + type(obj).__name__) if isinstance(obj, Response): if not unicode: return obj.body diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index de103fea5..74c22d420 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -49,7 +49,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = """ + body = u""" <þingflokkar> <þingflokkur id="26"> @@ -84,7 +84,22 @@ class XmliterTestCase(unittest.TestCase): """ - response = XmlResponse(url="http://example.com", body=body) + + # with bytes + response = XmlResponse(url="http://example.com", body=body.encode('utf-8')) + attrs = [] + for x in self.xmliter(response, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) + + # Unicode body needs encoding information + response = XmlResponse(url="http://example.com", body=body, encoding='utf-8') attrs = [] for x in self.xmliter(response, u'þingflokkur'): attrs.append((x.xpath('@id').extract(), From d4c7d72b2b6fc20c1df0f697dd60801b93848628 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:13:47 +0100 Subject: [PATCH 0042/3444] Add tests for input type in xmliter calls --- tests/test_utils_iterators.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 74c22d420..8c4d6cf59 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -160,6 +160,10 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + def test_xmliter_objtype_exception(self): + i = self.xmliter(42, 'product') + self.assertRaises(AssertionError, next, i) + def test_xmliter_encoding(self): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) @@ -233,6 +237,9 @@ class LxmlXmliterTestCase(XmliterTestCase): node = next(my_iter) self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table']) + def test_xmliter_objtype_exception(self): + i = self.xmliter(42, 'product') + self.assertRaises(TypeError, next, i) class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') From 1347015a80b9d8dafe0e1ac067d65b7e0e7c3f84 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:32:28 +0100 Subject: [PATCH 0043/3444] Refactored test code --- tests/test_utils_iterators.py | 37 +++++++++++++---------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 8c4d6cf59..b2e3889a4 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -85,31 +85,22 @@ class XmliterTestCase(unittest.TestCase): """ - # with bytes - response = XmlResponse(url="http://example.com", body=body.encode('utf-8')) - attrs = [] - for x in self.xmliter(response, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + for r in ( + # with bytes + XmlResponse(url="http://example.com", body=body.encode('utf-8')), + # Unicode body needs encoding information + XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + attrs = [] + for x in self.xmliter(r, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) - # Unicode body needs encoding information - response = XmlResponse(url="http://example.com", body=body, encoding='utf-8') - attrs = [] - for x in self.xmliter(response, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) - - self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" From a93d49a64ca170d98de98ee44a181ced04a23bea Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:47:42 +0100 Subject: [PATCH 0044/3444] Add Python 3.5 tox env and Python 3.4 tests in Travis CI --- .travis.yml | 1 + tox.ini | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index e857abbd8..65cfaad03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 + - TOXENV=py34 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index eae7e8e47..b8d45d5b9 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,10 @@ deps = basepython = python3.4 deps = {[testenv:py33]deps} +[testenv:py35] +basepython = python3.5 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps = From f3889b0bce84cbd46852799a7bc95128c4c3b2d5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 11:41:49 +0300 Subject: [PATCH 0045/3444] py3 compat: encode delimiter, method and path in ScrapyHTTPPageGetter --- scrapy/core/downloader/webclient.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index add5576ef..c335939d0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,6 +7,7 @@ from twisted.internet import defer from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes from scrapy.responsetypes import responsetypes @@ -29,13 +30,17 @@ def _parse(url): class ScrapyHTTPPageGetter(HTTPClient): - delimiter = '\n' + delimiter = b'\n' def connectionMade(self): self.headers = Headers() # bucket for response headers # Method command - self.sendCommand(self.factory.method, self.factory.path) + self.sendCommand( + to_bytes(self.factory.method, encoding='ascii'), + # XXX - do we need to percent-encode path somewhere? + # https://en.wikipedia.org/wiki/Percent-encoding#Character_data + to_bytes(self.factory.path)) # Headers for key, values in self.factory.headers.items(): for value in values: From 9f2be23a3957ff17b252c47958affb9f5fcd6e72 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 11:42:23 +0300 Subject: [PATCH 0046/3444] webclient tests, py3: fix setUp, pass test_getPage --- tests/test_webclient.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index e0b46286a..d56c9f683 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -14,18 +14,21 @@ from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client from scrapy.http import Request, Headers +from scrapy.utils.python import to_bytes, to_unicode def getPage(url, contextFactory=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" - def _clientfactory(*args, **kwargs): + def _clientfactory(url, *args, **kwargs): + url = to_unicode(url) timeout = kwargs.pop('timeout', 0) - f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout) + f = client.ScrapyHTTPClientFactory( + Request(url, *args, **kwargs), timeout=timeout) f.deferred.addCallback(lambda r: r.body) return f from twisted.web.client import _makeGetterFactory - return _makeGetterFactory(url, _clientfactory, + return _makeGetterFactory(to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs).deferred @@ -212,7 +215,7 @@ class WebClientTestCase(unittest.TestCase): def setUp(self): name = self.mktemp() os.mkdir(name) - FilePath(name).child("file").setContent("0123456789") + FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) r.putChild("redirect", util.Redirect("/file")) r.putChild("wait", ForeverTakingResource()) @@ -250,7 +253,7 @@ class WebClientTestCase(unittest.TestCase): the body of the response if the default method B{GET} is used. """ d = getPage(self.getURL("file")) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d From 73ff87c1dc05e98104f854d16792e302625c5e98 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:03:08 +0300 Subject: [PATCH 0047/3444] decode body from utf-8, as scrapy stores body as bytes, and twisted has already converted to unicode --- scrapy/core/downloader/webclient.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c335939d0..ba7bd798c 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -132,6 +132,9 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self.url) + # XXX - scrapy response stores body as bytes, + # but maybe it makes sense to be able to store unicode? + body = to_bytes(body) return respcls(url=self.url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): From 1d5ab671833b1e054bf15a1b562a33b5ac5f1af7 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:04:26 +0300 Subject: [PATCH 0048/3444] pass test_getPageHead on py3 --- tests/test_webclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d56c9f683..3784aa401 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -266,8 +266,8 @@ class WebClientTestCase(unittest.TestCase): def _getPage(method): return getPage(self.getURL("file"), method=method) return defer.gatherResults([ - _getPage("head").addCallback(self.assertEqual, ""), - _getPage("HEAD").addCallback(self.assertEqual, "")]) + _getPage("head").addCallback(self.assertEqual, b""), + _getPage("HEAD").addCallback(self.assertEqual, b"")]) def test_timeoutNotTriggering(self): From 945674eb8f351e71aea090f1e94c3fa337c52cf8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:25:54 +0300 Subject: [PATCH 0049/3444] pass test_externalUnicodeInterference - the logic for py3 is clearly inverse of what was expected in this test, as scrapy Request url must be unicode --- tests/test_webclient.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3784aa401..84717e5eb 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -3,6 +3,7 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ import os +import six from six.moves.urllib.parse import urlparse from twisted.trial import unittest @@ -75,8 +76,10 @@ class ParseUrlTestCase(unittest.TestCase): elements of its return tuple, even when passed an URL which has previously been passed to L{urlparse} as a C{unicode} string. """ - badInput = u'http://example.com/path' - goodInput = badInput.encode('ascii') + goodInput = u'http://example.com/path' + badInput = goodInput.encode('ascii') + if six.PY2: + goodInput, badInput = badInput, goodInput urlparse(badInput) scheme, netloc, host, port, path = self._parse(goodInput) self.assertTrue(isinstance(scheme, str)) From 325b6af6c2c55bcf47fe339cfe139b357eb1673e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:29:59 +0300 Subject: [PATCH 0050/3444] fix ScrapyHTTPPageGetterTests for py3 - we expect bytes here --- tests/test_webclient.py | 78 ++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 84717e5eb..84b1011b8 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -105,22 +105,22 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): 'Useful': 'value'})) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Content-Length: 9\r\n" - "Useful: value\r\n" - "Connection: close\r\n" - "User-Agent: fooble\r\n" - "Host: example.net\r\n" - "Cookie: blah blah\r\n" - "\r\n" - "some data") + b"GET /bar HTTP/1.0\r\n" + b"Content-Length: 9\r\n" + b"Useful: value\r\n" + b"Connection: close\r\n" + b"User-Agent: fooble\r\n" + b"Host: example.net\r\n" + b"Cookie: blah blah\r\n" + b"\r\n" + b"some data") # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "\r\n") + b"GET /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"\r\n") # test a simple POST with body and content-type factory = client.ScrapyHTTPClientFactory(Request( @@ -130,13 +130,13 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers={'Content-Type': 'application/x-www-form-urlencoded'})) self._test(factory, - "POST /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "Connection: close\r\n" - "Content-Type: application/x-www-form-urlencoded\r\n" - "Content-Length: 10\r\n" - "\r\n" - "name=value") + b"POST /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"Connection: close\r\n" + b"Content-Type: application/x-www-form-urlencoded\r\n" + b"Content-Length: 10\r\n" + b"\r\n" + b"name=value") # test a POST method with no body provided factory = client.ScrapyHTTPClientFactory(Request( @@ -145,10 +145,10 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): )) self._test(factory, - "POST /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "Content-Length: 0\r\n" - "\r\n") + b"POST /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"Content-Length: 0\r\n" + b"\r\n") # test with single and multivalued headers factory = client.ScrapyHTTPClientFactory(Request( @@ -159,12 +159,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): })) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "X-Meta-Multivalued: value1\r\n" - "X-Meta-Multivalued: value2\r\n" - "X-Meta-Single: single\r\n" - "\r\n") + b"GET /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"X-Meta-Multivalued: value1\r\n" + b"X-Meta-Multivalued: value2\r\n" + b"X-Meta-Single: single\r\n" + b"\r\n") # same test with single and multivalued headers but using Headers class factory = client.ScrapyHTTPClientFactory(Request( @@ -175,12 +175,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }))) self._test(factory, - "GET /bar HTTP/1.0\r\n" - "Host: foo\r\n" - "X-Meta-Multivalued: value1\r\n" - "X-Meta-Multivalued: value2\r\n" - "X-Meta-Single: single\r\n" - "\r\n") + b"GET /bar HTTP/1.0\r\n" + b"Host: foo\r\n" + b"X-Meta-Multivalued: value1\r\n" + b"X-Meta-Multivalued: value2\r\n" + b"X-Meta-Single: single\r\n" + b"\r\n") def _test(self, factory, testvalue): transport = StringTransport() @@ -199,10 +199,10 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.headers = Headers() - protocol.dataReceived("HTTP/1.0 200 OK\n") - protocol.dataReceived("Hello: World\n") - protocol.dataReceived("Foo: Bar\n") - protocol.dataReceived("\n") + protocol.dataReceived(b"HTTP/1.0 200 OK\n") + protocol.dataReceived(b"Hello: World\n") + protocol.dataReceived(b"Foo: Bar\n") + protocol.dataReceived(b"\n") self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']})) From 85b0e6c9c766218bd48268f0c115c77b3886e539 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 14 Jan 2016 10:50:51 +0100 Subject: [PATCH 0051/3444] Travis: run tox with Python 3.5 + add Python 3.5 tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65cfaad03..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 2.7 +python: 3.5 sudo: false branches: only: @@ -9,7 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 - - TOXENV=py34 + - TOXENV=py35 - TOXENV=docs install: - pip install -U tox twine wheel codecov From ae4aa2c3b24a967556f602a031520c535ebeb77d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:04:10 +0300 Subject: [PATCH 0052/3444] py3 test fix: putChild expects bytes as path --- tests/test_webclient.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 84b1011b8..02e24de05 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -220,13 +220,13 @@ class WebClientTestCase(unittest.TestCase): os.mkdir(name) FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) - r.putChild("redirect", util.Redirect("/file")) - r.putChild("wait", ForeverTakingResource()) - r.putChild("error", ErrorResource()) - r.putChild("nolength", NoLengthResource()) - r.putChild("host", HostHeaderResource()) - r.putChild("payload", PayloadResource()) - r.putChild("broken", BrokenDownloadResource()) + r.putChild(b"redirect", util.Redirect("/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"error", ErrorResource()) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = self._listen(self.wrapper) From b5f9bc8499293aced54f54250a18e148af356267 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:04:45 +0300 Subject: [PATCH 0053/3444] py3 test fixes in test_webclient - expect bytes as page body --- tests/test_webclient.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 02e24de05..05038c407 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -240,14 +240,17 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s) + return getPage(self.getURL("payload"), body=s).addCallback( + self.assertEquals, to_bytes(s)) def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ - getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno), - getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")]) + getPage(self.getURL("host")).addCallback( + self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)), + getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( + self.assertEquals, to_bytes("www.example.com"))]) def test_getPage(self): @@ -280,7 +283,8 @@ class WebClientTestCase(unittest.TestCase): called back with the contents of the page. """ d = getPage(self.getURL("host"), timeout=100) - d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno) + d.addCallback( + self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)) return d @@ -309,7 +313,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assert_('404 - No Such Resource' in pageData) + self.assert_(b'404 - No Such Resource' in pageData) def testFactoryInfo(self): url = self.getURL('file') @@ -329,6 +333,6 @@ class WebClientTestCase(unittest.TestCase): def _cbRedirect(self, pageData): self.assertEquals(pageData, - '\n\n \n \n' - ' \n \n ' - 'click here\n \n\n') + b'\n\n \n \n' + b' \n \n ' + b'click here\n \n\n') From 88f55312af0d9551f00686d79eaaf1c122740aa1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:43:37 +0300 Subject: [PATCH 0054/3444] py3 fix in testFactoryInfo - factory attirbutes are bytes in twisted --- tests/test_webclient.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 05038c407..0a31d8b5d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -323,10 +323,10 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEquals(factory.status, '200') - self.assert_(factory.version.startswith('HTTP/')) - self.assertEquals(factory.message, 'OK') - self.assertEquals(factory.response_headers['content-length'], '10') + self.assertEquals(factory.status, b'200') + self.assert_(factory.version.startswith(b'HTTP/')) + self.assertEquals(factory.message, b'OK') + self.assertEquals(factory.response_headers[b'content-length'], b'10') def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) From 30c7b4e4cc5411af1015aa292e20e4c453607428 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:44:14 +0300 Subject: [PATCH 0055/3444] py3 compat in test_timeoutTriggering cleanup --- tests/test_webclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0a31d8b5d..fa0083d44 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -300,7 +300,7 @@ class WebClientTestCase(unittest.TestCase): def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything. - connected = self.wrapper.protocols.keys() + connected = list(six.iterkeys(self.wrapper.protocols)) # There might be nothing here if the server managed to already see # that the connection was lost. if connected: From 01783561781e1620580e0f1b66f32a514ac2b960 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:46:46 +0300 Subject: [PATCH 0056/3444] py3 fix testRedirect: url is bytes here --- tests/test_webclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index fa0083d44..66f8ed4cf 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -220,7 +220,7 @@ class WebClientTestCase(unittest.TestCase): os.mkdir(name) FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) - r.putChild(b"redirect", util.Redirect("/file")) + r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"error", ErrorResource()) r.putChild(b"nolength", NoLengthResource()) From 6a412d25037d40dfbcc6392b6a862488c153e46f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 13:48:48 +0300 Subject: [PATCH 0057/3444] all tests pass in test_webclient.py on py3 - removing from py3-ignores --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 015578e1e..9e75ecf92 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -17,7 +17,6 @@ tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py tests/test_utils_template.py -tests/test_webclient.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py @@ -29,7 +28,6 @@ scrapy/core/downloader/handlers/s3.py scrapy/core/downloader/handlers/http11.py scrapy/core/downloader/handlers/http.py scrapy/core/downloader/handlers/ftp.py -scrapy/core/downloader/webclient.py scrapy/pipelines/images.py scrapy/pipelines/files.py scrapy/linkextractors/sgml.py From e5fb6094384f8d96e8a677b5fd9f2e6bfe2b97d3 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 14:45:02 +0300 Subject: [PATCH 0058/3444] make ScrapyHTTPClientFactory comply to twisted HTTPClientFactory protocol - use bytes (encoding are likely wrong at this stage) --- scrapy/core/downloader/webclient.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ba7bd798c..841322bdd 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,13 +12,15 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): + b = lambda x: to_bytes(x, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - host = parsed.hostname + path = to_bytes(path) # FIXME + host = b(parsed.hostname) # FIXME port = parsed.port - scheme = parsed.scheme - netloc = parsed.netloc + scheme = b(parsed.scheme) + netloc = b(parsed.netloc) # FIXME - host + port if port is None: - port = 443 if scheme == 'https' else 80 + port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path @@ -36,11 +38,7 @@ class ScrapyHTTPPageGetter(HTTPClient): self.headers = Headers() # bucket for response headers # Method command - self.sendCommand( - to_bytes(self.factory.method, encoding='ascii'), - # XXX - do we need to percent-encode path somewhere? - # https://en.wikipedia.org/wiki/Percent-encoding#Character_data - to_bytes(self.factory.path)) + self.sendCommand(self.factory.method, self.factory.path) # Headers for key, values in self.factory.headers.items(): for value in values: @@ -96,8 +94,10 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): afterFoundGet = False def __init__(self, request, timeout=180): - self.url = urldefrag(request.url)[0] - self.method = request.method + self._url = urldefrag(request.url)[0] + # converting to bytes to comply to Twisted interface + self.url = to_bytes(self._url) # FIXME + self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers) self.response_headers = None @@ -131,11 +131,11 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): request.meta['download_latency'] = self.headers_time-self.start_time status = int(self.status) headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self.url) + respcls = responsetypes.from_args(headers=headers, url=self._url) # XXX - scrapy response stores body as bytes, # but maybe it makes sense to be able to store unicode? body = to_bytes(body) - return respcls(url=self.url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) From ac2cf191d1868f19897a108a861e3170485a676a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 14:50:58 +0300 Subject: [PATCH 0059/3444] py3: remove comments, utf-8 is fine here: as twisted ultimately uses urllib.parse.quote that assepts bytes and assumes utf-8 --- scrapy/core/downloader/webclient.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 841322bdd..2d848aee0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,13 +12,12 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): - b = lambda x: to_bytes(x, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - path = to_bytes(path) # FIXME - host = b(parsed.hostname) # FIXME + path = to_bytes(path) + host = to_bytes(parsed.hostname) port = parsed.port - scheme = b(parsed.scheme) - netloc = b(parsed.netloc) # FIXME - host + port + scheme = to_bytes(parsed.scheme, encoding='ascii') + netloc = to_bytes(parsed.netloc) if port is None: port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path @@ -96,7 +95,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url) # FIXME + self.url = to_bytes(self._url) self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers) From 8df35bcac6190ab92635e46009c2e099c8e6010d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 14:52:44 +0300 Subject: [PATCH 0060/3444] rm note to self: to be discussed in PR --- scrapy/core/downloader/webclient.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 2d848aee0..15d14ae49 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -131,8 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - # XXX - scrapy response stores body as bytes, - # but maybe it makes sense to be able to store unicode? body = to_bytes(body) return respcls(url=self._url, status=status, headers=headers, body=body) From 5c2241ccc7a886ea48ce12c2fb67cee5db24ce4e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 15:30:28 +0300 Subject: [PATCH 0061/3444] py3: fix webclient tests after making ScrapyHTTPClientFactory use bytes as in twisted --- scrapy/core/downloader/webclient.py | 4 ++-- tests/test_webclient.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 15d14ae49..d2cdd6f98 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -65,7 +65,7 @@ class ScrapyHTTPPageGetter(HTTPClient): self.factory.noPage(reason) def handleResponse(self, response): - if self.factory.method.upper() == 'HEAD': + if self.factory.method.upper() == b'HEAD': self.factory.page('') elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) @@ -123,7 +123,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") # Content-Length must be specified in POST method even with no body - elif self.method == 'POST': + elif self.method == b'POST': self.headers['Content-Length'] = 0 def _build_response(self, body, request): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 66f8ed4cf..412e10c89 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -68,6 +68,8 @@ class ParseUrlTestCase(unittest.TestCase): ) for url, test in tests: + test = tuple( + to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEquals(client._parse(url), test, url) def test_externalUnicodeInterference(self): @@ -82,10 +84,10 @@ class ParseUrlTestCase(unittest.TestCase): goodInput, badInput = badInput, goodInput urlparse(badInput) scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, str)) - self.assertTrue(isinstance(netloc, str)) - self.assertTrue(isinstance(host, str)) - self.assertTrue(isinstance(path, str)) + self.assertTrue(isinstance(scheme, bytes)) + self.assertTrue(isinstance(netloc, bytes)) + self.assertTrue(isinstance(host, bytes)) + self.assertTrue(isinstance(path, bytes)) self.assertTrue(isinstance(port, int)) @@ -317,9 +319,9 @@ class WebClientTestCase(unittest.TestCase): def testFactoryInfo(self): url = self.getURL('file') - scheme, netloc, host, port, path = client._parse(url) + _, _, host, port, _ = client._parse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(host, port, factory) + reactor.connectTCP(to_unicode(host), port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): From 32bb5b682a7cf4f2baca78914f18297c120eef2d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:11:16 +0300 Subject: [PATCH 0062/3444] fix import of test_downloader_handlers.py: use @implementer, move failing on py3 imports into corresponding tests --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_downloader_handlers.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..7c937a036 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -6,7 +6,7 @@ from io import BytesIO from time import time from six.moves.urllib.parse import urldefrag -from zope.interface import implements +from zope.interface import implementer from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH @@ -265,8 +265,8 @@ class ScrapyAgent(object): return respcls(url=url, status=status, headers=headers, body=body, flags=flags) +@implementer(IBodyProducer) class _RequestBodyProducer(object): - implements(IBodyProducer) def __init__(self, body): self.body = body diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d2a349b40..5f1703c5c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -10,9 +10,7 @@ from twisted.web import server, static, util, resource from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource -from twisted.protocols.ftp import FTPRealm, FTPFactory from twisted.cred import portal, checkers, credentials -from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri from scrapy import twisted_version @@ -22,7 +20,6 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownlo from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.spiders import Spider from scrapy.http import Request @@ -520,6 +517,9 @@ class FTPTestCase(unittest.TestCase): skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" def setUp(self): + from twisted.protocols.ftp import FTPRealm, FTPFactory + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler + # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) @@ -601,6 +601,8 @@ class FTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_invalid_credentials(self): + from twisted.protocols.ftp import ConnectionLost + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": 'invalid'}) d = self.download_handler.download_request(request, None) From 3509378b8be0df7cb38d3823068288b5daa37612 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:29:19 +0300 Subject: [PATCH 0063/3444] py3: pass first http downloader test, simple crawler works now, yay! --- scrapy/core/downloader/handlers/http11.py | 9 ++++++--- scrapy/http/response/__init__.py | 3 ++- tests/test_downloader_handlers.py | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7c937a036..34070ebc6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -19,6 +19,7 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object +from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version logger = logging.getLogger(__name__) @@ -200,8 +201,8 @@ class ScrapyAgent(object): agent = self._get_agent(request, timeout) # request details - url = urldefrag(request.url)[0] - method = request.method + url = to_bytes(urldefrag(request.url)[0]) + method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader('Proxy-Authorization') @@ -261,8 +262,10 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) + url = to_unicode(url) respcls = responsetypes.from_args(headers=headers, url=url) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags) + return respcls( + url=url, status=status, headers=headers, body=body, flags=flags) @implementer(IBodyProducer) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 983154001..59ef15682 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ +import six from six.moves.urllib.parse import urljoin from scrapy.http.headers import Headers @@ -34,7 +35,7 @@ class Response(object_ref): return self._url def _set_url(self, url): - if isinstance(url, str): + if isinstance(url, six.string_types): self._url = url else: raise TypeError('%s url must be str, got %s:' % (type(self).__name__, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 5f1703c5c..cdb1ad02d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -88,7 +88,7 @@ class FileTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.url, request.url) self.assertEquals(response.status, 200) - self.assertEquals(response.body, '0123456789') + self.assertEquals(response.body, b'0123456789') request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -107,15 +107,15 @@ class HttpTestCase(unittest.TestCase): def setUp(self): name = self.mktemp() os.mkdir(name) - FilePath(name).child("file").setContent("0123456789") + FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) - r.putChild("redirect", util.Redirect("/file")) - r.putChild("wait", ForeverTakingResource()) - r.putChild("hang-after-headers", ForeverTakingResource(write=True)) - r.putChild("nolength", NoLengthResource()) - r.putChild("host", HostHeaderResource()) - r.putChild("payload", PayloadResource()) - r.putChild("broken", BrokenDownloadResource()) + r.putChild(b"redirect", util.Redirect(b"/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') @@ -136,7 +136,7 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d def test_download_head(self): From 6b79fffa9a4c53cb6a6af2e9d9251b95c28496b4 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:37:18 +0300 Subject: [PATCH 0064/3444] py3: pass all of HttpTestCase --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_downloader_handlers.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 34070ebc6..dbb002710 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -182,7 +182,7 @@ class ScrapyAgent(object): _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] omitConnectTunnel = proxyParams.find('noconnect') >= 0 - if scheme == 'https' and not omitConnectTunnel: + if scheme == b'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, request.headers.get('Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, @@ -233,7 +233,7 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, '', None + return txresponse, b'', None maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index cdb1ad02d..c017a9eb2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,4 @@ import os -import twisted import six from twisted.trial import unittest @@ -25,6 +24,7 @@ from scrapy.spiders import Spider from scrapy.http import Request from scrapy.settings import Settings from scrapy.utils.test import get_crawler +from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured from tests.mockserver import MockServer @@ -143,7 +143,7 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, '') + d.addCallback(self.assertEquals, b'') return d def test_redirect_status(self): @@ -175,7 +175,7 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals(response.body, '127.0.0.1:%d' % self.portno) + self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno)) self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) @@ -183,19 +183,19 @@ class HttpTestCase(unittest.TestCase): def test_host_header_seted_in_request_headers(self): def _test(response): - self.assertEquals(response.body, 'example.com') - self.assertEquals(request.headers.get('Host'), 'example.com') + self.assertEquals(response.body, b'example.com') + self.assertEquals(request.headers.get('Host'), b'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, 'example.com') + d.addCallback(self.assertEquals, b'example.com') return d def test_payload(self): - body = '1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) From c6f14a39de14b15b78d3fd4a098ec246536e12c4 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 16:50:16 +0300 Subject: [PATCH 0065/3444] py3: fix http10 downloader - unicode host expected here --- scrapy/core/downloader/handlers/http10.py | 5 +++-- tests/test_downloader_handlers.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 11b2acdae..0322bbe49 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -2,6 +2,7 @@ """ from twisted.internet import reactor from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode class HTTP10DownloadHandler(object): @@ -17,8 +18,8 @@ class HTTP10DownloadHandler(object): return factory.deferred def _connect(self, factory): - host, port = factory.host, factory.port - if factory.scheme == 'https': + host, port = to_unicode(factory.host), factory.port + if factory.scheme == b'https': return reactor.connectSSL(host, port, factory, self.ClientContextFactory()) else: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c017a9eb2..780f08806 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -223,7 +223,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d @defer.inlineCallbacks @@ -234,7 +234,7 @@ class Http11TestCase(HttpTestCase): # response body. (regardless of headers) d = self.download_request(request, Spider('foo', download_maxsize=10)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") yield d d = self.download_request(request, Spider('foo', download_maxsize=9)) @@ -257,7 +257,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo', download_maxsize=100)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, "0123456789") + d.addCallback(self.assertEquals, b"0123456789") return d From 4950f5988ef1df5bc6b6ec2c4a70a7956f64b539 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 17:24:08 +0300 Subject: [PATCH 0066/3444] py3: pass http proxy tests --- scrapy/core/downloader/handlers/http11.py | 7 ++++--- tests/test_downloader_handlers.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index dbb002710..d81093a9f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -181,10 +181,11 @@ class ScrapyAgent(object): if proxy: _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - omitConnectTunnel = proxyParams.find('noconnect') >= 0 + proxyHost = to_unicode(proxyHost) + omitConnectTunnel = proxyParams.find(b'noconnect') >= 0 if scheme == b'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, - request.headers.get('Proxy-Authorization', None)) + request.headers.get(b'Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) @@ -205,7 +206,7 @@ class ScrapyAgent(object): method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): - headers.removeHeader('Proxy-Authorization') + headers.removeHeader(b'Proxy-Authorization') bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 780f08806..ebf1d2f9c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -351,7 +351,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, 'http://example.com') + self.assertEquals(response.body, b'http://example.com') http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -361,7 +361,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, 'https://example.com') + self.assertEquals(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) @@ -371,7 +371,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, '/path/to/resource') + self.assertEquals(response.body, b'/path/to/resource') request = Request(self.getURL('path/to/resource')) return self.download_request(request, Spider('foo')).addCallback(_test) From f46a9d595dee801d0ea13d7cdaab8b8de952929f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 17:31:58 +0300 Subject: [PATCH 0067/3444] skip ftp tests on py3 for now --- tests/test_downloader_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ebf1d2f9c..b3c1c565f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -515,6 +515,8 @@ class FTPTestCase(unittest.TestCase): if twisted_version < (10, 2, 0): skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" + if six.PY3: + skip = "Twisted missing ftp support for PY3" def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory From 2aa6c92ffca50c8f6e5d057ac2808a99785eb88f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 17:52:50 +0300 Subject: [PATCH 0068/3444] py3 fixes in tests.mockserver --- tests/mockserver.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 3e4f8c0e5..1ab8e4b8d 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -134,12 +134,12 @@ class Echo(LeafResource): class Partial(LeafResource): def render_GET(self, request): - request.setHeader("Content-Length", "1024") + request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET def _delayedRender(self, request): - request.write("partial content\n") + request.write(b"partial content\n") request.finish() @@ -147,7 +147,7 @@ class Drop(Partial): def _delayedRender(self, request): abort = getarg(request, "abort", 0, type=int) - request.write("this connection will be dropped\n") + request.write(b"this connection will be dropped\n") tr = request.channel.transport try: if abort and hasattr(tr, 'abortConnection'): @@ -162,13 +162,13 @@ class Root(Resource): def __init__(self): Resource.__init__(self) - self.putChild("status", Status()) - self.putChild("follow", Follow()) - self.putChild("delay", Delay()) - self.putChild("partial", Partial()) - self.putChild("drop", Drop()) - self.putChild("raw", Raw()) - self.putChild("echo", Echo()) + self.putChild(b"status", Status()) + self.putChild(b"follow", Follow()) + self.putChild(b"delay", Delay()) + self.putChild(b"partial", Partial()) + self.putChild(b"drop", Drop()) + self.putChild(b"raw", Raw()) + self.putChild(b"echo", Echo()) if six.PY2 and twisted_version > (12, 3, 0): from twisted.web.test.test_webclient import PayloadResource @@ -181,7 +181,7 @@ class Root(Resource): return self def render(self, request): - return 'Scrapy mock HTTP server\n' + return b'Scrapy mock HTTP server\n' class MockServer(): From 81a90c3af65ce863c073e2d83a6b149a03e7d4cb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 18:47:06 +0300 Subject: [PATCH 0069/3444] unskip part of test_download_gzip_response on py3, file a twisted issue for the remaining part --- tests/mockserver.py | 6 +++--- tests/test_downloader_handlers.py | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 1ab8e4b8d..02bab0efd 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -170,12 +170,12 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) - if six.PY2 and twisted_version > (12, 3, 0): + if twisted_version > (12, 3, 0): from twisted.web.test.test_webclient import PayloadResource from twisted.web.server import GzipEncoderFactory from twisted.web.resource import EncodingResourceWrapper - self.putChild('payload', PayloadResource()) - self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"payload", PayloadResource()) + self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b3c1c565f..a8de28d4b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -294,27 +294,30 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): - if six.PY2 and twisted_version > (12, 3, 0): + if twisted_version > (12, 3, 0): crawler = get_crawler(SingleRequestSpider) - body = '1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') + if six.PY2: + # download_maxsize = 50 is enough for the gzipped response + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload only enabled for PY2") + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') else: - raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x") + raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") class UriResource(resource.Resource): From 99f1f2ad1dbff9ae2e97755b2d61e03ab2339a6d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 19:00:48 +0300 Subject: [PATCH 0070/3444] unskip tests and modules ported to py3 --- tests/py3-ignores.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 9e75ecf92..57e80f590 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,6 @@ tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloader_handlers.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py @@ -25,8 +24,6 @@ scrapy/xlib/tx/client.py scrapy/xlib/tx/_newclient.py scrapy/xlib/tx/__init__.py scrapy/core/downloader/handlers/s3.py -scrapy/core/downloader/handlers/http11.py -scrapy/core/downloader/handlers/http.py scrapy/core/downloader/handlers/ftp.py scrapy/pipelines/images.py scrapy/pipelines/files.py From 94ab7bee6c5630bcb36e7b758daf31e45d493613 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:26:01 +0300 Subject: [PATCH 0071/3444] py3: body to bytes in tests, unskip test file --- tests/py3-ignores.txt | 1 - tests/test_downloadermiddleware.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 57e80f590..0da1b6089 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -7,7 +7,6 @@ tests/test_crawl.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py tests/test_engine.py tests/test_mail.py diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 13f35b92a..fb51392b2 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -5,6 +5,7 @@ from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler +from scrapy.utils.python import to_bytes from tests import mock @@ -68,7 +69,7 @@ class DefaultsTest(ManagerTestCase): """ req = Request('http://example.com') - body = '

You are being redirected

' + body = b'

You are being redirected

' resp = Response(req.url, status=302, body=body, headers={ 'Content-Length': str(len(body)), 'Content-Type': 'text/html', @@ -78,12 +79,12 @@ class DefaultsTest(ManagerTestCase): ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), "Not redirected: {0!r}".format(ret)) - self.assertEqual(ret.url, resp.headers['Location'], + self.assertEqual(to_bytes(ret.url), resp.headers['Location'], "Not redirected to location header") def test_200_and_invalid_gzipped_body_must_fail(self): req = Request('http://example.com') - body = '

You are being redirected

' + body = b'

You are being redirected

' resp = Response(req.url, status=200, body=body, headers={ 'Content-Length': str(len(body)), 'Content-Type': 'text/html', From dbf6cc73d96f5015e14d4075e5a0d50db2a96453 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:46:56 +0300 Subject: [PATCH 0072/3444] py3: add leveldb to py33 test env, fix anydbm module name on py3 --- scrapy/settings/default_settings.py | 4 +++- tox.ini | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8435b0354..b151933b6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,6 +18,8 @@ import sys from importlib import import_module from os.path import join, abspath, dirname +import six + AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -163,7 +165,7 @@ HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] -HTTPCACHE_DBM_MODULE = 'anydbm' +HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False diff --git a/tox.ini b/tox.ini index eae7e8e47..874a22ee2 100644 --- a/tox.ini +++ b/tox.ini @@ -42,6 +42,7 @@ deps = -rrequirements-py3.txt # Extras Pillow + leveldb -rtests/requirements-py3.txt [testenv:py34] From e7ed1fd70df28d529b5f1df04bb850bb260dfc01 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:48:07 +0300 Subject: [PATCH 0073/3444] py3 compat in httpcache - headers are bytes --- scrapy/extensions/httpcache.py | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 2911dd6bc..80f615818 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -45,7 +45,7 @@ class RFC2616Policy(object): def _parse_cachecontrol(self, r): if r not in self._cc_parsed: - cch = r.headers.get('Cache-Control', '') + cch = r.headers.get(b'Cache-Control', b'') parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -58,7 +58,7 @@ class RFC2616Policy(object): return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if 'no-store' in cc: + if b'no-store' in cc: return False # Any other is eligible for caching return True @@ -69,7 +69,7 @@ class RFC2616Policy(object): # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) # obey directive "Cache-Control: no-store" - if 'no-store' in cc: + if b'no-store' in cc: return False # Never cache 304 (Not Modified) responses elif response.status == 304: @@ -78,14 +78,14 @@ class RFC2616Policy(object): elif self.always_store: return True # Any hint on response expiration is good - elif 'max-age' in cc or 'Expires' in response.headers: + elif b'max-age' in cc or b'Expires' in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set elif response.status in (300, 301, 308): return True # Other statuses without expiration requires at least one validator elif response.status in (200, 203, 401): - return 'Last-Modified' in response.headers or 'ETag' in response.headers + return b'Last-Modified' in response.headers or b'ETag' in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated @@ -95,7 +95,7 @@ class RFC2616Policy(object): def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) - if 'no-cache' in cc or 'no-cache' in ccreq: + if b'no-cache' in cc or b'no-cache' in ccreq: return False now = time() @@ -109,7 +109,7 @@ class RFC2616Policy(object): if currentage < freshnesslifetime: return True - if 'max-stale' in ccreq and 'must-revalidate' not in cc: + if b'max-stale' in ccreq and b'must-revalidate' not in cc: # From RFC2616: "Indicates that the client is willing to # accept a response that has exceeded its expiration time. # If max-stale is assigned a value, then the client is @@ -117,7 +117,7 @@ class RFC2616Policy(object): # expiration time by no more than the specified number of # seconds. If no value is assigned to max-stale, then the # client is willing to accept a stale response of any age." - staleage = ccreq['max-stale'] + staleage = ccreq[b'max-stale'] if staleage is None: return True @@ -136,22 +136,22 @@ class RFC2616Policy(object): # as long as the old response didn't specify must-revalidate. if response.status >= 500: cc = self._parse_cachecontrol(cachedresponse) - if 'must-revalidate' not in cc: + if b'must-revalidate' not in cc: return True # Use the cached response if the server says it hasn't changed. return response.status == 304 def _set_conditional_validators(self, request, cachedresponse): - if 'Last-Modified' in cachedresponse.headers: - request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] + if b'Last-Modified' in cachedresponse.headers: + request.headers[b'If-Modified-Since'] = cachedresponse.headers[b'Last-Modified'] - if 'ETag' in cachedresponse.headers: - request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + if b'ETag' in cachedresponse.headers: + request.headers[b'If-None-Match'] = cachedresponse.headers[b'ETag'] def _get_max_age(self, cc): try: - return max(0, int(cc['max-age'])) + return max(0, int(cc[b'max-age'])) except (KeyError, ValueError): return None @@ -164,18 +164,18 @@ class RFC2616Policy(object): return maxage # Parse date header or synthesize it if none exists - date = rfc1123_to_epoch(response.headers.get('Date')) or now + date = rfc1123_to_epoch(response.headers.get(b'Date')) or now # Try HTTP/1.0 Expires header - if 'Expires' in response.headers: - expires = rfc1123_to_epoch(response.headers['Expires']) + if b'Expires' in response.headers: + expires = rfc1123_to_epoch(response.headers[b'Expires']) # When parsing Expires header fails RFC 2616 section 14.21 says we # should treat this as an expiration time in the past. return max(0, expires - date) if expires else 0 # Fallback to heuristic using last-modified header # This is not in RFC but on Firefox caching implementation - lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) + lastmodified = rfc1123_to_epoch(response.headers.get(b'Last-Modified')) if lastmodified and lastmodified <= date: return (date - lastmodified) / 10 @@ -192,13 +192,13 @@ class RFC2616Policy(object): currentage = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server - date = rfc1123_to_epoch(response.headers.get('Date')) or now + date = rfc1123_to_epoch(response.headers.get(b'Date')) or now if now > date: currentage = now - date - if 'Age' in response.headers: + if b'Age' in response.headers: try: - age = int(response.headers['Age']) + age = int(response.headers[b'Age']) currentage = max(currentage, age) except ValueError: pass @@ -404,16 +404,16 @@ def parse_cachecontrol(header): http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 - >>> parse_cachecontrol('public, max-age=3600') == {'public': None, - ... 'max-age': '3600'} + >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None, + ... b'max-age': b'3600'} True - >>> parse_cachecontrol('') == {} + >>> parse_cachecontrol(b'') == {} True """ directives = {} - for directive in header.split(','): - key, sep, val = directive.strip().partition('=') + for directive in header.split(b','): + key, sep, val = directive.strip().partition(b'=') if key: directives[key.lower()] = val if sep else None return directives From ddc91dda270e44d80dbe08cc79d4249f82cde093 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 11:49:28 +0300 Subject: [PATCH 0074/3444] py3: fix _BaseTest in httpcache --- tests/test_downloadermiddleware_httpcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 53389ae3b..5a636cc53 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -31,7 +31,7 @@ class _BaseTest(unittest.TestCase): headers={'User-Agent': 'test'}) self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, - body='test body', + body=b'test body', status=202) self.crawler.stats.open_spider(self.spider) @@ -84,9 +84,9 @@ class _BaseTest(unittest.TestCase): def assertEqualRequestButWithCacheValidators(self, request1, request2): self.assertEqual(request1.url, request2.url) - assert not 'If-None-Match' in request1.headers - assert not 'If-Modified-Since' in request1.headers - assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since')) + assert not b'If-None-Match' in request1.headers + assert not b'If-Modified-Since' in request1.headers + assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) def test_dont_cache(self): From ea0471e33a769f98f853b5ba145b75a9227111a1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 12:22:41 +0300 Subject: [PATCH 0075/3444] py3: fix LeveldbCacheStorage - using bytes as keys and values in leveldb --- scrapy/extensions/httpcache.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 80f615818..02f9fcee6 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -12,6 +12,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes class DummyPolicy(object): @@ -305,7 +306,7 @@ class FilesystemCacheStorage(object): 'timestamp': time(), } with self._open(os.path.join(rpath, 'meta'), 'wb') as f: - f.write(repr(metadata)) + f.write(to_bytes(repr(metadata))) with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: pickle.dump(metadata, f, protocol=2) with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: @@ -373,14 +374,14 @@ class LeveldbCacheStorage(object): 'body': response.body, } batch = self._leveldb.WriteBatch() - batch.Put('%s_data' % key, pickle.dumps(data, protocol=2)) - batch.Put('%s_time' % key, str(time())) + batch.Put(key + b'_data', pickle.dumps(data, protocol=2)) + batch.Put(key + b'_time', to_bytes(str(time()))) self.db.Write(batch) def _read_data(self, spider, request): key = self._request_key(request) try: - ts = self.db.Get('%s_time' % key) + ts = self.db.Get(key + b'_time') except KeyError: return # not found or invalid entry @@ -388,14 +389,14 @@ class LeveldbCacheStorage(object): return # expired try: - data = self.db.Get('%s_data' % key) + data = self.db.Get(key + b'_data') except KeyError: return # invalid entry else: return pickle.loads(data) def _request_key(self, request): - return request_fingerprint(request) + return to_bytes(request_fingerprint(request)) From 87849780bcf77aa47ac42cc1c7b24e1185563c7d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 13:13:10 +0300 Subject: [PATCH 0076/3444] some py3 fixes for RFC2616Policy --- scrapy/extensions/httpcache.py | 3 ++- tests/test_downloadermiddleware_httpcache.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 02f9fcee6..91b3ef262 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -41,7 +41,8 @@ class RFC2616Policy(object): def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') + self.ignore_response_cache_controls = map( + to_bytes, settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')) self._cc_parsed = WeakKeyDictionary() def _parse_cachecontrol(self, r): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 5a636cc53..4e0c72304 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -291,7 +291,7 @@ class RFC2616PolicyTest(DefaultStorageTest): self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response # but it allows new response to be stored - res0b = res0.replace(body='foo') + res0b = res0.replace(body=b'foo') res4 = self._process_requestresponse(mw, req2, res0b) self.assertEqualResponse(res4, res0b) assert 'cached' not in res4.flags @@ -435,7 +435,7 @@ class RFC2616PolicyTest(DefaultStorageTest): assert 'cached' not in res1.flags # Same request but as cached response is stale a new response must # be returned - res0b = res0a.replace(body='bar') + res0b = res0a.replace(body=b'bar') res2 = self._process_requestresponse(mw, req0, res0b) self.assertEqualResponse(res2, res0b) assert 'cached' not in res2.flags From 131f4632472922234a7abde116eb566329a952a2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 11:17:52 +0100 Subject: [PATCH 0077/3444] Allow failures for Python 3.5 for now --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ae9c745ac..ac93e337d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs +matrix: + allow_failures: + - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox From 96fcf4cea41a067b4feb4f8adaa5b9ae1d5d38dd Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 13:27:28 +0300 Subject: [PATCH 0078/3444] add a check that byte url is not accepted in http.Response on py3 --- tests/test_http_response.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b49d46ea8..710a5b29d 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -17,6 +17,8 @@ class BaseResponseTest(unittest.TestCase): # Response requires url in the consturctor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) + if not six.PY2: + self.assertRaises(TypeError, self.response_class, b"http://example.com") # body can be str or None self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) From a4ca1668d894920e4a74c8f4204fa3ee53039a1d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 14:20:19 +0300 Subject: [PATCH 0079/3444] add https test for http10 handler (no luck with testing https with http11 so far) --- tests/mockserver.py | 12 ++++++++---- tests/test_downloader_handlers.py | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 02bab0efd..e7953c4d4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -199,14 +199,18 @@ class MockServer(): time.sleep(0.2) +def ssl_context_factory(): + return ssl.DefaultOpenSSLContextFactory( + os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + ) + + if __name__ == "__main__": root = Root() factory = Site(root) httpPort = reactor.listenTCP(8998, factory) - contextFactory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), - ) + contextFactory = ssl_context_factory() httpsPort = reactor.listenSSL(8999, factory, contextFactory) def print_listening(): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8de28d4b..84d1aa191 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -27,7 +27,7 @@ from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured -from tests.mockserver import MockServer +from tests.mockserver import MockServer, ssl_context_factory from tests.spiders import SingleRequestSpider class DummyDH(object): @@ -102,6 +102,7 @@ class FileTestCase(unittest.TestCase): class HttpTestCase(unittest.TestCase): + scheme = 'http' download_handler_cls = HTTPDownloadHandler def setUp(self): @@ -118,7 +119,12 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') + self.host = '127.0.0.1' + if self.scheme == 'https': + self.port = reactor.listenSSL( + 0, self.wrapper, ssl_context_factory(), interface=self.host) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port self.download_handler = self.download_handler_cls(Settings()) self.download_request = self.download_handler.download_request @@ -130,7 +136,7 @@ class HttpTestCase(unittest.TestCase): yield self.download_handler.close() def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) def test_download(self): request = Request(self.getURL('file')) @@ -213,6 +219,12 @@ class Http10TestCase(HttpTestCase): download_handler_cls = HTTP10DownloadHandler +class Https10TestCase(Http10TestCase): + scheme = 'https' + def test_timeout_download_from_spider(self): + raise unittest.SkipTest("test_timeout_download_from_spider skipped under https") + + class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler From 4398d95a02659c90df496a9156c7b3297cc42f50 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 14:54:12 +0300 Subject: [PATCH 0080/3444] skip this file on py3 again - it has one compression test, sould be done separately --- tests/py3-ignores.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 0da1b6089..57e80f590 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -7,6 +7,7 @@ tests/test_crawl.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py +tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py tests/test_engine.py tests/test_mail.py From 8330776c2193c3954cba2277525401de35672e00 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:16:12 +0300 Subject: [PATCH 0081/3444] fix error reporting in test: we can fail in process_request too, so result should always be defined --- tests/test_downloadermiddleware_httpcache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 4e0c72304..12b69860a 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -257,6 +257,7 @@ class RFC2616PolicyTest(DefaultStorageTest): policy_class = 'scrapy.extensions.httpcache.RFC2616Policy' def _process_requestresponse(self, mw, request, response): + result = None try: result = mw.process_request(request, self.spider) if result: From b0648271d69fc3e5b50c430ad50f1afe44acf0d4 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:17:30 +0300 Subject: [PATCH 0082/3444] py3 fix for rfc1123_to_epoch - "except Exception" was hiding bytes/str error --- scrapy/extensions/httpcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 91b3ef262..03ea88a10 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -12,7 +12,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode class DummyPolicy(object): @@ -423,6 +423,7 @@ def parse_cachecontrol(header): def rfc1123_to_epoch(date_str): try: + date_str = to_unicode(date_str, encoding='ascii') return mktime_tz(parsedate_tz(date_str)) except Exception: return None From 085fdd628314d6d3268c6ebbd6e9388fbbb2d37f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:40:45 +0300 Subject: [PATCH 0083/3444] py3 fix for ignoring cache controls - map is not a list --- scrapy/extensions/httpcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 03ea88a10..a871cc895 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -41,8 +41,8 @@ class RFC2616Policy(object): def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = map( - to_bytes, settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')) + self.ignore_response_cache_controls = [to_bytes(cc) for cc in + settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')] self._cc_parsed = WeakKeyDictionary() def _parse_cachecontrol(self, r): From 7d44c5dcea6c523b4cac19eefa5eb22966c5a90f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 15 Jan 2016 15:41:31 +0300 Subject: [PATCH 0084/3444] py3: unskip tests/test_downloadermiddleware_httpcache.py and scrapy/downloadermiddlewares/httpcache.py --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 57e80f590..2a9f06c8c 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,6 @@ tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware.py @@ -31,7 +30,6 @@ scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py scrapy/downloadermiddlewares/retry.py -scrapy/downloadermiddlewares/httpcache.py scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py From ee4fadc00724f02f9098625ac4d72fb29eac4dcf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 14:57:15 +0100 Subject: [PATCH 0085/3444] Use .read1() if available when using GzipFile --- .travis.yml | 3 --- scrapy/utils/gz.py | 20 ++++++++++++++------ tests/test_squeues.py | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac93e337d..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,6 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs -matrix: - allow_failures: - - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fa4bba57..df1d29698 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,30 +4,38 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import UnsupportedOperation from gzip import GzipFile +class ReadOneGzipFile(GzipFile): + def readone(self, size=-1): + try: + return self.read1(size) + except UnsupportedOperation: + return self.read(size) def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = GzipFile(fileobj=BytesIO(data)) + f = ReadOneGzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.read(8196) + chunk = f.readone(8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf # contains the whole page content - if output or f.extrabuf: - output += f.extrabuf - break + if output or getattr(f, 'extrabuf', None): + try: + output += f.extrabuf + finally: + break else: raise return output diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..232f539e6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From 73d78ec99fc622fe9a552c78193c0b4bf652533b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 17:59:20 +0100 Subject: [PATCH 0086/3444] Add Code of Conduct Version 1.3.0 from http://contributor-covenant.org/ Closes #1645 --- CODE_OF_CONDUCT.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 6 ++++++ 2 files changed, 56 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..95b4a7e3c --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,50 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer at opensource@scrapinghub.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/README.rst b/README.rst index 6cbed75ee..8a7d2c71d 100644 --- a/README.rst +++ b/README.rst @@ -73,6 +73,12 @@ See http://scrapy.org/community/ Contributing ============ +Please note that this project is released with a Contributor Code of Conduct +(see CODE_OF_CONDUCT.md). + +By participating in this project you agree to abide by its terms. +Please report unacceptable behavior to opensource@scrapinghub.com. + See http://doc.scrapy.org/en/master/contributing.html Companies using Scrapy From bb38400db560a4d814ea6c12364404d6daa75da5 Mon Sep 17 00:00:00 2001 From: Ralph Gutkowski Date: Fri, 15 Jan 2016 19:00:58 +0100 Subject: [PATCH 0087/3444] Update Stats Collection documentation `pages_crawled` value doesn't exist. Replace it with `downloader/response_count` in order to avoid confusion. --- docs/topics/stats.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0837610d0..290fc065c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -47,7 +47,7 @@ Set stat value:: Increment stat value:: - stats.inc_value('pages_crawled') + stats.inc_value('downloader/response_count') Set stat value only if greater than previous:: @@ -59,13 +59,13 @@ Set stat value only if lower than previous:: Get stat value:: - >>> stats.get_value('pages_crawled') + >>> stats.get_value('downloader/response_count') 8 Get all stats:: >>> stats.get_stats() - {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 79147a61a797f81fee44b000eb5ac9591593ff88 Mon Sep 17 00:00:00 2001 From: Ralph Gutkowski Date: Fri, 15 Jan 2016 19:25:56 +0100 Subject: [PATCH 0088/3444] Update stats.rst --- docs/topics/stats.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 290fc065c..dd0c6216b 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -47,7 +47,7 @@ Set stat value:: Increment stat value:: - stats.inc_value('downloader/response_count') + stats.inc_value('custom_count') Set stat value only if greater than previous:: @@ -59,13 +59,13 @@ Set stat value only if lower than previous:: Get stat value:: - >>> stats.get_value('downloader/response_count') - 8 + >>> stats.get_value('custom_count') + 1 Get all stats:: >>> stats.get_stats() - {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 4e44766653c294c54a3bb960b3734ee70bc663a4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 19:51:21 +0100 Subject: [PATCH 0089/3444] Use "unicode" string for lxml.etree.tostring() serialization --- scrapy/utils/iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 69c7f2c23..b0688791e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding=six.text_type) + nodetext = etree.tostring(node, encoding='unicode') node.clear() xs = Selector(text=nodetext, type='xml') if namespace: From cd735e377c2a26bc8ebb925bf6031429549faadf Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 18 Jan 2016 07:45:36 +0100 Subject: [PATCH 0090/3444] Simplify if statement --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index b0ac1badd..8b4faf8fc 100644 --- a/conftest.py +++ b/conftest.py @@ -34,7 +34,7 @@ if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, if six.PY3: for line in open('tests/py3-ignores.txt'): file_path = line.strip() - if len(file_path) > 0 and file_path[0] != '#': + if file_path and file_path[0] != '#': collect_ignore.append(file_path) From d3c2b0cf7ec185d18464e85974ee7499acecad79 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 10:06:14 +0300 Subject: [PATCH 0091/3444] py3 webclient: I was mistaken about unicode body, revert conversion to bytes and fix HEAD response --- scrapy/core/downloader/webclient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index d2cdd6f98..bbbb98f60 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -66,7 +66,7 @@ class ScrapyHTTPPageGetter(HTTPClient): def handleResponse(self, response): if self.factory.method.upper() == b'HEAD': - self.factory.page('') + self.factory.page(b'') elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) else: @@ -131,7 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - body = to_bytes(body) return respcls(url=self._url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): From 673df5e4161337136154eeccfc4e327d053dbd12 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 11:44:49 +0300 Subject: [PATCH 0092/3444] add webclient test - check that non-standart body encoding matches Content-Encoding header --- tests/test_webclient.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 412e10c89..3ee6c24c2 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -7,7 +7,7 @@ import six from six.moves.urllib.parse import urlparse from twisted.trial import unittest -from twisted.web import server, static, error, util +from twisted.web import server, static, util, resource from twisted.internet import reactor, defer from twisted.test.proto_helpers import StringTransport from twisted.python.filepath import FilePath @@ -18,14 +18,14 @@ from scrapy.http import Request, Headers from scrapy.utils.python import to_bytes, to_unicode -def getPage(url, contextFactory=None, *args, **kwargs): +def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" def _clientfactory(url, *args, **kwargs): url = to_unicode(url) timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory( Request(url, *args, **kwargs), timeout=timeout) - f.deferred.addCallback(lambda r: r.body) + f.deferred.addCallback(r_transform or (lambda r: r.body)) return f from twisted.web.client import _makeGetterFactory @@ -213,6 +213,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ ErrorResource, NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource + +class EncodingResource(resource.Resource): + out_encoding = 'cp1251' + + def render(self, request): + body = to_unicode(request.content.read()) + request.setHeader(b'content-encoding', self.out_encoding) + return body.encode(self.out_encoding) + + class WebClientTestCase(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -229,6 +239,7 @@ class WebClientTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"encoding", EncodingResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = self._listen(self.wrapper) @@ -338,3 +349,17 @@ class WebClientTestCase(unittest.TestCase): b'\n\n \n \n' b' \n \n ' b'click here\n \n\n') + + def test_Encoding(self): + """ Test that non-standart body encoding matches + Content-Encoding header """ + body = b'\xd0\x81\xd1\x8e\xd0\xaf' + return getPage( + self.getURL('encoding'), body=body, r_transform=lambda r: r)\ + .addCallback(self._check_Encoding, body) + + def _check_Encoding(self, response, original_body): + content_encoding = to_unicode(response.headers[b'Content-Encoding']) + self.assertEquals(content_encoding, EncodingResource.out_encoding) + self.assertEquals( + response.body.decode(content_encoding), to_unicode(original_body)) From 6b905a9aecb0e0e22339353a983257f3707d98b8 Mon Sep 17 00:00:00 2001 From: palego Date: Sat, 16 Jan 2016 14:23:58 +0100 Subject: [PATCH 0093/3444] split-up the assertIn to deal with OS X intricacies (directories prefixed with /private) --- tests/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 8edccd4bd..aa1b7cc7a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -90,8 +90,8 @@ class StartprojectTemplatesTest(ProjectTest): args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] p = self.proc('startproject', self.project_name, *args) out = to_native_str(retry_on_eintr(p.stdout.read)) - self.assertIn("New Scrapy project %r, using template directory %r, created in:" % \ - (self.project_name, join(self.tmpl, 'project')), out) + self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) + self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) From f2e2ff5e1f2eb1d11e18be248e27bef7fc9b4ef7 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:20:01 +0300 Subject: [PATCH 0094/3444] py3: webclient assumes that urls come from Request.url and are ascii-only --- scrapy/core/downloader/webclient.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index bbbb98f60..9bcc51943 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): + # Assume parsed is urlparse-d from Request.url, + # which was passed via safe_url_string and is ascii-only. + b = lambda s: to_bytes(s, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - path = to_bytes(path) - host = to_bytes(parsed.hostname) + path = b(path) + host = b(parsed.hostname) port = parsed.port - scheme = to_bytes(parsed.scheme, encoding='ascii') - netloc = to_bytes(parsed.netloc) + scheme = b(parsed.scheme) + netloc = b(parsed.netloc) if port is None: port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path def _parse(url): + """ Return tuple of (scheme, netloc, host, port, path), + all in bytes except for port which is int. + Assume url is from Request.url, which was passed via safe_url_string + and is ascii-only. + """ url = url.strip() parsed = urlparse(url) return _parsed_url_args(parsed) @@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url) + self.url = to_bytes(self._url, encoding='ascii') self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers) From 04f69fd18406a9361f3c13df32b5f6ea295ecdbf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:37:46 +0300 Subject: [PATCH 0095/3444] add https 1.1 downloader test - localhost is a valid DNS-ID --- tests/test_downloader_handlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 84d1aa191..80eed86f2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -119,7 +119,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = '127.0.0.1' + self.host = 'localhost' if self.scheme == 'https': self.port = reactor.listenSSL( 0, self.wrapper, ssl_context_factory(), interface=self.host) @@ -273,6 +273,10 @@ class Http11TestCase(HttpTestCase): return d +class Https11TestCase(Http11TestCase): + scheme = 'https' + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" if twisted_version < (11, 1, 0): From 98c060d0b2cc76934e16abc03a033f21850fd565 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:42:21 +0300 Subject: [PATCH 0096/3444] py3: fix http 1.1 test with https, and use self.host everywhere --- tests/test_downloader_handlers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 80eed86f2..999fa4c0a 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -168,6 +168,9 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider(self): + if self.scheme == 'https': + raise unittest.SkipTest( + 'test_timeout_download_from_spider skipped under https') spider = Spider('foo') meta = {'download_timeout': 0.2} # client connects but no data is received @@ -181,7 +184,8 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno)) + self.assertEquals( + response.body, to_bytes('%s:%d' % (self.host, self.portno))) self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) @@ -221,8 +225,6 @@ class Http10TestCase(HttpTestCase): class Https10TestCase(Http10TestCase): scheme = 'https' - def test_timeout_download_from_spider(self): - raise unittest.SkipTest("test_timeout_download_from_spider skipped under https") class Http11TestCase(HttpTestCase): From 0f527849f2e8eddaf5d756b061699f2eca522a18 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 14:44:04 +0300 Subject: [PATCH 0097/3444] https proxy tunneling - add a test (not perfect, but covers all impl) and fix for py3 --- scrapy/core/downloader/handlers/http11.py | 14 +++++++++----- tests/test_downloader_handlers.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index d81093a9f..729b80b05 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -78,7 +78,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile('HTTP/1\.. 200') + _responseMatcher = re.compile(b'HTTP/1\.. 200') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): @@ -92,11 +92,15 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = 'CONNECT %s:%s HTTP/1.1\r\n' % (self._tunneledHost, - self._tunneledPort) + tunnelReq = ( + b'CONNECT ' + + to_bytes(self._tunneledHost, encoding='ascii') + b':' + + to_bytes(str(self._tunneledPort)) + + b' HTTP/1.1\r\n') if self._proxyAuthHeader: - tunnelReq += 'Proxy-Authorization: %s\r\n' % self._proxyAuthHeader - tunnelReq += '\r\n' + tunnelReq += \ + b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n' + tunnelReq += b'\r\n' protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived protocol.dataReceived = self.processProxyResponse diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 999fa4c0a..2d6c05741 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -388,6 +388,16 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('https://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + http_proxy = self.getURL('') + domain = 'https://no-such-domain.nosuch' + request = Request( + domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) + d = self.download_request(request, Spider('foo')) + timeout = yield self.assertFailure(d, error.TimeoutError) + self.assertIn(domain, timeout.osError) + def test_download_without_proxy(self): def _test(response): self.assertEquals(response.status, 200) From 7af64e8fd2a90102d85ff0f453cd6ba1dde71caa Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:00:43 +0300 Subject: [PATCH 0098/3444] py3: remove extra encoding/decoding of url: pass it as bytes only when required --- scrapy/core/downloader/handlers/http11.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 729b80b05..82cf507f7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -206,7 +206,7 @@ class ScrapyAgent(object): agent = self._get_agent(request, timeout) # request details - url = to_bytes(urldefrag(request.url)[0]) + url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): @@ -214,7 +214,8 @@ class ScrapyAgent(object): bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d = agent.request(method, url, headers, bodyproducer) + d = agent.request( + method, to_bytes(url, encoding='ascii'), headers, bodyproducer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed @@ -267,10 +268,8 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) - url = to_unicode(url) respcls = responsetypes.from_args(headers=headers, url=url) - return respcls( - url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, flags=flags) @implementer(IBodyProducer) From b940606b7e3dacbf5d639965c54ccbb87a214c95 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:06:15 +0300 Subject: [PATCH 0099/3444] this is a test for TunnelingTCP4ClientEndpoint - move into Http11ProxyTestCase --- tests/test_downloader_handlers.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2d6c05741..59320597e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -388,16 +388,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('https://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): - http_proxy = self.getURL('') - domain = 'https://no-such-domain.nosuch' - request = Request( - domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) - d = self.download_request(request, Spider('foo')) - timeout = yield self.assertFailure(d, error.TimeoutError) - self.assertIn(domain, timeout.osError) - def test_download_without_proxy(self): def _test(response): self.assertEquals(response.status, 200) @@ -422,6 +412,17 @@ class Http11ProxyTestCase(HttpProxyTestCase): if twisted_version < (11, 1, 0): skip = 'HTTP1.1 not supported in twisted < 11.1.0' + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + """ Test TunnelingTCP4ClientEndpoint """ + http_proxy = self.getURL('') + domain = 'https://no-such-domain.nosuch' + request = Request( + domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) + d = self.download_request(request, Spider('foo')) + timeout = yield self.assertFailure(d, error.TimeoutError) + self.assertIn(domain, timeout.osError) + class HttpDownloadHandlerMock(object): def __init__(self, settings): From a2efd389b09984de15ee0653d169c0bbbfd61a05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:09:54 +0300 Subject: [PATCH 0100/3444] clarify: rename r_transform to response_transform --- tests/test_webclient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3ee6c24c2..dbe659d5c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,14 +18,14 @@ from scrapy.http import Request, Headers from scrapy.utils.python import to_bytes, to_unicode -def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs): +def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" def _clientfactory(url, *args, **kwargs): url = to_unicode(url) timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory( Request(url, *args, **kwargs), timeout=timeout) - f.deferred.addCallback(r_transform or (lambda r: r.body)) + f.deferred.addCallback(response_transform or (lambda r: r.body)) return f from twisted.web.client import _makeGetterFactory @@ -355,7 +355,7 @@ class WebClientTestCase(unittest.TestCase): Content-Encoding header """ body = b'\xd0\x81\xd1\x8e\xd0\xaf' return getPage( - self.getURL('encoding'), body=body, r_transform=lambda r: r)\ + self.getURL('encoding'), body=body, response_transform=lambda r: r)\ .addCallback(self._check_Encoding, body) def _check_Encoding(self, response, original_body): From 494643458270311341c509f5476c61737aa27a70 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 15:23:01 +0300 Subject: [PATCH 0101/3444] revert most changes to this test, and clarify - it is valid only on py2, because urls are strictly unicode on py3 --- tests/test_webclient.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index dbe659d5c..9b5beda4c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -78,16 +78,17 @@ class ParseUrlTestCase(unittest.TestCase): elements of its return tuple, even when passed an URL which has previously been passed to L{urlparse} as a C{unicode} string. """ - goodInput = u'http://example.com/path' - badInput = goodInput.encode('ascii') - if six.PY2: - goodInput, badInput = badInput, goodInput - urlparse(badInput) + if not six.PY2: + raise unittest.SkipTest( + "Applies only to Py2, as urls can be ONLY unicode on Py3") + badInput = u'http://example.com/path' + goodInput = badInput.encode('ascii') + self._parse(badInput) # cache badInput in urlparse_cached scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, bytes)) - self.assertTrue(isinstance(netloc, bytes)) - self.assertTrue(isinstance(host, bytes)) - self.assertTrue(isinstance(path, bytes)) + self.assertTrue(isinstance(scheme, str)) + self.assertTrue(isinstance(netloc, str)) + self.assertTrue(isinstance(host, str)) + self.assertTrue(isinstance(path, str)) self.assertTrue(isinstance(port, int)) From 0b9336418ef40ca95052ebbaa02f12953e165115 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 16:43:58 +0300 Subject: [PATCH 0102/3444] py3: port compression downloader middleware and tests --- .../downloadermiddlewares/httpcompression.py | 8 +++---- tests/py3-ignores.txt | 1 - ...st_downloadermiddleware_httpcompression.py | 24 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 719507396..7ab304c17 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -9,13 +9,13 @@ from scrapy.exceptions import NotConfigured class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured return cls() - + def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', 'gzip,deflate') @@ -39,10 +39,10 @@ class HttpCompressionMiddleware(object): return response def _decode(self, body, encoding): - if encoding == 'gzip' or encoding == 'x-gzip': + if encoding == b'gzip' or encoding == b'x-gzip': body = gunzip(body) - if encoding == 'deflate': + if encoding == b'deflate': try: body = zlib.decompress(body) except zlib.error: diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 2a9f06c8c..dbf63f0f5 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,6 @@ tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a18994ef3..2e6e47fef 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -50,46 +50,46 @@ class HttpCompressionTest(TestCase): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual(request.headers.get('Accept-Encoding'), 'gzip,deflate') + self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate') def test_process_response_gzip(self): response = self._getresponse('gzip') request = response.request - self.assertEqual(response.headers['Content-Encoding'], 'gzip') + self.assertEqual(response.headers['Content-Encoding'], b'gzip') newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response - assert newresponse.body.startswith(' Date: Mon, 18 Jan 2016 16:45:22 +0300 Subject: [PATCH 0103/3444] common test_downloadermiddleware.py also passes now due to fixes in compression middleware --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index dbf63f0f5..1f7d85ef7 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -5,7 +5,6 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware.py tests/test_downloadermiddleware_retry.py tests/test_engine.py tests/test_mail.py From bcbad2905d3ff375e85ede723d13560d765ee729 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Jan 2016 15:22:29 +0100 Subject: [PATCH 0104/3444] Stick with ValueError for queue/serialization exception tests --- tests/test_squeues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 232f539e6..48871ceeb 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) + self.assertRaises(ValueError, q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) + self.assertRaises(ValueError, q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From 120fb4adeb957a348f01bf60e821b2dcab2f9de1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 19:00:40 +0300 Subject: [PATCH 0105/3444] revert bogus change --- scrapy/http/response/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 59ef15682..09c4e725a 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -35,7 +35,7 @@ class Response(object_ref): return self._url def _set_url(self, url): - if isinstance(url, six.string_types): + if isinstance(url, str): self._url = url else: raise TypeError('%s url must be str, got %s:' % (type(self).__name__, From 7fdd3225b293d4951ba079e1141683e7fc55f905 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 19:09:09 +0300 Subject: [PATCH 0106/3444] fix test skipping logic - this is (temporary) py2-only part --- tests/test_downloader_handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 59320597e..e6d219168 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -322,18 +322,18 @@ class Http11MockServerTestCase(unittest.TestCase): # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') - yield crawler.crawl(seed=request) - if six.PY2: + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') + yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') + else: + raise unittest.SkipTest("xpayload only enabled for PY2") else: raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") From de98d8d00658181ed46a96835e2ab95f2f6cd457 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 19:27:31 +0300 Subject: [PATCH 0107/3444] move comment about test skipped on py3 into a proper place --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e6d219168..c936b72ed 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -327,12 +327,12 @@ class Http11MockServerTestCase(unittest.TestCase): request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response - # See issue https://twistedmatrix.com/trac/ticket/8175 failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') else: + # See issue https://twistedmatrix.com/trac/ticket/8175 raise unittest.SkipTest("xpayload only enabled for PY2") else: raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") From fd99ef86dfca50dbd36b2c1a022cf30a0720dbea Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Jan 2016 17:57:55 +0100 Subject: [PATCH 0108/3444] Test for AttributeError when pickling objects (Python>=3.5) Same "fix" as in e.g. https://github.com/joblib/joblib/pull/246 --- scrapy/squeues.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6e2a60fd2..21520f454 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,7 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - except pickle.PicklingError as e: + # Python>=3.5 raises AttributeError here while + # Python<=3.4 raises pickle.PicklingError + except (pickle.PicklingError, AttributeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ From ff235fa19ad30f34be7ff435497e19d8e97a8970 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 19 Jan 2016 00:00:31 +0500 Subject: [PATCH 0109/3444] Remove --lsprof command-line option. Fixes GH-1531 --- scrapy/cmdline.py | 9 +-- scrapy/commands/__init__.py | 2 - scrapy/xlib/lsprofcalltree.py | 120 ---------------------------------- 3 files changed, 1 insertion(+), 130 deletions(-) delete mode 100644 scrapy/xlib/lsprofcalltree.py diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 35050c13d..cb7bbd64d 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -7,7 +7,6 @@ import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.xlib import lsprofcalltree from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules @@ -144,7 +143,7 @@ def execute(argv=None, settings=None): sys.exit(cmd.exitcode) def _run_command(cmd, args, opts): - if opts.profile or opts.lsprof: + if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) @@ -152,17 +151,11 @@ def _run_command(cmd, args, opts): def _run_command_profiled(cmd, args, opts): if opts.profile: sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) - if opts.lsprof: - sys.stderr.write("scrapy: writing lsprof stats to %r\n" % opts.lsprof) loc = locals() p = cProfile.Profile() p.runctx('cmd.run(args, opts)', globals(), loc) if opts.profile: p.dump_stats(opts.profile) - k = lsprofcalltree.KCacheGrind(p) - if opts.lsprof: - with open(opts.lsprof, 'w') as f: - k.output(f) if __name__ == '__main__': execute() diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9ac013098..43b420821 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -65,8 +65,6 @@ class ScrapyCommand(object): help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, help="write python cProfile stats to FILE") - group.add_option("--lsprof", metavar="FILE", default=None, - help="write lsprof profiling stats to FILE") group.add_option("--pidfile", metavar="FILE", help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", diff --git a/scrapy/xlib/lsprofcalltree.py b/scrapy/xlib/lsprofcalltree.py deleted file mode 100644 index a604016cc..000000000 --- a/scrapy/xlib/lsprofcalltree.py +++ /dev/null @@ -1,120 +0,0 @@ -# lsprofcalltree.py: lsprof output which is readable by kcachegrind -# David Allouche -# Jp Calderone & Itamar Shtull-Trauring -# Johan Dahlin - -from __future__ import print_function -import optparse -import os -import sys - -try: - import cProfile -except ImportError: - raise SystemExit("This script requires cProfile from Python 2.5") - -def label(code): - if isinstance(code, str): - return ('~', 0, code) # built-in functions ('~' sorts at the end) - else: - return '%s %s:%d' % (code.co_name, - code.co_filename, - code.co_firstlineno) - -class KCacheGrind(object): - def __init__(self, profiler): - self.data = profiler.getstats() - self.out_file = None - - def output(self, out_file): - self.out_file = out_file - print('events: Ticks', file=out_file) - self._print_summary() - for entry in self.data: - self._entry(entry) - - def _print_summary(self): - max_cost = 0 - for entry in self.data: - totaltime = int(entry.totaltime * 1000) - max_cost = max(max_cost, totaltime) - print('summary: %d' % (max_cost,), file=self.out_file) - - def _entry(self, entry): - out_file = self.out_file - - code = entry.code - #print >> out_file, 'ob=%s' % (code.co_filename,) - if isinstance(code, str): - print('fi=~', file=out_file) - else: - print('fi=%s' % (code.co_filename,), file=out_file) - print('fn=%s' % (label(code),), file=out_file) - - inlinetime = int(entry.inlinetime * 1000) - if isinstance(code, str): - print('0 ', inlinetime, file=out_file) - else: - print('%d %d' % (code.co_firstlineno, inlinetime), file=out_file) - - # recursive calls are counted in entry.calls - if entry.calls: - calls = entry.calls - else: - calls = [] - - if isinstance(code, str): - lineno = 0 - else: - lineno = code.co_firstlineno - - for subentry in calls: - self._subentry(lineno, subentry) - print(file=out_file) - - def _subentry(self, lineno, subentry): - out_file = self.out_file - code = subentry.code - #print >> out_file, 'cob=%s' % (code.co_filename,) - print('cfn=%s' % (label(code),), file=out_file) - if isinstance(code, str): - print('cfi=~', file=out_file) - print('calls=%d 0' % (subentry.callcount,), file=out_file) - else: - print('cfi=%s' % (code.co_filename,), file=out_file) - print('calls=%d %d' % ( - subentry.callcount, code.co_firstlineno), file=out_file) - - totaltime = int(subentry.totaltime * 1000) - print('%d %d' % (lineno, totaltime), file=out_file) - -def main(args): - usage = "%s [-o output_file_path] scriptfile [arg] ..." - parser = optparse.OptionParser(usage=usage % sys.argv[0]) - parser.allow_interspersed_args = False - parser.add_option('-o', '--outfile', dest="outfile", - help="Save stats to ", default=None) - - if not sys.argv[1:]: - parser.print_usage() - sys.exit(2) - - options, args = parser.parse_args() - - if not options.outfile: - options.outfile = '%s.log' % os.path.basename(args[0]) - - sys.argv[:] = args - - prof = cProfile.Profile() - try: - try: - prof = prof.run('execfile(%r)' % (sys.argv[0],)) - except SystemExit: - pass - finally: - kg = KCacheGrind(prof) - kg.output(file(options.outfile, 'w')) - -if __name__ == '__main__': - sys.exit(main(sys.argv)) From 5ac54ed3391b027cccb140731f82f1f026ea2f7c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 17:25:50 +0300 Subject: [PATCH 0110/3444] raise minimal twisted version for py3 --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index a9a2e3be0..065095101 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 15.1.0 +Twisted >= 15.5.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From bb50c0be2fd7e737f2e2bd772f333e68cd02db06 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 17:30:59 +0300 Subject: [PATCH 0111/3444] remove unused import --- scrapy/http/response/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 09c4e725a..983154001 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,6 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -import six from six.moves.urllib.parse import urljoin from scrapy.http.headers import Headers From 9c3117a914c27b3acbb2b0300d8938d6e3b49b9e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 17:32:53 +0300 Subject: [PATCH 0112/3444] more pythonic check of noconnect in proxy params --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 82cf507f7..bda72f5e6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -186,7 +186,7 @@ class ScrapyAgent(object): _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) - omitConnectTunnel = proxyParams.find(b'noconnect') >= 0 + omitConnectTunnel = b'noconnect' in proxyParams if scheme == b'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, request.headers.get(b'Proxy-Authorization', None)) From 4c8417284062af7fe1c0107a0fcfdd377424dc50 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:24:58 +0300 Subject: [PATCH 0113/3444] move leveldb to tests/requirements-py3.txt --- tests/requirements-py3.txt | 1 + tox.ini | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index a709a734e..5cf786a89 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -3,6 +3,7 @@ pytest-twisted pytest-cov testfixtures jmespath +leveldb # optional for shell wrapper tests bpython ipython diff --git a/tox.ini b/tox.ini index 874a22ee2..eae7e8e47 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,6 @@ deps = -rrequirements-py3.txt # Extras Pillow - leveldb -rtests/requirements-py3.txt [testenv:py34] From 1f2233837a4219d02be73dc0836dfc885d47fffb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 16:58:24 +0100 Subject: [PATCH 0114/3444] Use if Py2/Py3 function instead of custom GzipFile class method --- scrapy/utils/gz.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index df1d29698..3e6596b0b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,24 +7,35 @@ except ImportError: from io import UnsupportedOperation from gzip import GzipFile -class ReadOneGzipFile(GzipFile): - def readone(self, size=-1): - try: - return self.read1(size) - except UnsupportedOperation: - return self.read(size) +import six + + +# - Python>=3.5 GzipFile's read() has issues returning leftover +# uncompressed data when input is corrupted +# (regression or bug-fix compared to Python 3.4) +# - read1(), which fetches data before raising EOFError on next call +# works here but is only available from Python>=3.3 +# - scrapy does not support Python 3.2 +# - Python 2.7 GzipFile works fine with standard read() + extrabuf +if six.PY3: + def read1(gzf, size=-1): + return gzf.read1(size) +else: + def read1(gzf, size=-1): + return gzf.read(size) + def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = ReadOneGzipFile(fileobj=BytesIO(data)) + f = GzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.readone(8196) + chunk = read1(f, 8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise From 2b5245839ceaaa256c7edccee351ea79129fa3e4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 17:04:57 +0100 Subject: [PATCH 0115/3444] Remove unused import statement --- scrapy/utils/gz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 3e6596b0b..d69fb598d 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,7 +4,6 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO -from io import UnsupportedOperation from gzip import GzipFile import six From e15f361b0580eb85a13f79b2c96417d17b9a0de6 Mon Sep 17 00:00:00 2001 From: carlosp420 Date: Tue, 19 Jan 2016 11:12:43 -0500 Subject: [PATCH 0116/3444] fixed typo You -> you --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 79f0b3b53..28ed7c064 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -50,7 +50,7 @@ many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU you crawler will have available. A good starting point is ``100``, but the best way to find out is by doing some trials and identifying at what concurrency your Scrapy process gets CPU -bounded. For optimum performance, You should pick a concurrency where CPU usage +bounded. For optimum performance, you should pick a concurrency where CPU usage is at 80-90%. To increase the global concurrency use:: From 176610f91068cb9663be184094e450de0aac77bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 19 Jan 2016 15:34:26 -0300 Subject: [PATCH 0117/3444] optional_features has been removed --- tests/test_downloader_handlers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a474b75d2..1eb6192ce 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -431,8 +431,12 @@ class HttpDownloadHandlerMock(object): def download_request(self, request, spider): return request + class S3AnonTestCase(unittest.TestCase): - skip = 'boto' not in optional_features and 'missing boto library' + try: + import boto + except ImportError: + skip = 'missing boto library' def setUp(self): self.s3reqh = S3DownloadHandler(Settings(), @@ -448,14 +452,13 @@ class S3AnonTestCase(unittest.TestCase): self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) self.assertEqual(self.s3reqh.conn.anon, True) + class S3TestCase(unittest.TestCase): download_handler_cls = S3DownloadHandler try: - # can't instance without settings, but ignore that - download_handler_cls({}) - except NotConfigured: + import boto + except ImportError: skip = 'missing boto library' - except KeyError: pass # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf From f0cf5463c85bd38243ac1539487d7118003a790e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 20 Jan 2016 12:23:48 +0300 Subject: [PATCH 0118/3444] cleanup http11 tunneling connection after #1678 - construct string and then convert to bytes --- scrapy/core/downloader/handlers/http11.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a4d5a28c8..ad3285a32 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,11 +92,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = ( - b'CONNECT ' + - to_bytes(self._tunneledHost, encoding='ascii') + b':' + - to_bytes(str(self._tunneledPort)) + - b' HTTP/1.1\r\n') + tunnelReq = to_bytes( + 'CONNECT %s:%s HTTP/1.1\r\n' % ( + self._tunneledHost, self._tunneledPort), encoding='ascii') if self._proxyAuthHeader: tunnelReq += \ b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n' From 29ff84a7920dd27a7b723f21b68ccc6e2076c08a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 20 Jan 2016 12:03:38 +0100 Subject: [PATCH 0119/3444] Invert PY2/PY3 test for conditional read1() definition --- scrapy/utils/gz.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index d69fb598d..d035f9fdf 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -16,12 +16,12 @@ import six # works here but is only available from Python>=3.3 # - scrapy does not support Python 3.2 # - Python 2.7 GzipFile works fine with standard read() + extrabuf -if six.PY3: - def read1(gzf, size=-1): - return gzf.read1(size) -else: +if six.PY2: def read1(gzf, size=-1): return gzf.read(size) +else: + def read1(gzf, size=-1): + return gzf.read1(size) def gunzip(data): From a32b59ac7507b9b909edfde4cfa9a9effae27641 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 18:54:32 +0300 Subject: [PATCH 0120/3444] py3: fix EngineTest.test_crawler --- tests/test_engine.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index dad921a60..df68e5281 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -55,11 +55,12 @@ class TestSpider(Spider): def parse_item(self, response): item = self.item_cls() - m = self.name_re.search(response.body) + body = response.body_as_unicode() + m = self.name_re.search(body) if m: item['name'] = m.group(1) item['url'] = response.url - m = self.price_re.search(response.body) + m = self.price_re.search(body) if m: item['price'] = m.group(1) return item @@ -77,8 +78,8 @@ class DictItemsSpider(TestSpider): def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: From 0b08b4bfcfb629aefd4f83cf85e64c74bc1077c3 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 10:07:00 +0300 Subject: [PATCH 0121/3444] fix engine tests - this callback (spider_closed_callback) should accept one argument, but the error was hidden on py2 --- tests/test_engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index df68e5281..9f2c02bff 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -238,12 +238,12 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_close_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) self.assertEqual(len(e.open_spiders), 1) yield e.close() @@ -251,7 +251,7 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_engine_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() self.assertTrue(e.running) From c1db60188aaf05d3a7f055d9b736669e679aed52 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 10:07:12 +0300 Subject: [PATCH 0122/3444] py3: unskip passing test_engine --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f7d85ef7..3e147e9e9 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -6,7 +6,6 @@ tests/test_linkextractors_deprecated.py tests/test_crawl.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware_retry.py -tests/test_engine.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py From 47d3c63338c4550341cc7518bcb48d0fc606b67d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:28:22 +0300 Subject: [PATCH 0123/3444] py3: port fetch and shell commands, and review + enable already passing test_closespider.py and tests/test_utils_template.py --- scrapy/commands/fetch.py | 6 ++++-- scrapy/utils/testsite.py | 12 ++++++------ tests/py3-ignores.txt | 4 ---- tests/test_command_fetch.py | 8 ++++---- tests/test_command_shell.py | 14 +++++++------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index e61eedf50..3888da210 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.python import to_unicode class Command(ScrapyCommand): @@ -30,7 +31,8 @@ class Command(ScrapyCommand): def _print_headers(self, headers, prefix): for key, values in headers.items(): for value in values: - print('%s %s: %s' % (prefix, key, value)) + print('%s %s: %s' % ( + prefix, to_unicode(key), to_unicode(value))) def _print_response(self, response, opts): if opts.headers: @@ -38,7 +40,7 @@ class Command(ScrapyCommand): print('>') self._print_headers(response.headers, '<') else: - print(response.body) + print(to_unicode(response.body)) def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 01508bdb4..ad0375443 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -22,13 +22,13 @@ class SiteTest(object): def test_site(): r = resource.Resource() - r.putChild("text", static.Data("Works", "text/plain")) - r.putChild("html", static.Data("

Works

World

", "text/html")) - r.putChild("enc-gb18030", static.Data("

gb18030 encoding

", "text/html; charset=gb18030")) - r.putChild("redirect", util.Redirect("/redirected")) - r.putChild("redirected", static.Data("Redirected here", "text/plain")) + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) + r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) - + if __name__ == '__main__': port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f7d85ef7..1f0f34c49 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,3 @@ -tests/test_closespider.py -tests/test_command_fetch.py -tests/test_command_shell.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py @@ -12,7 +9,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_template.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 5283852b7..4843a9a2f 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -12,11 +12,11 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url('/text')]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace('\r', '') # required on win32 - assert 'Server: TwistedWeb' in out - assert 'Content-Type: text/plain' in out + out = out.replace(b'\r', b'') # required on win32 + assert b'Server: TwistedWeb' in out, out + assert b'Content-Type: text/plain' in out diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index a56236d54..105202754 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -12,38 +12,38 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(['-c', 'item']) - assert '{}' in out + assert b'{}' in out @defer.inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body']) - assert 'Works' in out + assert b'Works' in out @defer.inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)']) - assert 'TextResponse' in out + assert b'TextResponse' in out @defer.inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)']) - assert 'HtmlResponse' in out + assert b'HtmlResponse' in out @defer.inlineCallbacks def test_response_selector_html(self): xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) - self.assertEqual(out.strip(), 'Works') + self.assertEqual(out.strip(), b'Works') @defer.inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding']) - self.assertEqual(out.strip(), 'gb18030') + self.assertEqual(out.strip(), b'gb18030') @defer.inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) - assert out.strip().endswith('/redirected') + assert out.strip().endswith(b'/redirected') @defer.inlineCallbacks def test_request_replace(self): From c6d013ec85747fe7a0ec57852667bc4e29ec3e9c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:42:34 +0300 Subject: [PATCH 0124/3444] py3 fetch command: it may actually be better to try to print bytes as-is --- scrapy/commands/fetch.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 3888da210..49fa18ab2 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,11 +1,11 @@ from __future__ import print_function +import sys, six from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError from scrapy.utils.spider import spidercls_for_request, DefaultSpider -from scrapy.utils.python import to_unicode class Command(ScrapyCommand): @@ -29,10 +29,10 @@ class Command(ScrapyCommand): help="print response HTTP headers instead of body") def _print_headers(self, headers, prefix): + prefix = prefix.encode() for key, values in headers.items(): for value in values: - print('%s %s: %s' % ( - prefix, to_unicode(key), to_unicode(value))) + self._print_bytes(prefix + b' ' + key + b': ' + value) def _print_response(self, response, opts): if opts.headers: @@ -40,7 +40,11 @@ class Command(ScrapyCommand): print('>') self._print_headers(response.headers, '<') else: - print(to_unicode(response.body)) + self._print_bytes(response.body) + + def _print_bytes(self, bytes_): + bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer + bytes_writer.write(bytes_ + b'\n') def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): From a5da7531c42bb26af49787aa901c3d19b80abf2f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:06:20 +0300 Subject: [PATCH 0125/3444] py3 backout skipping test_closespider - it was fixed on another branch --- tests/py3-ignores.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 1f0f34c49..f8c631827 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,3 +1,4 @@ +tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py From fd24e22442d900170e7a5888b27698c9414b2983 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 20 Jan 2016 23:30:58 +0300 Subject: [PATCH 0126/3444] use byte constants for prefix instead of encoding it --- scrapy/commands/fetch.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 49fa18ab2..f09a873c1 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -29,16 +29,15 @@ class Command(ScrapyCommand): help="print response HTTP headers instead of body") def _print_headers(self, headers, prefix): - prefix = prefix.encode() for key, values in headers.items(): for value in values: self._print_bytes(prefix + b' ' + key + b': ' + value) def _print_response(self, response, opts): if opts.headers: - self._print_headers(response.request.headers, '>') + self._print_headers(response.request.headers, b'>') print('>') - self._print_headers(response.headers, '<') + self._print_headers(response.headers, b'<') else: self._print_bytes(response.body) From 659715ecd92c3f39ed0b52509adefb73c49fa56c Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Fri, 24 Jul 2015 12:07:59 -0300 Subject: [PATCH 0127/3444] implements FormRequest.from_response CSS support --- docs/topics/request-response.rst | 8 +++++++- scrapy/http/request/form.py | 10 ++++++++-- tests/test_http_request.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 0abec1f96..8f519b459 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -282,7 +282,7 @@ fields with form data from :class:`Response` objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ```` element contained @@ -310,6 +310,9 @@ fields with form data from :class:`Response` objects. :param formxpath: if given, the first form that matches the xpath will be used. :type formxpath: string + :param formcss: if given, the first form that matches the css selector will be used. + :type formcss: string + :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. :type formnumber: integer @@ -339,6 +342,9 @@ fields with form data from :class:`Response` objects. .. versionadded:: 0.17 The ``formxpath`` parameter. + .. versionadded:: 1.1.5 + The ``formcss`` parameter. + Request usage examples ---------------------- diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f623a5aa3..5501634d3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,8 +34,14 @@ class FormRequest(Request): @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, **kwargs): + clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): + kwargs.setdefault('encoding', response.encoding) + + if formcss is not None: + from parsel.csstranslator import HTMLTranslator + formxpath = HTMLTranslator().css_to_xpath(formcss) + form = _get_form(response, formname, formid, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = _get_form_url(form, kwargs.pop('url', None)) @@ -73,7 +79,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): f = root.xpath('//form[@id="%s"]' % formid) if f: return f[0] - + # Get form element from xpath, if not found, go up if formxpath is not None: nodes = root.xpath(formxpath) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index f82b2de8d..b81d43c41 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -846,6 +846,26 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response) self.assertEqual(req.url, 'http://b.com/test_form') + def test_from_response_css(self): + response = _buildresponse( + """ + + + +
+ + +
""") + r1 = self.request_class.from_response(response, formcss="form[action='post.php']") + fs = _qs(r1) + self.assertEqual(fs[b'one'], [b'1']) + + r1 = self.request_class.from_response(response, formcss="input[name='four']") + fs = _qs(r1) + self.assertEqual(fs[b'three'], [b'3']) + + self.assertRaises(ValueError, self.request_class.from_response, + response, formcss="input[name='abc']") def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) From d4c4ca80624c2540d5dedd218695d38542d92a01 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 09:42:15 -0200 Subject: [PATCH 0128/3444] fix version number to appear new feature --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8f519b459..ea64d1599 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -342,7 +342,7 @@ fields with form data from :class:`Response` objects. .. versionadded:: 0.17 The ``formxpath`` parameter. - .. versionadded:: 1.1.5 + .. versionadded:: 1.1.0 The ``formcss`` parameter. Request usage examples From 80c55f19a143d8938ced81a599100259509567a1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 21 Jan 2016 18:31:58 +0500 Subject: [PATCH 0129/3444] PY3 fixed scrapy bench command --- scrapy/utils/benchserver.py | 8 ++++---- tests/test_commands.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 4385d72a9..a9a2c938e 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -16,15 +16,15 @@ class Root(Resource): total = _getarg(request, 'total', 100, int) show = _getarg(request, 'show', 10, int) nlist = [random.randint(1, total) for _ in range(show)] - request.write("") + request.write(b"") args = request.args.copy() for nl in nlist: args['n'] = nl argstr = urlencode(args, doseq=True) request.write("follow {1}
" - .format(argstr, nl)) - request.write("") - return '' + .format(argstr, nl).encode('utf8')) + request.write(b"") + return b'' def _getarg(request, name, default=None, type=str): diff --git a/tests/test_commands.py b/tests/test_commands.py index 5755b3881..057112d12 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -72,7 +72,7 @@ class StartprojectTest(ProjectTest): self.assertEqual(1, self.call('startproject', self.project_name)) self.assertEqual(1, self.call('startproject', 'wrong---project---name')) self.assertEqual(1, self.call('startproject', 'sys')) - + class StartprojectTemplatesTest(ProjectTest): @@ -80,7 +80,7 @@ class StartprojectTemplatesTest(ProjectTest): super(StartprojectTemplatesTest, self).setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') - + def test_startproject_template_override(self): copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) os.mknod(join(self.tmpl_proj, 'root_template')) @@ -276,3 +276,4 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') log = to_native_str(p.stderr.read()) self.assertIn('INFO: Crawled', log) + self.assertNotIn('Unhandled Error', log) From a18dc24471f121ed85d1bee4281d43c3a3728162 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 21 Jan 2016 18:44:37 +0500 Subject: [PATCH 0130/3444] correctly process arguments for bench server --- scrapy/utils/benchserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index a9a2c938e..5bbda6e27 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -13,8 +13,8 @@ class Root(Resource): return self def render(self, request): - total = _getarg(request, 'total', 100, int) - show = _getarg(request, 'show', 10, int) + total = _getarg(request, b'total', 100, int) + show = _getarg(request, b'show', 10, int) nlist = [random.randint(1, total) for _ in range(show)] request.write(b"") args = request.args.copy() From f042ad0f39594d59a1a2032e6294ff1890638138 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Thu, 10 Dec 2015 23:04:25 -0500 Subject: [PATCH 0131/3444] py3 fix HttpProxy and Retry Middlewares --- scrapy/downloadermiddlewares/httpproxy.py | 6 +++--- tests/py3-ignores.txt | 4 ---- tests/test_downloadermiddleware_httpproxy.py | 4 ++-- tests/test_downloadermiddleware_retry.py | 8 ++++---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index dda6a3d2a..8c3514fd0 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,7 +9,7 @@ from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured - +from scrapy.utils.python import to_bytes class HttpProxyMiddleware(object): @@ -26,7 +26,7 @@ class HttpProxyMiddleware(object): proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = '%s:%s' % (unquote(user), unquote(password)) + user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password))) creds = base64.b64encode(user_pass).strip() else: creds = None @@ -52,4 +52,4 @@ class HttpProxyMiddleware(object): creds, proxy = self.proxies[scheme] request.meta['proxy'] = proxy if creds: - request.headers['Proxy-Authorization'] = 'Basic ' + creds + request.headers['Proxy-Authorization'] = b'Basic ' + creds diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 8c883ee3c..185a278fb 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -2,8 +2,6 @@ tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware_retry.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py @@ -23,8 +21,6 @@ scrapy/pipelines/files.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py -scrapy/downloadermiddlewares/retry.py -scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 5b9717a89..7676b2a00 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -52,7 +52,7 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz') + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -60,7 +60,7 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjo=') + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 20561e771..3de9399cf 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -21,20 +21,20 @@ class RetryTest(unittest.TestCase): def test_priority_adjust(self): req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority < req.priority def test_404(self): req = Request('http://www.scrapytest.org/404') - rsp = Response('http://www.scrapytest.org/404', body='', status=404) + rsp = Response('http://www.scrapytest.org/404', body=b'', status=404) # dont retry 404s assert self.mw.process_response(req, rsp, self.spider) is rsp def test_dont_retry(self): req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True}) - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) # first retry r = self.mw.process_response(req, rsp, self.spider) @@ -56,7 +56,7 @@ class RetryTest(unittest.TestCase): def test_503(self): req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) # first retry req = self.mw.process_response(req, rsp, self.spider) From a06a5f00f4a62028416ada4919994efd7d439eb2 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Wed, 20 Jan 2016 13:52:52 -0500 Subject: [PATCH 0132/3444] adding configurable encoding for httpproxy authentication --- docs/topics/downloader-middleware.rst | 12 ++++++++++++ scrapy/downloadermiddlewares/httpproxy.py | 13 +++++++++++-- scrapy/settings/default_settings.py | 2 ++ tests/test_downloadermiddleware_httpproxy.py | 19 +++++++++++++++++-- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3641da231..a97d5a696 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -951,6 +951,18 @@ Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to enable it for :ref:`broad crawls `. +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + .. _DBM: http://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 8c3514fd0..b01bab76d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes + class HttpProxyMiddleware(object): - def __init__(self): + def __init__(self, auth_encoding='latin-1'): + self.auth_encoding = auth_encoding self.proxies = {} for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) @@ -21,12 +23,19 @@ class HttpProxyMiddleware(object): if not self.proxies: raise NotConfigured + @classmethod + def from_crawler(cls, crawler): + auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') + return cls(auth_encoding) + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password))) + user_pass = to_bytes( + '%s:%s' % (unquote(user), unquote(password)), + encoding=self.auth_encoding) creds = base64.b64encode(user_pass).strip() else: creds = None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b151933b6..44e74dc61 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_AUTH_ENCODING = 'latin-1' + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7676b2a00..2b26431a4 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -9,6 +9,7 @@ from scrapy.spiders import Spider spider = Spider('foo') + class TestDefaultHeadersMiddleware(TestCase): failureException = AssertionError @@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + def test_proxy_auth_encoding(self): + # utf-8 encoding + os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + mw = HttpProxyMiddleware(auth_encoding='utf-8') + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + + # default latin-1 encoding + mw = HttpProxyMiddleware(auth_encoding='latin-1') + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() @@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None - def test_no_proxy(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() @@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta - From 20b839b44ba5795aa0a6cf96ac71d4524072fcab Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 12:42:45 +0300 Subject: [PATCH 0133/3444] py3: pass first crawl test (test_follow_all): fix mock server --- tests/mockserver.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index e7953c4d4..336633f4b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -7,6 +7,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl from scrapy import twisted_version +from scrapy.utils.python import to_bytes if twisted_version < (11, 0, 0): @@ -55,12 +56,12 @@ class LeafResource(Resource): class Follow(LeafResource): def render(self, request): - total = getarg(request, "total", 100, type=int) - show = getarg(request, "show", 1, type=int) - order = getarg(request, "order", "desc") - maxlatency = getarg(request, "maxlatency", 0, type=float) - n = getarg(request, "n", total, type=int) - if order == "rand": + total = getarg(request, b"total", 100, type=int) + show = getarg(request, b"show", 1, type=int) + order = getarg(request, b"order", b"desc") + maxlatency = getarg(request, b"maxlatency", 0, type=float) + n = getarg(request, b"n", total, type=int) + if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" nlist = range(n, max(n - show, 0), -1) @@ -73,19 +74,19 @@ class Follow(LeafResource): s = """ """ args = request.args.copy() for nl in nlist: - args["n"] = [str(nl)] + args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) s += "follow %d
" % (argstr, nl) s += """""" - request.write(s) + request.write(to_bytes(s)) request.finish() class Delay(LeafResource): def render_GET(self, request): - n = getarg(request, "n", 1, type=float) - b = getarg(request, "b", 1, type=int) + n = getarg(request, b"n", 1, type=float) + b = getarg(request, b"b", 1, type=int) if b: # send headers now and delay body request.write('') @@ -93,16 +94,16 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write("Response delayed for %0.3f seconds\n" % n) + request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) request.finish() class Status(LeafResource): def render_GET(self, request): - n = getarg(request, "n", 200, type=int) + n = getarg(request, b"n", 200, type=int) request.setResponseCode(n) - return "" + return b"" class Raw(LeafResource): @@ -114,7 +115,7 @@ class Raw(LeafResource): render_POST = render_GET def _delayedRender(self, request): - raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n') + raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n') request.startedWriting = 1 request.write(raw) request.channel.transport.loseConnection() @@ -128,7 +129,7 @@ class Echo(LeafResource): 'headers': dict(request.requestHeaders.getAllRawHeaders()), 'body': request.content.read(), } - return json.dumps(output) + return to_bytes(json.dumps(output)) class Partial(LeafResource): @@ -146,7 +147,7 @@ class Partial(LeafResource): class Drop(Partial): def _delayedRender(self, request): - abort = getarg(request, "abort", 0, type=int) + abort = getarg(request, b"abort", 0, type=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport try: From 0680950b9898aa77e4c494ab8a318d791ef0d55f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 13:06:31 +0300 Subject: [PATCH 0134/3444] py3: pass CrawlTestCase.test_referer_header, fixing Echo resource in mockserver and json decoding in test --- tests/mockserver.py | 9 +++++---- tests/test_crawl.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 336633f4b..6877c786c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,13 +1,12 @@ from __future__ import print_function import sys, time, random, os, json -import six from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl from scrapy import twisted_version -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode if twisted_version < (11, 0, 0): @@ -126,8 +125,10 @@ class Echo(LeafResource): def render_GET(self, request): output = { - 'headers': dict(request.requestHeaders.getAllRawHeaders()), - 'body': request.content.read(), + 'headers': dict( + (to_unicode(k), [to_unicode(v) for v in vs]) + for k, vs in request.requestHeaders.getAllRawHeaders()), + 'body': to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..90fd921c8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -8,6 +8,7 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.crawler import CrawlerRunner +from scrapy.utils.python import to_unicode from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider @@ -201,16 +202,16 @@ with multiples lines self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) # start requests doesn't set Referer header - echo0 = json.loads(crawler.spider.meta['responses'][2].body) + echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) self.assertNotIn('Referer', echo0['headers']) # following request sets Referer to start request url - echo1 = json.loads(crawler.spider.meta['responses'][1].body) + echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body)) self.assertEqual(echo1['headers'].get('Referer'), [req0.url]) # next request avoids Referer header - echo2 = json.loads(crawler.spider.meta['responses'][2].body) + echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) self.assertNotIn('Referer', echo2['headers']) # last request explicitly sets a Referer header - echo3 = json.loads(crawler.spider.meta['responses'][3].body) + echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body)) self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) @defer.inlineCallbacks From ad2b3321b90245e59b2e4b99eb7a4296f6c7e768 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 13:13:41 +0300 Subject: [PATCH 0135/3444] py3 compat: use range, fixes CrawlTestCase.test_start_requests_bug_yielding --- tests/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 516062929..711d80cac 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -119,7 +119,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_before_yield: 1 / 0 - for s in xrange(100): + for s in range(100): qargs = {'total': 10, 'seed': s} url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) From bf5f54fa339b44fec3451c88a78e4620f56c3bc8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:01:30 +0300 Subject: [PATCH 0136/3444] py3: fix getarg --- tests/mockserver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 6877c786c..365ec81fd 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -30,9 +30,12 @@ else: from twisted.internet.task import deferLater -def getarg(request, name, default=None, type=str): +def getarg(request, name, default=None, type=None): if name in request.args: - return type(request.args[name][0]) + value = request.args[name][0] + if type is not None: + value = type(value) + return value else: return default From 4607f2843e25b559f7483186916a1d91329c9dd8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:01:43 +0300 Subject: [PATCH 0137/3444] py3: unskip test_crawl --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 185a278fb..c8beea8a3 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,7 +1,6 @@ tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_crawl.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py From 5813de883888a69cf015ad506d6052e6191feb6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:08:05 +0300 Subject: [PATCH 0138/3444] py3: unskip test_closespider - it passes after fixing mockserver.Follow resouce on py3 --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index c8beea8a3..f189a4c86 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,4 +1,3 @@ -tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_mail.py From a76ecd4ef0bd7fa2dbe2e02d5b5721b39ead18c0 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 8 Oct 2015 22:18:14 -0300 Subject: [PATCH 0139/3444] remove test_exporters from py3 ignores --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index f189a4c86..570287d9d 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,4 +1,3 @@ -tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_mail.py tests/test_pipeline_files.py From b6ef1f19fd768243407206a882d689764624b42c Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 9 Oct 2015 00:19:05 -0300 Subject: [PATCH 0140/3444] make BaseItemExporter export unicode, pushed down previous behavior for classes that need it --- scrapy/exporters.py | 15 +++++++------ tests/test_exporters.py | 48 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 7e1d01a0a..6f679480d 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -38,7 +38,7 @@ class BaseItemExporter(object): raise NotImplementedError def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._to_str_if_unicode) + serializer = field.get('serializer', lambda x: x) return serializer(value) def start_exporting(self): @@ -47,9 +47,6 @@ class BaseItemExporter(object): def finish_exporting(self): pass - def _to_str_if_unicode(self, value): - return value.encode(self.encoding) if isinstance(value, unicode) else value - def _get_serialized_fields(self, item, default_value=None, include_empty=None): """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -89,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter): self.file.write(self.encoder.encode(itemdict) + '\n') -class JsonItemExporter(JsonLinesItemExporter): +class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) @@ -170,13 +167,17 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued + def serialize_field(self, field, name, value): + serializer = field.get('serializer', self._to_str_if_unicode) + return serializer(value) + def _to_str_if_unicode(self, value): if isinstance(value, (list, tuple)): try: value = self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return super(CsvItemExporter, self)._to_str_if_unicode(value) + return value.encode(self.encoding) if isinstance(value, unicode) else value def export_item(self, item): if self._headers_not_written: @@ -251,7 +252,7 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_dict(value)) if hasattr(value, '__iter__'): return [self._serialize_value(v) for v in value] - return self._to_str_if_unicode(value) + return value.encode(self.encoding) if isinstance(value, unicode) else value def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b24633959..c84fb978a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -23,7 +23,7 @@ class TestItem(Item): class BaseItemExporterTest(unittest.TestCase): def setUp(self): - self.i = TestItem(name=u'John\xa3', age='22') + self.i = TestItem(name=u'John\xa3', age=u'22') self.output = BytesIO() self.ie = self._get_exporter() @@ -55,6 +55,42 @@ class BaseItemExporterTest(unittest.TestCase): self.assertItemExportWorks(dict(self.i)) def test_serialize_field(self): + res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) + self.assertEqual(res, u'John\xa3') + + res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) + self.assertEqual(res, u'22') + + def test_fields_to_export(self): + ie = self._get_exporter(fields_to_export=['name']) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + + def test_field_custom_serializer(self): + def custom_serializer(value): + return str(int(value) + 2) + + class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + i = CustomFieldItem(name=u'John\xa3', age=u'22') + + ie = self._get_exporter() + self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + + +class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): + """Class introduced just to keep old behavior of BaseItemExporterTest for the + test cases that inherit from it while we make changes to exporters one by + one -- a needed refactoring trick because the test cases are quite coupled. + + When we're done with the changes, we'll have ditched this class. + """ + def test_serialize_field(self): + if self.ie.__class__ is BaseItemExporter: + return + res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) self.assertEqual(res, 'John\xc2\xa3') @@ -62,6 +98,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(res, '22') def test_fields_to_export(self): + if self.ie.__class__ is BaseItemExporter: + return + ie = self._get_exporter(fields_to_export=['name']) self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) @@ -71,6 +110,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, 'John\xa3') def test_field_custom_serializer(self): + if self.ie.__class__ is BaseItemExporter: + return + def custom_serializer(value): return str(int(value) + 2) @@ -85,7 +127,7 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class PythonItemExporterTest(BaseItemExporterTest): +class PythonItemExporterTest(MidRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(**kwargs) @@ -152,7 +194,7 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -class CsvItemExporterTest(BaseItemExporterTest): +class CsvItemExporterTest(MidRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) From c76190d491fca9f35b6758bdc06c34d77f5d9be9 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 18:24:06 -0200 Subject: [PATCH 0141/3444] PY3: ported json(lines), xml exporters --- scrapy/exporters.py | 39 +++++++++++++++------- tests/test_exporters.py | 72 +++++++++++++++++++++-------------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 6f679480d..4138f6192 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,7 +11,10 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder +from scrapy.utils.python import to_bytes, to_unicode from scrapy.item import BaseItem +import warnings + __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', @@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict) + '\n') + self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n')) class JsonItemExporter(BaseItemExporter): @@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter): self.first_item = True def start_exporting(self): - self.file.write("[") + self.file.write(b"[") def finish_exporting(self): - self.file.write("]") + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(',\n') + self.file.write(b',\n') itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict)) + self.file.write(to_bytes(self.encoder.encode(itemdict))) class XmlItemExporter(BaseItemExporter): @@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) - elif hasattr(serialized_value, '__iter__'): - for value in serialized_value: + elif (hasattr(serialized_value, '__iter__') + and not isinstance(serialized_value, six.string_types)): + for value in serialized_value: self._export_xml_field('value', value) else: self._xg_characters(serialized_value) @@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter): # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. if sys.version_info[:3] >= (2, 7, 4): def _xg_characters(self, serialized_value): - if not isinstance(serialized_value, unicode): + if not isinstance(serialized_value, six.text_type): serialized_value = serialized_value.decode(self.encoding) return self.xg.characters(serialized_value) else: @@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter): value = self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return value.encode(self.encoding) if isinstance(value, unicode) else value + return value.encode(self.encoding) if isinstance(value, six.text_type) else value def export_item(self, item): if self._headers_not_written: @@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(pprint.pformat(itemdict) + '\n') + self.file.write(to_bytes(pprint.pformat(itemdict) + '\n')) class PythonItemExporter(BaseItemExporter): @@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter): json, msgpack, binc, etc) can be used on top of it. Its main goal is to seamless support what BaseItemExporter does plus nested items. """ + def _configure(self, options, dont_fail=False): + self.binary = options.pop('binary', True) + super(PythonItemExporter, self)._configure(options, dont_fail) + if self.binary: + warnings.warn( + "PythonItemExporter will drop support for binary export in the future", + PendingDeprecationWarning) def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) @@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) - if hasattr(value, '__iter__'): + if hasattr(value, '__iter__') \ + and not isinstance(value, six.string_types): return [self._serialize_value(v) for v in value] - return value.encode(self.encoding) if isinstance(value, unicode) else value + if self.binary: + return to_bytes(value, encoding=self.encoding) + else: + return to_unicode(value, encoding=self.encoding) def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c84fb978a..05374e617 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,6 +3,7 @@ import re import json import unittest from io import BytesIO +import six from six.moves import cPickle as pickle import lxml.etree @@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): +class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest): """Class introduced just to keep old behavior of BaseItemExporterTest for the test cases that inherit from it while we make changes to exporters one by one -- a needed refactoring trick because the test cases are quite coupled. @@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class PythonItemExporterTest(MidRefactoringBaseItemExporterTest): +class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(**kwargs) + return PythonItemExporter(binary=False, **kwargs) def test_nested_item(self): i1 = TestItem(name=u'Joseph', age='22') @@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -class CsvItemExporterTest(MidRefactoringBaseItemExporterTest): +@unittest.skipUnless(six.PY2, "TODO") +class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = '\n22John\xc2\xa3' + expected_value = u'\n22John\xa3' self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - '\nJohn\xc2\xa3Doe' + u'\nJohn\xa3Doe' ) def test_nested_item(self): @@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=i2) self.assertExportResult(i3, - '\n' - '' - '' - '' - '' - '22' - 'foo\xc2\xa3hoo' - '' - 'bar' - '' - 'buz' - '' - '' + u'\n' + u'' + u'' + u'' + u'' + u'22' + u'foo\xa3hoo' + u'' + u'bar' + u'' + u'buz' + u'' + u'' ) def test_nested_list_item(self): @@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=[i1, i2]) self.assertExportResult(i3, - '\n' - '' - '' - '' - 'foo' - 'barspam' - '' - 'buz' - '' - '' + u'\n' + u'' + u'' + u'' + u'foo' + u'barspam' + u'' + u'buz' + u'' + u'' ) @@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, dict(self.i)) def test_nested_item(self): @@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) def test_extra_keywords(self): @@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): return JsonItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, [dict(self.i)]) def assertTwoItemsExported(self, item): @@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, [dict(item), dict(item)]) def test_two_items(self): @@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} self.assertEqual(exported, [expected]) @@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) From fed7c8b4fca3bb2722eebca97b298b0316ebfbc2 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 18:39:59 -0200 Subject: [PATCH 0142/3444] fix: use is_listlike --- scrapy/exporters.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 4138f6192..ad14f38b3 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,7 +11,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem import warnings @@ -139,8 +139,7 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) - elif (hasattr(serialized_value, '__iter__') - and not isinstance(serialized_value, six.string_types)): + elif is_listlike(serialized_value): for value in serialized_value: self._export_xml_field('value', value) else: @@ -261,8 +260,7 @@ class PythonItemExporter(BaseItemExporter): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) - if hasattr(value, '__iter__') \ - and not isinstance(value, six.string_types): + if is_listlike(value): return [self._serialize_value(v) for v in value] if self.binary: return to_bytes(value, encoding=self.encoding) From 9f35c286431584e07f9a53b1b2e7b7822a18f7e1 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 18:43:36 -0200 Subject: [PATCH 0143/3444] fix indentation --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ad14f38b3..c029ac473 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -140,7 +140,7 @@ class XmlItemExporter(BaseItemExporter): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) elif is_listlike(serialized_value): - for value in serialized_value: + for value in serialized_value: self._export_xml_field('value', value) else: self._xg_characters(serialized_value) From 240ecbf32378a8be9d87c98bec348301fdd05dc4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 21 Jan 2016 22:59:48 +0100 Subject: [PATCH 0144/3444] Add local file tests for scrapy shell command Continuation of #1579 --- tests/test_command_shell.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 105202754..7ae685c64 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,9 +1,14 @@ +from os.path import join, abspath, dirname, relpath, commonprefix +import os + from twisted.trial import unittest from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from tests import tests_datadir + class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -51,3 +56,39 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): code = "fetch('{0}') or fetch(response.request.replace(method='POST'))" errcode, out, _ = yield self.execute(['-c', code.format(url)]) self.assertEqual(errcode, 0, out) + + @defer.inlineCallbacks + def test_local_files(self): + test_file_path = join(tests_datadir, 'test_site/index.html') + valid_paths = [ + test_file_path, + relpath(test_file_path), + 'file://'+test_file_path, + './tests/sample_data/test_site/index.html', + 'tests/sample_data/test_site/index.html', + ] + for filepath in valid_paths: + _, out, _ = yield self.execute([filepath, '-c', 'item']) + assert b'{}' in out + + @defer.inlineCallbacks + def test_local_files_invalid(self): + invalid_filepaths = [ + '../nothinghere.html', + './tests/sample_data/test_site/nothinghere.html' + ] + for filepath in invalid_filepaths: + errcode, out, err = yield self.execute([filepath, '-c', 'item'], + check_code=False) + self.assertEqual(errcode, 1, out or err) + self.assertIn(b'No such file or directory', err) + + # currently, this will try to find a host... + invalid_paths = [ + 'nothinghere.html', + ] + for filepath in invalid_paths: + errcode, out, err = yield self.execute([filepath, '-c', 'item'], + check_code=False) + self.assertEqual(errcode, 1, out or err) + self.assertIn(b'DNS lookup failed', err) From 8bd5b60889bef67cb32879562d3cc0e751431ed2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 21 Jan 2016 23:23:50 +0100 Subject: [PATCH 0145/3444] Remove relpath filepath --- tests/test_command_shell.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7ae685c64..dd201cff3 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,5 +1,4 @@ -from os.path import join, abspath, dirname, relpath, commonprefix -import os +from os.path import join from twisted.trial import unittest from twisted.internet import defer @@ -62,7 +61,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): test_file_path = join(tests_datadir, 'test_site/index.html') valid_paths = [ test_file_path, - relpath(test_file_path), + # relpath(test_file_path), 'file://'+test_file_path, './tests/sample_data/test_site/index.html', 'tests/sample_data/test_site/index.html', From b746d85f4ca11f7f0149d06f7a4b58501ad3ee23 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:12:43 -0200 Subject: [PATCH 0146/3444] PY3 port csv exporter --- scrapy/exporters.py | 17 ++++++----- tests/test_exporters.py | 67 ++++++----------------------------------- 2 files changed, 20 insertions(+), 64 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c029ac473..8d7ffbc71 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats. """ import csv +import io import sys import pprint import marshal @@ -11,7 +12,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, is_listlike +from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike from scrapy.item import BaseItem import warnings @@ -166,21 +167,22 @@ class CsvItemExporter(BaseItemExporter): def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): self._configure(kwargs, dont_fail=True) self.include_headers_line = include_headers_line + file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True) self.csv_writer = csv.writer(file, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._to_str_if_unicode) + serializer = field.get('serializer', self._join_if_needed) return serializer(value) - def _to_str_if_unicode(self, value): + def _join_if_needed(self, value): if isinstance(value, (list, tuple)): try: - value = self._join_multivalued.join(value) + return self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return value.encode(self.encoding) if isinstance(value, six.text_type) else value + return value def export_item(self, item): if self._headers_not_written: @@ -189,7 +191,7 @@ class CsvItemExporter(BaseItemExporter): fields = self._get_serialized_fields(item, default_value='', include_empty=True) - values = [x[1] for x in fields] + values = [to_native_str(x) for _, x in fields] self.csv_writer.writerow(values) def _write_headers_and_set_fields_to_export(self, item): @@ -201,7 +203,8 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - self.csv_writer.writerow(self.fields_to_export) + row = [to_native_str(s) for s in self.fields_to_export] + self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 05374e617..39e996062 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,7 +3,6 @@ import re import json import unittest from io import BytesIO -import six from six.moves import cPickle as pickle import lxml.etree @@ -81,53 +80,6 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest): - """Class introduced just to keep old behavior of BaseItemExporterTest for the - test cases that inherit from it while we make changes to exporters one by - one -- a needed refactoring trick because the test cases are quite coupled. - - When we're done with the changes, we'll have ditched this class. - """ - def test_serialize_field(self): - if self.ie.__class__ is BaseItemExporter: - return - - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) - self.assertEqual(res, 'John\xc2\xa3') - - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) - self.assertEqual(res, '22') - - def test_fields_to_export(self): - if self.ie.__class__ is BaseItemExporter: - return - - ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) - - ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') - name = list(ie._get_serialized_fields(self.i))[0][1] - assert isinstance(name, str) - self.assertEqual(name, 'John\xa3') - - def test_field_custom_serializer(self): - if self.ie.__class__ is BaseItemExporter: - return - - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age='22') - - ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') - - class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(binary=False, **kwargs) @@ -195,19 +147,19 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -@unittest.skipUnless(six.PY2, "TODO") -class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): - +class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): + first = to_unicode(first) + second = to_unicode(second) csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line)) for line in csv.splitlines(True)] return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -221,13 +173,13 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): self.assertExportResult( item=self.i, fields_to_export=self.i.fields.keys(), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( item=dict(self.i), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): @@ -235,7 +187,7 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): self.assertExportResult( item=item, fields_to_export=['age'], - expected='age\r\n22\r\n', + expected=b'age\r\n22\r\n', ) def test_header_export_two_items(self): @@ -246,14 +198,15 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() - self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), + b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): for item in [self.i, dict(self.i)]: self.assertExportResult( item=item, include_headers_line=False, - expected='22,John\xc2\xa3\r\n', + expected=b'22,John\xc2\xa3\r\n', ) def test_join_multivalue(self): From 2514973242e35831fd90493b3db17227c3c0195e Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:22:12 -0200 Subject: [PATCH 0147/3444] re-enable skipped feed export tests --- tests/test_feedexport.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d6c96ca74..8e1cadc74 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage ) from scrapy.utils.test import assert_aws_environ +from scrapy.utils.python import to_native_str class FileFeedStorageTest(unittest.TestCase): @@ -120,8 +121,6 @@ class StdoutFeedStorageTest(unittest.TestCase): class FeedExportTest(unittest.TestCase): - skip = not six.PY2 - class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -170,7 +169,7 @@ class FeedExportTest(unittest.TestCase): settings.update({'FEED_FORMAT': 'csv'}) data = yield self.exported_data(items, settings) - reader = csv.DictReader(data.splitlines()) + reader = csv.DictReader(to_native_str(data).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -184,7 +183,7 @@ class FeedExportTest(unittest.TestCase): settings = settings or {} settings.update({'FEED_FORMAT': 'jl'}) data = yield self.exported_data(items, settings) - parsed = [json.loads(line) for line in data.splitlines()] + parsed = [json.loads(to_native_str(line)) for line in data.splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) From e938752973b4fc53e0fa0c0bc68a431613b987e4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:51:59 -0200 Subject: [PATCH 0148/3444] add test for PythonItemExporter binary mode --- scrapy/exporters.py | 6 +++++- tests/test_exporters.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 8d7ffbc71..118df34a7 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -272,7 +272,11 @@ class PythonItemExporter(BaseItemExporter): def _serialize_dict(self, value): for key, val in six.iteritems(value): + key = to_bytes(key) if self.binary else key yield key, self._serialize_value(val) def export_item(self, item): - return dict(self._get_serialized_fields(item)) + result = dict(self._get_serialized_fields(item)) + if self.binary: + result = dict(self._serialize_dict(result)) + return result diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 39e996062..9e57745dc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import re import json import unittest +import warnings from io import BytesIO from six.moves import cPickle as pickle @@ -115,6 +116,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) + def test_export_binary(self): + exporter = PythonItemExporter(binary=True) + value = TestItem(name=u'John\xa3', age=u'22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) + class PprintItemExporterTest(BaseItemExporterTest): From 35ada107297fc900e48824f9a94da478a5ae1c7a Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 22 Jan 2016 13:39:27 +0500 Subject: [PATCH 0149/3444] PY3 enable tests for scrapy parse command scrapy parse command is already ported --- tests/test_commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index a5230eb13..1a30368ba 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -205,8 +205,6 @@ from scrapy.spiders import Spider class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - skip = not six.PY2 - command = 'parse' def setUp(self): From 6d73e057b500f5063df63abd856ccd77cbafa1f8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 22 Jan 2016 13:07:42 +0100 Subject: [PATCH 0150/3444] Extract guess_scheme function and refactor tests --- scrapy/commands/shell.py | 30 ++++++---- tests/test_command_shell.py | 113 ++++++++++++++++++++++++++---------- 2 files changed, 102 insertions(+), 41 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 14dd0a41a..c975387be 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -16,6 +16,24 @@ from scrapy.utils.url import add_http_if_no_scheme from scrapy.utils.spider import spidercls_for_request, DefaultSpider +def guess_scheme(url): + """Given an URL as string, + returns a FileURI if it looks like a file path, + otherwise returns an HTTP URL + """ + parts = urlparse(url) + if not parts.scheme: + if "." not in parts.path.split("/", 1)[0]: + url = any_to_uri(url) + + for pattern in ["/", "./", "../"]: + if url.startswith(pattern): + url = any_to_uri(url) + break + url = add_http_if_no_scheme(url) + return url + + class Command(ScrapyCommand): requires_project = False @@ -50,16 +68,8 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None if url: - parts = urlparse(url) - if not parts.scheme: - if "." not in parts.path.split("/", 1)[0]: - url = any_to_uri(url) - - for pattern in ["/", "./", "../"]: - if url.startswith(pattern): - url = any_to_uri(url) - break - url = add_http_if_no_scheme(url) + # first argument may be a local file + url = guess_scheme(url) spider_loader = self.crawler_process.spider_loader diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index dd201cff3..37c6b8c90 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -3,12 +3,77 @@ from os.path import join from twisted.trial import unittest from twisted.internet import defer +from scrapy.commands.shell import guess_scheme from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest from tests import tests_datadir +class ShellURLTest(unittest.TestCase): + + def test_file_uri_relative001(self): + # FIXME: 'index.html' is interpreted as a domain name + # is this correct? + url = guess_scheme('index.html') + assert url.startswith('http://') + + def test_file_uri_relative002(self): + url = guess_scheme('./index.html') + assert url.startswith('file://') + + def test_file_uri_relative003(self): + url = guess_scheme('../data/index.html') + assert url.startswith('file://') + + def test_file_uri_relative004(self): + url = guess_scheme('subdir/index.html') + assert url.startswith('file://') + + def test_file_uri_absolute001(self): + """Absolute file paths get prepended with "file://" scheme""" + iurl = '/home/user/www/index.html' + url = guess_scheme(iurl) + self.assertEquals(url, 'file://'+iurl) + + def test_file_uri_scheme(self): + """Output File URI does not change if "file://" scheme is set""" + iurl = 'file:///home/user/www/index.html' + url = guess_scheme(iurl) + self.assertEquals(url, iurl) + + def test_file_uri_windows(self): + raise unittest.SkipTest("Windows filepath are not supported for scrapy shell") + url = guess_scheme('C:\absolute\path\to\a\file.html') + assert url.startswith('file://') + + def test_http_url_001(self): + url = guess_scheme('index.html') + assert url.startswith('http://') + + def test_http_url_002(self): + url = guess_scheme('example.com') + assert url.startswith('http://') + + def test_http_url_003(self): + url = guess_scheme('www.example.com') + assert url.startswith('http://') + + def test_http_url_004(self): + url = guess_scheme('www.example.com/index') + assert url.startswith('http://') + + def test_http_url_005(self): + url = guess_scheme('www.example.com/index.html') + assert url.startswith('http://') + + def test_http_url_scheme(self): + """An full HTTP URL is unaltered""" + iurl = 'http://www.example.com/index.html' + url = guess_scheme(iurl) + self.assertEquals(url, iurl) + + class ShellTest(ProcessTest, SiteTest, unittest.TestCase): command = 'shell' @@ -57,37 +122,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): self.assertEqual(errcode, 0, out) @defer.inlineCallbacks - def test_local_files(self): - test_file_path = join(tests_datadir, 'test_site/index.html') - valid_paths = [ - test_file_path, - # relpath(test_file_path), - 'file://'+test_file_path, - './tests/sample_data/test_site/index.html', - 'tests/sample_data/test_site/index.html', - ] - for filepath in valid_paths: - _, out, _ = yield self.execute([filepath, '-c', 'item']) - assert b'{}' in out + def test_local_file(self): + filepath = join(tests_datadir, 'test_site/index.html') + _, out, _ = yield self.execute([filepath, '-c', 'item']) + assert b'{}' in out @defer.inlineCallbacks - def test_local_files_invalid(self): - invalid_filepaths = [ - '../nothinghere.html', - './tests/sample_data/test_site/nothinghere.html' - ] - for filepath in invalid_filepaths: - errcode, out, err = yield self.execute([filepath, '-c', 'item'], - check_code=False) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b'No such file or directory', err) + def test_local_nofile(self): + filepath = 'file:///tests/sample_data/test_site/nothinghere.html' + errcode, out, err = yield self.execute([filepath, '-c', 'item'], + check_code=False) + self.assertEqual(errcode, 1, out or err) + self.assertIn(b'No such file or directory', err) - # currently, this will try to find a host... - invalid_paths = [ - 'nothinghere.html', - ] - for filepath in invalid_paths: - errcode, out, err = yield self.execute([filepath, '-c', 'item'], - check_code=False) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b'DNS lookup failed', err) + @defer.inlineCallbacks + def test_dns_failures(self): + url = 'www.somedomainthatdoesntexi.st' + errcode, out, err = yield self.execute([url, '-c', 'item'], + check_code=False) + self.assertEqual(errcode, 1, out or err) + self.assertIn(b'DNS lookup failed', err) From d0955fd08320f8402303bb4424e7dfab384068f4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 22 Jan 2016 10:07:55 -0200 Subject: [PATCH 0151/3444] add back test for latin-1 encoding --- tests/test_exporters.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 9e57745dc..070624830 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -2,11 +2,11 @@ from __future__ import absolute_import import re import json import unittest -import warnings from io import BytesIO from six.moves import cPickle as pickle import lxml.etree +import six from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -66,6 +66,11 @@ class BaseItemExporterTest(unittest.TestCase): ie = self._get_exporter(fields_to_export=['name']) self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') + _, name = list(ie._get_serialized_fields(self.i))[0] + assert isinstance(name, six.text_type) + self.assertEqual(name, u'John\xa3') + def test_field_custom_serializer(self): def custom_serializer(value): return str(int(value) + 2) From be239f339c4d4d43ff4f7bd5d3248bb4abd32834 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 22 Jan 2016 13:13:46 +0100 Subject: [PATCH 0152/3444] Remove unused import --- scrapy/commands/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index c975387be..1e8427753 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -5,7 +5,7 @@ See documentation in docs/topics/shell.rst """ import re -from six.moves.urllib.parse import urlparse, urlunparse +from six.moves.urllib.parse import urlparse from threading import Thread from w3lib.url import any_to_uri From 60052b3c68267eca6c0ed2a178378a2a777b3898 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 22 Jan 2016 13:18:08 +0100 Subject: [PATCH 0153/3444] Remove unused re import --- scrapy/commands/shell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 1e8427753..25ceb5f99 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,7 +4,6 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -import re from six.moves.urllib.parse import urlparse from threading import Thread from w3lib.url import any_to_uri From 7a51d370f3a59bcfe118f3ba079c5ea6eda5af75 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 22 Jan 2016 17:16:27 +0100 Subject: [PATCH 0154/3444] Regex-based guess_scheme() + refactor tests --- scrapy/commands/shell.py | 28 ++++++---- tests/test_command_shell.py | 102 +++++++++++++++++------------------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 25ceb5f99..5201feb42 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,7 +3,7 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ - +import re from six.moves.urllib.parse import urlparse from threading import Thread from w3lib.url import any_to_uri @@ -21,16 +21,22 @@ def guess_scheme(url): otherwise returns an HTTP URL """ parts = urlparse(url) - if not parts.scheme: - if "." not in parts.path.split("/", 1)[0]: - url = any_to_uri(url) - - for pattern in ["/", "./", "../"]: - if url.startswith(pattern): - url = any_to_uri(url) - break - url = add_http_if_no_scheme(url) - return url + if parts.scheme: + return url + # Note: this does not match Windows filepath + if re.match(r'''^ # start with... + ( + \. # ...a single dot, + ( + \. | [^/\.]+ # optionally followed by + )? # either a second dot or some characters + )? # optional match of ".", ".." or ".blabla" + / # at least one "/" for a file path, + . # and something after the "/" + ''', parts.path, flags=re.VERBOSE): + return any_to_uri(url) + else: + return add_http_if_no_scheme(url) class Command(ScrapyCommand): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 37c6b8c90..a61d520fa 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -11,67 +11,61 @@ from tests import tests_datadir class ShellURLTest(unittest.TestCase): + pass - def test_file_uri_relative001(self): - # FIXME: 'index.html' is interpreted as a domain name - # is this correct? - url = guess_scheme('index.html') - assert url.startswith('http://') +def create_guess_scheme_t(args): + def do_expected(self): + url = guess_scheme(args[0]) + assert url.startswith(args[1]), \ + 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( + args[0], url, args[1]) + return do_expected - def test_file_uri_relative002(self): - url = guess_scheme('./index.html') - assert url.startswith('file://') +def create_skipped_scheme_t(args): + def do_expected(self): + raise unittest.SkipTest(args[2]) + url = guess_scheme(args[0]) + assert url.startswith(args[1]) + return do_expected - def test_file_uri_relative003(self): - url = guess_scheme('../data/index.html') - assert url.startswith('file://') +for k, args in enumerate ([ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), - def test_file_uri_relative004(self): - url = guess_scheme('subdir/index.html') - assert url.startswith('file://') + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), - def test_file_uri_absolute001(self): - """Absolute file paths get prepended with "file://" scheme""" - iurl = '/home/user/www/index.html' - url = guess_scheme(iurl) - self.assertEquals(url, 'file://'+iurl) + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), - def test_file_uri_scheme(self): - """Output File URI does not change if "file://" scheme is set""" - iurl = 'file:///home/user/www/index.html' - url = guess_scheme(iurl) - self.assertEquals(url, iurl) + ], start=1): + t_method = create_guess_scheme_t(args) + t_method.__name__ = 'test_uri_%03d' % k + setattr (ShellURLTest, t_method.__name__, t_method) - def test_file_uri_windows(self): - raise unittest.SkipTest("Windows filepath are not supported for scrapy shell") - url = guess_scheme('C:\absolute\path\to\a\file.html') - assert url.startswith('file://') - - def test_http_url_001(self): - url = guess_scheme('index.html') - assert url.startswith('http://') - - def test_http_url_002(self): - url = guess_scheme('example.com') - assert url.startswith('http://') - - def test_http_url_003(self): - url = guess_scheme('www.example.com') - assert url.startswith('http://') - - def test_http_url_004(self): - url = guess_scheme('www.example.com/index') - assert url.startswith('http://') - - def test_http_url_005(self): - url = guess_scheme('www.example.com/index.html') - assert url.startswith('http://') - - def test_http_url_scheme(self): - """An full HTTP URL is unaltered""" - iurl = 'http://www.example.com/index.html' - url = guess_scheme(iurl) - self.assertEquals(url, iurl) +# TODO: the following tests do not pass with current implementation +for k, args in enumerate ([ + ('C:\absolute\path\to\a\file.html', 'file://', + 'Windows filepath are not supported for scrapy shell'), + ], start=1): + t_method = create_skipped_scheme_t(args) + t_method.__name__ = 'test_uri_skipped_%03d' % k + setattr (ShellURLTest, t_method.__name__, t_method) class ShellTest(ProcessTest, SiteTest, unittest.TestCase): From 1a30a7774b7d530394fdd761f2a59403b778fe10 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 22 Jan 2016 18:22:19 +0100 Subject: [PATCH 0155/3444] Use pytest.mark.parametrize decorator --- tests/test_command_shell.py | 96 +++++++++++++++---------------------- 1 file changed, 39 insertions(+), 57 deletions(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index a61d520fa..9032e4124 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,5 +1,7 @@ from os.path import join +import pytest + from twisted.trial import unittest from twisted.internet import defer @@ -10,63 +12,6 @@ from scrapy.utils.testproc import ProcessTest from tests import tests_datadir -class ShellURLTest(unittest.TestCase): - pass - -def create_guess_scheme_t(args): - def do_expected(self): - url = guess_scheme(args[0]) - assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( - args[0], url, args[1]) - return do_expected - -def create_skipped_scheme_t(args): - def do_expected(self): - raise unittest.SkipTest(args[2]) - url = guess_scheme(args[0]) - assert url.startswith(args[1]) - return do_expected - -for k, args in enumerate ([ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), - - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), - - # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), - - ], start=1): - t_method = create_guess_scheme_t(args) - t_method.__name__ = 'test_uri_%03d' % k - setattr (ShellURLTest, t_method.__name__, t_method) - -# TODO: the following tests do not pass with current implementation -for k, args in enumerate ([ - ('C:\absolute\path\to\a\file.html', 'file://', - 'Windows filepath are not supported for scrapy shell'), - ], start=1): - t_method = create_skipped_scheme_t(args) - t_method.__name__ = 'test_uri_skipped_%03d' % k - setattr (ShellURLTest, t_method.__name__, t_method) - class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -136,3 +81,40 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) + + +@pytest.mark.parametrize("url, scheme", [ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), + + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), + + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), + + pytest.mark.xfail( + (r'C:\absolute\path\to\a\file.html', 'file://'), + reason = 'Windows filepath are not supported for scrapy shell' + ), +]) +def test_guess_scheme(url, scheme): + guessed_url = guess_scheme(url) + assert guessed_url.startswith(scheme), \ + 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( + url, guessed_url, scheme) From 5f09da60c1a360cf0bd55cfcc892e44a13d5c583 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 22 Jan 2016 23:48:58 +0100 Subject: [PATCH 0156/3444] Revert "Use pytest.mark.parametrize decorator" This reverts commit 1a30a7774b7d530394fdd761f2a59403b778fe10. --- tests/test_command_shell.py | 96 ++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 39 deletions(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 9032e4124..a61d520fa 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,7 +1,5 @@ from os.path import join -import pytest - from twisted.trial import unittest from twisted.internet import defer @@ -12,6 +10,63 @@ from scrapy.utils.testproc import ProcessTest from tests import tests_datadir +class ShellURLTest(unittest.TestCase): + pass + +def create_guess_scheme_t(args): + def do_expected(self): + url = guess_scheme(args[0]) + assert url.startswith(args[1]), \ + 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( + args[0], url, args[1]) + return do_expected + +def create_skipped_scheme_t(args): + def do_expected(self): + raise unittest.SkipTest(args[2]) + url = guess_scheme(args[0]) + assert url.startswith(args[1]) + return do_expected + +for k, args in enumerate ([ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), + + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), + + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), + + ], start=1): + t_method = create_guess_scheme_t(args) + t_method.__name__ = 'test_uri_%03d' % k + setattr (ShellURLTest, t_method.__name__, t_method) + +# TODO: the following tests do not pass with current implementation +for k, args in enumerate ([ + ('C:\absolute\path\to\a\file.html', 'file://', + 'Windows filepath are not supported for scrapy shell'), + ], start=1): + t_method = create_skipped_scheme_t(args) + t_method.__name__ = 'test_uri_skipped_%03d' % k + setattr (ShellURLTest, t_method.__name__, t_method) + class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -81,40 +136,3 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) - - -@pytest.mark.parametrize("url, scheme", [ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), - - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), - - # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), - - pytest.mark.xfail( - (r'C:\absolute\path\to\a\file.html', 'file://'), - reason = 'Windows filepath are not supported for scrapy shell' - ), -]) -def test_guess_scheme(url, scheme): - guessed_url = guess_scheme(url) - assert guessed_url.startswith(scheme), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( - url, guessed_url, scheme) From c75f1fe46a8a2a3c471eeef2c023754ee5e6c2f1 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 23 Jan 2016 16:09:57 -0200 Subject: [PATCH 0157/3444] restore bytes instead of text, for easier reviewing --- tests/test_exporters.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 070624830..00352f61e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -261,13 +261,13 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = u'\n22John\xa3' + expected_value = b'\n22John\xc2\xa3' self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - u'\nJohn\xa3Doe' + b'\nJohn\xc2\xa3Doe' ) def test_nested_item(self): @@ -276,19 +276,19 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=i2) self.assertExportResult(i3, - u'\n' - u'' - u'' - u'' - u'' - u'22' - u'foo\xa3hoo' - u'' - u'bar' - u'' - u'buz' - u'' - u'' + b'\n' + b'' + b'' + b'' + b'' + b'22' + b'foo\xc2\xa3hoo' + b'' + b'bar' + b'' + b'buz' + b'' + b'' ) def test_nested_list_item(self): @@ -297,16 +297,16 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=[i1, i2]) self.assertExportResult(i3, - u'\n' - u'' - u'' - u'' - u'foo' - u'barspam' - u'' - u'buz' - u'' - u'' + b'\n' + b'' + b'' + b'' + b'foo' + b'barspam' + b'' + b'buz' + b'' + b'' ) From 935b1da8c2d1d801f4d7d9ce8f498c1d7a527644 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 23 Jan 2016 16:13:42 -0200 Subject: [PATCH 0158/3444] uses ScrapyDeprecationWarning instead of silenced PendingDeprecationWarning --- scrapy/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 118df34a7..fa6663ed4 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -14,6 +14,7 @@ from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike from scrapy.item import BaseItem +from scrapy.exceptions import ScrapyDeprecationWarning import warnings @@ -252,7 +253,7 @@ class PythonItemExporter(BaseItemExporter): if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", - PendingDeprecationWarning) + ScrapyDeprecationWarning) def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) From 9fbe6f3e814578f90f206031ae99a344b842e400 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sat, 23 Jan 2016 17:17:40 -0200 Subject: [PATCH 0159/3444] added feedexport test for xml output --- tests/test_feedexport.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e1cadc74..8db9d589e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,6 @@ import json from io import BytesIO import tempfile import shutil -import six from six.moves.urllib.parse import urlparse from zope.interface.verify import verifyObject @@ -187,10 +186,22 @@ class FeedExportTest(unittest.TestCase): rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'xml'}) + data = yield self.exported_data(items, settings) + rows = [{k: v for k, v in row.items() if v} for row in rows] + import lxml.etree + root = lxml.etree.fromstring(data) + got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] + self.assertEqual(rows, got_rows) + @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) @defer.inlineCallbacks def test_export_items(self): From 9704226ee4c933e1214e029d66005f1bee2fb766 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 13:25:14 +0300 Subject: [PATCH 0160/3444] py3: fix test_mail - get_payload returns bytes when decode is True --- tests/test_mail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mail.py b/tests/test_mail.py index 58d44bdb3..25dd35099 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -53,8 +53,8 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(len(payload), 2) text, attach = payload - self.assertEqual(text.get_payload(decode=True), 'body') - self.assertEqual(attach.get_payload(decode=True), 'content') + self.assertEqual(text.get_payload(decode=True), b'body') + self.assertEqual(attach.get_payload(decode=True), b'content') def _catch_mail_sent(self, **kwargs): self.catched_msg = dict(**kwargs) From 860353b0c03fffb2fed44942b4389f92bd7f7c09 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 13:27:41 +0300 Subject: [PATCH 0161/3444] py3: unskip test_mail and scrapy/mail.py --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index f189a4c86..70d3fb905 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,5 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py @@ -22,4 +21,3 @@ scrapy/linkextractors/htmlparser.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py -scrapy/mail.py From 333d4c91fb998b4f7f8a9e184e73715c33a38ce9 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 22:52:50 +0300 Subject: [PATCH 0162/3444] py3: add boto to py3 test requirements, test_pipeline_files and test_pipeline_images passing now --- tests/py3-ignores.txt | 2 -- tests/requirements-py3.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 70d3fb905..212f40f23 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,7 +1,5 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_pipeline_files.py -tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 5cf786a89..73e73e651 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,6 +4,7 @@ pytest-cov testfixtures jmespath leveldb +boto # optional for shell wrapper tests bpython ipython From 097082cffa0a9f11f72a3cee4d8941b7c2566538 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 23:05:23 +0300 Subject: [PATCH 0163/3444] reviewed py3 compat in pipelines/images.py and pipelines/files.py --- tests/py3-ignores.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 212f40f23..eb2cc4f5a 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -11,8 +11,6 @@ scrapy/xlib/tx/_newclient.py scrapy/xlib/tx/__init__.py scrapy/core/downloader/handlers/s3.py scrapy/core/downloader/handlers/ftp.py -scrapy/pipelines/images.py -scrapy/pipelines/files.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py From 4233b3cda4514a364511bc6f35a495cfbe50c4a6 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 23:10:03 +0300 Subject: [PATCH 0164/3444] py3: reviewed passing test_spidermiddleware_httperror.py --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 70d3fb905..e753b993e 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -3,7 +3,6 @@ tests/test_linkextractors_deprecated.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py -tests/test_spidermiddleware_httperror.py scrapy/xlib/tx/iweb.py scrapy/xlib/tx/interfaces.py From 1be90323c27bfda7588d4437a1996c5e92eb452d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 24 Jan 2016 23:44:56 +0300 Subject: [PATCH 0165/3444] py3: properly skip s3 tests on py3 --- tests/test_downloader_handlers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1eb6192ce..56608bfc6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -437,6 +437,8 @@ class S3AnonTestCase(unittest.TestCase): import boto except ImportError: skip = 'missing boto library' + if six.PY3: + skip = 'S3 not supported on Py3' def setUp(self): self.s3reqh = S3DownloadHandler(Settings(), @@ -459,6 +461,8 @@ class S3TestCase(unittest.TestCase): import boto except ImportError: skip = 'missing boto library' + if six.PY3: + skip = 'S3 not supported on Py3' # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf From 0c44fac2b54e72202787fda9d221f62a234151d4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Sun, 24 Jan 2016 19:17:42 -0200 Subject: [PATCH 0166/3444] added tests for feed export marshal and pickle --- tests/test_feedexport.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8db9d589e..176fd93e3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -197,11 +197,42 @@ class FeedExportTest(unittest.TestCase): got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] self.assertEqual(rows, got_rows) + def _load_until_eof(self, data, load_func): + bytes_output = BytesIO(data) + result = [] + while True: + try: + result.append(load_func(bytes_output)) + except EOFError: + break + return result + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'pickle'}) + data = yield self.exported_data(items, settings) + expected = [{k: v for k, v in row.items() if v} for row in rows] + import pickle + result = self._load_until_eof(data, load_func=pickle.load) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'marshal'}) + data = yield self.exported_data(items, settings) + expected = [{k: v for k, v in row.items() if v} for row in rows] + import marshal + result = self._load_until_eof(data, load_func=marshal.load) + self.assertEqual(expected, result) + @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) @defer.inlineCallbacks def test_export_items(self): From fb8ab2427bff636375b64fdc459562432c8ac420 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 25 Jan 2016 13:13:35 +0100 Subject: [PATCH 0167/3444] Move urlparsing statement in add_http_if_no_scheme() --- scrapy/utils/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 0e36003ad..0acbbb6ab 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -116,8 +116,8 @@ def escape_ajax(url): def add_http_if_no_scheme(url): """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.I) - parts = urlparse(url) if not match: + parts = urlparse(url) scheme = "http:" if parts.netloc else "http://" url = scheme + url From 23b3336c1feb3acb603a935f11e442557d6434f4 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Mon, 25 Jan 2016 22:11:04 -0200 Subject: [PATCH 0168/3444] add test for invalid option --- tests/test_exporters.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 00352f61e..61a0229a4 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -90,6 +90,10 @@ class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(binary=False, **kwargs) + def test_invalid_option(self): + with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"): + PythonItemExporter(invalid_option='something') + def test_nested_item(self): i1 = TestItem(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) From 2dfdde3c79a5be468302a1e825cc5ad77444a8ac Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Mon, 25 Jan 2016 22:24:35 -0200 Subject: [PATCH 0169/3444] fallback to repr when can't convert to native string --- scrapy/exporters.py | 11 +++++++++-- tests/test_exporters.py | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fa6663ed4..69c180ea4 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -192,9 +192,16 @@ class CsvItemExporter(BaseItemExporter): fields = self._get_serialized_fields(item, default_value='', include_empty=True) - values = [to_native_str(x) for _, x in fields] + values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def _build_row(self, values): + for s in values: + try: + yield to_native_str(s) + except TypeError: + yield to_native_str(repr(s)) + def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: @@ -204,7 +211,7 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - row = [to_native_str(s) for s in self.fields_to_export] + row = list(self._build_row(self.fields_to_export)) self.csv_writer.writerow(row) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 61a0229a4..8930545a6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -237,6 +237,13 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"Mary,Paul",John\r\n', ) + def test_join_multivalue_not_strings(self): + self.assertExportResult( + item=dict(name='John', friends=[4, 8]), + include_headers_line=False, + expected='"[4, 8]",John\r\n', + ) + class XmlItemExporterTest(BaseItemExporterTest): From d0eacfe0f90263035f98beb8fe6b5a8f182d5a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 26 Jan 2016 00:26:27 -0300 Subject: [PATCH 0170/3444] Add test case for marshal item exporter --- scrapy/exporters.py | 2 +- tests/test_exporters.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 69c180ea4..145468dbe 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -158,7 +158,7 @@ class XmlItemExporter(BaseItemExporter): if not isinstance(serialized_value, six.text_type): serialized_value = serialized_value.decode(self.encoding) return self.xg.characters(serialized_value) - else: + else: # pragma: no cover def _xg_characters(self, serialized_value): return self.xg.characters(serialized_value) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8930545a6..1633e1039 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,6 +1,8 @@ from __future__ import absolute_import import re import json +import marshal +import tempfile import unittest from io import BytesIO from six.moves import cPickle as pickle @@ -12,7 +14,8 @@ from scrapy.item import Item, Field from scrapy.utils.python import to_unicode from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, - XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter + XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, + PythonItemExporter, MarshalItemExporter ) @@ -163,6 +166,17 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) +class MarshalItemExporterTest(BaseItemExporterTest): + + def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() + return MarshalItemExporter(self.output, **kwargs) + + def _check_output(self): + self.output.seek(0) + self._assert_expected_item(marshal.load(self.output)) + + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) From 7070dae48da02fa29aa1650af4df28561e343436 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 26 Jan 2016 13:56:16 +0500 Subject: [PATCH 0171/3444] deprecate unused and untested scrapy.utils.datatypes.SiteNode --- scrapy/utils/datatypes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 097bd1ac9..2b54982b8 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -137,10 +137,18 @@ class MultiValueDict(dict): for key, value in six.iteritems(kwargs): self.setlistdefault(key, []).append(value) + class SiteNode(object): """Class to represent a site node (page, image or any other file)""" def __init__(self, url): + warnings.warn( + "scrapy.utils.datatypes.SiteNode is deprecated " + "and will be removed in future releases.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + self.url = url self.itemnames = [] self.children = [] From 9c2aa50ea20d2333eb131c6aae7b9842d646e32e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 26 Jan 2016 13:58:20 +0500 Subject: [PATCH 0172/3444] deprecate unused and untested scrapy.utils.datatypes.MultiValueDict --- scrapy/utils/datatypes.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2b54982b8..d04b43176 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -7,11 +7,22 @@ This module must not depend on any module outside the Standard Library. import copy import six +import warnings from collections import OrderedDict +from scrapy.exceptions import ScrapyDeprecationWarning + class MultiValueDictKeyError(KeyError): - pass + def __init__(self, *args, **kwargs): + warnings.warn( + "scrapy.utils.datatypes.MultiValueDictKeyError is deprecated " + "and will be removed in future releases.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + super(MultiValueDictKeyError, self).__init__(*args, **kwargs) + class MultiValueDict(dict): """ @@ -31,6 +42,10 @@ class MultiValueDict(dict): single name-value pairs. """ def __init__(self, key_to_list_mapping=()): + warnings.warn("scrapy.utils.datatypes.MultiValueDict is deprecated " + "and will be removed in future releases.", + category=ScrapyDeprecationWarning, + stacklevel=2) dict.__init__(self, key_to_list_mapping) def __repr__(self): From 713e1eee9b14ef95515b20baacb83daba9c1277a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 10:44:38 +0100 Subject: [PATCH 0173/3444] Update docs about local files support for "scrapy shell" --- docs/topics/commands.rst | 4 +++- docs/topics/shell.rst | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 16af52eea..9a40a2c29 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -373,7 +373,9 @@ shell * Requires project: *no* Starts the Scrapy shell for the given URL (if given) or empty if no URL is -given. See :ref:`topics-shell` for more info. +given. Also supports UNIX-style local file paths, either relative with +``./`` or ``../`` prefixes or absolute file paths. +See :ref:`topics-shell` for more info. Usage example:: diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 3569cbf37..4af11fbb6 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -53,6 +53,36 @@ this:: Where the ```` is the URL you want to scrape. +:command:`shell` also works for local files. This can be handy if you want +to play around with a local copy of a web page. :command:`shell` understands +the following syntaxes for local files:: + + # UNIX-style + scrapy shell ./path/to/file.html + scrapy shell ../other/path/to/file.html + scrapy shell /absolute/path/to/file.html + + # File URI + scrapy shell file:///absolute/path/to/file.html + +.. warning:: :command:`shell` will interpret ``index.html`` as a domain name, + not as a relative path to a local file, and will trigger a DNS lookup error:: + + $ scrapy shell index.html + [ ... scrapy shell starts ... ] + 2016-01-26 10:29:51 [scrapy] DEBUG: Gave up retrying + (failed 3 times): DNS lookup failed: + address 'index.html' not found: [Errno -5] No address associated with hostname. + [ ... traceback ... ] + twisted.internet.error.DNSLookupError: DNS lookup failed: + address 'index.html' not found: [Errno -5] No address associated with hostname. + + Use ``./`` prefix instead:: + + $ scrapy shell ./index.html + [ ... scrapy shell starts ... ] + + Using the shell =============== From 1cffa99e0d524ad6a3f989893de1693042e92f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Tue, 26 Jan 2016 12:35:40 +0200 Subject: [PATCH 0174/3444] tests+doc for subdomains in offsite middleware --- docs/topics/spider-middleware.rst | 3 +++ docs/topics/spiders.rst | 2 +- tests/test_spidermiddleware_offsite.py | 9 ++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 84daaaa55..ced481c71 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -273,6 +273,9 @@ OffsiteMiddleware This middleware filters out every request whose host names aren't in the spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5fd187e4e..b700ea0ef 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -76,7 +76,7 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names - specified in this list won't be followed if + specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index f88c806d7..37c3a450b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,7 +16,7 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org']) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -24,13 +24,16 @@ class TestOffsiteMiddleware(TestCase): onsite_reqs = [Request('http://scrapytest.org/1'), Request('http://scrapy.org/1'), Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True)] + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/')] offsite_reqs = [Request('http://scrapy2.org'), Request('http://offsite.tld/'), Request('http://offsite.tld/scrapytest.org'), Request('http://offsite.tld/rogue.scrapytest.org'), Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org')] + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/')] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) From 7608da8868baba981fa1d84f7691caaf1e0bd5d8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 13:01:12 +0100 Subject: [PATCH 0175/3444] Fix logging of enabled middlewares Wrong middlewares list was being pretty-printed (introduced in #1263) --- scrapy/middleware.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2ef5f30e2..6120488e2 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -44,9 +44,11 @@ class MiddlewareManager(object): logger.warning("Disabled %(clsname)s: %(eargs)s", {'clsname': clsname, 'eargs': e.args[0]}, extra={'crawler': crawler}) + + enabled = [x.__class__.__name__ for x in middlewares] logger.info("Enabled %(componentname)ss:\n%(enabledlist)s", {'componentname': cls.component_name, - 'enabledlist': pprint.pformat(mwlist)}, + 'enabledlist': pprint.pformat(enabled)}, extra={'crawler': crawler}) return cls(*middlewares) From 6ee8d8650a5d9041d6eeddddabcd16c0e0e8d9c9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 13:08:42 +0100 Subject: [PATCH 0176/3444] Disable CloseSpider extension if no CLOSPIDER_* setting set --- scrapy/extensions/closespider.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a5df5e8a7..9ccf356ec 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -9,6 +9,7 @@ from collections import defaultdict from twisted.internet import reactor from scrapy import signals +from scrapy.exceptions import NotConfigured class CloseSpider(object): @@ -23,6 +24,9 @@ class CloseSpider(object): 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), } + if not any(self.close_on.values()): + raise NotConfigured + self.counter = defaultdict(int) if self.close_on.get('errorcount'): From f30758c246ef10dc5ddb2316b747b28e109c9327 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 26 Jan 2016 17:47:46 +0500 Subject: [PATCH 0177/3444] Enable robots.txt handling by default for new projects. Fixes GH-1668. For backwards compatibility reasons the default value is not changed. --- docs/topics/settings.rst | 14 ++++++++++---- scrapy/templates/project/module/settings.py.tmpl | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cc070d8c0..0959a87a7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -750,8 +750,8 @@ Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` The :ref:`Memory usage extension ` -checks the current memory usage, versus the limits set by -:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, +checks the current memory usage, versus the limits set by +:setting:`MEMUSAGE_LIMIT_MB` and :setting:`MEMUSAGE_WARNING_MB`, at fixed time intervals. This sets the length of these intervals, in seconds. @@ -877,7 +877,13 @@ Default: ``False`` Scope: ``scrapy.downloadermiddlewares.robotstxt`` If enabled, Scrapy will respect robots.txt policies. For more information see -:ref:`topics-dlmw-robots` +:ref:`topics-dlmw-robots`. + +.. note:: + + While the default value is ``False`` for historical reasons, + this option is enabled by default in settings.py file generated + by ``scrapy startproject`` command. .. setting:: SCHEDULER @@ -1036,7 +1042,7 @@ TEMPLATES_DIR Default: ``templates`` dir inside scrapy module The directory where to look for templates when creating new projects with -:command:`startproject` command and new spiders with :command:`genspider` +:command:`startproject` command and new spiders with :command:`genspider` command. The project name must not conflict with the name of custom files or directories diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 822812c9a..f13e85871 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -18,6 +18,9 @@ NEWSPIDER_MODULE = '$project_name.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = '$project_name (+http://www.yourdomain.com)' +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 From 0349bbf9d3691ad89a5e265db4220fb3e64324ff Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 15:25:15 +0100 Subject: [PATCH 0178/3444] Disable SpiderState extension if no JOBDIR set --- scrapy/extensions/spiderstate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 3799c7c66..2220cbd8f 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -2,6 +2,7 @@ import os from six.moves import cPickle as pickle from scrapy import signals +from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir class SpiderState(object): @@ -12,7 +13,11 @@ class SpiderState(object): @classmethod def from_crawler(cls, crawler): - obj = cls(job_dir(crawler.settings)) + jobdir = job_dir(crawler.settings) + if not jobdir: + raise NotConfigured + + obj = cls(jobdir) crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed) crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj From 29695375d16ae20e3a97dc78bc12662996a9319b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 16:33:24 +0100 Subject: [PATCH 0179/3444] Add test for raised exception with SpiderState extension when no JOBDIR used --- tests/test_spiderstate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index d83015bd9..d1d6debec 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -4,6 +4,8 @@ from twisted.trial import unittest from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider +from scrapy.exceptions import NotConfigured +from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): @@ -34,3 +36,7 @@ class SpiderStateTest(unittest.TestCase): ss.spider_opened(spider) self.assertEqual(spider.state, {}) ss.spider_closed(spider) + + def test_not_configured(self): + crawler = get_crawler(Spider) + self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler) From c22a4e3bb84448f778613717c3749f226df7880f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 16:41:16 +0100 Subject: [PATCH 0180/3444] Use long classes names for enabled middlewares in startup logs --- scrapy/middleware.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 6120488e2..be36f977e 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -28,6 +28,7 @@ class MiddlewareManager(object): def from_settings(cls, settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] + enabled = [] for clspath in mwlist: try: mwcls = load_object(clspath) @@ -38,6 +39,7 @@ class MiddlewareManager(object): else: mw = mwcls() middlewares.append(mw) + enabled.append(clspath) except NotConfigured as e: if e.args: clsname = clspath.split('.')[-1] @@ -45,7 +47,6 @@ class MiddlewareManager(object): {'clsname': clsname, 'eargs': e.args[0]}, extra={'crawler': crawler}) - enabled = [x.__class__.__name__ for x in middlewares] logger.info("Enabled %(componentname)ss:\n%(enabledlist)s", {'componentname': cls.component_name, 'enabledlist': pprint.pformat(enabled)}, From bb1f4013a3883d72bcaa14b06a86ab428b99adfa Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 17:23:28 +0100 Subject: [PATCH 0181/3444] Rewrite warning about shell with local files as note --- docs/topics/shell.rst | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4af11fbb6..a6ca036d2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -65,22 +65,24 @@ the following syntaxes for local files:: # File URI scrapy shell file:///absolute/path/to/file.html -.. warning:: :command:`shell` will interpret ``index.html`` as a domain name, - not as a relative path to a local file, and will trigger a DNS lookup error:: +.. note:: When using relative file paths, be explicit and prepend them + with ``./`` (or ``../`` when relevant). + ``scrapy shell index.html`` will not work as one might expect (and + this is by design, not a bug). - $ scrapy shell index.html - [ ... scrapy shell starts ... ] - 2016-01-26 10:29:51 [scrapy] DEBUG: Gave up retrying - (failed 3 times): DNS lookup failed: - address 'index.html' not found: [Errno -5] No address associated with hostname. - [ ... traceback ... ] - twisted.internet.error.DNSLookupError: DNS lookup failed: - address 'index.html' not found: [Errno -5] No address associated with hostname. + Because :command:`shell` favors HTTP URLs over File URIs, + and ``index.html`` being syntactically similar to ``example.com``, + :command:`shell` will treat ``index.html`` as a domain name and trigger + a DNS lookup error:: - Use ``./`` prefix instead:: + $ scrapy shell index.html + [ ... scrapy shell starts ... ] + [ ... traceback ... ] + twisted.internet.error.DNSLookupError: DNS lookup failed: + address 'index.html' not found: [Errno -5] No address associated with hostname. - $ scrapy shell ./index.html - [ ... scrapy shell starts ... ] + :command:`shell` will not test beforehand if a file called ``index.html`` + exists in the current directory. Again, be explicit. Using the shell From 1c83108893cd3cc05e5b8b16e9c03d4a4786fdd6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 19:24:11 +0100 Subject: [PATCH 0182/3444] Clarify priority adjust settings docs Fixes #1593 --- docs/topics/settings.rst | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0959a87a7..116a10f83 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -864,8 +864,26 @@ REDIRECT_PRIORITY_ADJUST Default: ``+2`` -Adjust redirect request priority relative to original request. -A negative priority adjust means more priority. +Scope: ``scrapy.downloadermiddlewares.redirect.RedirectMiddleware`` + +Adjust redirect request priority relative to original request: + +- **a positive priority adjust (default) means higher priority.** +- a negative priority adjust means lower priority. + +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** .. setting:: ROBOTSTXT_OBEY From 4bcbb77bcc7d7668340baa064db15f29617cf0cb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jan 2016 01:28:11 +0500 Subject: [PATCH 0183/3444] response.text. Fixes GH-1729. --- docs/topics/request-response.rst | 42 +++++++++++++---------- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 4 +-- scrapy/http/request/form.py | 4 +-- scrapy/http/response/text.py | 5 +++ scrapy/selector/unified.py | 2 +- scrapy/utils/iterators.py | 2 +- scrapy/utils/response.py | 4 +-- tests/test_engine.py | 5 ++- tests/test_http_response.py | 30 +++++++++------- 10 files changed, 58 insertions(+), 42 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ea64d1599..2e92961a9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -445,10 +445,10 @@ Response objects .. attribute:: Response.body - A str containing the body of this Response. Keep in mind that Response.body - is always a str. If you want the unicode version use - :meth:`TextResponse.body_as_unicode` (only available in - :class:`TextResponse` and subclasses). + The body of this Response. Keep in mind that Response.body + is always a bytes object. If you want the unicode version use + :attr:`TextResponse.txt` (only available in :class:`TextResponse` + and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -542,6 +542,21 @@ TextResponse objects :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: + .. attribute:: TextResponse.text + + Response body, as unicode. + + The same as ``response.body.decode(response.encoding)``, but the + result is cached after the first call, so you can access + ``response.text`` multiple times without extra overhead. + + .. note:: + + ``unicode(response.body)`` is not a correct way to convert response + body to unicode: you would be using the system default encoding + (typically `ascii`) instead of the response encoding. + + .. attribute:: TextResponse.encoding A string with the encoding of this response. The encoding is resolved by @@ -568,20 +583,6 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.body_as_unicode() - - Returns the body of the response as unicode. This is equivalent to:: - - response.body.decode(response.encoding) - - But **not** equivalent to:: - - unicode(response.body) - - Since, in the latter case, you would be using the system default encoding - (typically `ascii`) to convert the body to unicode, instead of the response - encoding. - .. method:: TextResponse.xpath(query) A shortcut to ``TextResponse.selector.xpath(query)``:: @@ -594,6 +595,11 @@ TextResponse objects response.css('p') + .. method:: TextResponse.body_as_unicode() + + The same as :attr:`text`, but available as a method. This method is + kept for backwards compatibility; please prefer ``response.text``. + HtmlResponse objects -------------------- diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 6b543b823..da373eca2 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object): Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - body = response.body_as_unicode()[:self.lookup_bytes] + body = response.text[:self.lookup_bytes] return _has_ajaxcrawlable_meta(body) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..d4a33dc36 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object): def _parse_robots(self, response, netloc): rp = robotparser.RobotFileParser(response.url) body = '' - if hasattr(response, 'body_as_unicode'): - body = response.body_as_unicode() + if hasattr(response, 'text'): + body = response.text else: # last effort try try: body = response.body.decode('utf-8') diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 5501634d3..2862dc096 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -64,8 +64,8 @@ def _urlencode(seq, enc): def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ - text = response.body_as_unicode() - root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response)) + root = create_root_node(response.text, lxml.html.HTMLParser, + base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError("No
element found in %s" % response) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1c416bf82..9c667ab7e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -67,6 +67,11 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody + @property + def text(self): + """ Body as unicode """ + return self.body_as_unicode() + def urljoin(self, url): """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 5d77f7624..15f3d26df 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref): response = _response_from_text(text, st) if response is not None: - text = response.body_as_unicode() + text = response.text kwargs.setdefault('base_url', response.url) self.response = response diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b0688791e..73857b410 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): if not unicode: return obj.body elif isinstance(obj, TextResponse): - return obj.body_as_unicode() + return obj.text else: return obj.body.decode('utf-8') elif isinstance(obj, six.text_type): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c4ad52f14..73db2641e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary() def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: - text = response.body_as_unicode()[0:4096] + text = response.text[0:4096] _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] @@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: - text = response.body_as_unicode()[0:4096] + text = response.text[0:4096] text = _noscript_re.sub(u'', text) text = _script_re.sub(u'', text) _metaref_cache[response] = html.get_meta_refresh(text, response.url, diff --git a/tests/test_engine.py b/tests/test_engine.py index 9f2c02bff..baf6ef1bf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -55,12 +55,11 @@ class TestSpider(Spider): def parse_item(self, response): item = self.item_cls() - body = response.body_as_unicode() - m = self.name_re.search(body) + m = self.name_re.search(response.text) if m: item['name'] = m.group(1) item['url'] = response.url - m = self.price_re.search(body) + m = self.price_re.search(response.text) if m: item['price'] = m.group(1) return item diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 710a5b29d..c7f36687a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase): body_bytes = body assert isinstance(response.body, bytes) + assert isinstance(response.text, six.text_type) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) self.assertEqual(response.body_as_unicode(), body_unicode) + self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): self.assertEqual(response.encoding, resolve_encoding(encoding)) @@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest): self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type)) self.assertEqual(r1.body_as_unicode(), unicode_string) + # check response.text + self.assertTrue(isinstance(r1.text, six.text_type)) + self.assertEqual(r1.text, unicode_string) + def test_encoding(self): r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3") r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") @@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd') + self.assertEqual(r6.text, u'WORD\ufffd\ufffd') def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and - # response.body_as_unicode() in indistint order doesn't affect final + # response.text in indistint order doesn't affect final # values for encoding and decoded body. url = 'http://example.com' body = b"\xef\xbb\xbfWORD" @@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.body_as_unicode(), u'WORD') + self.assertEqual(response.text, u'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): @@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest): r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) - assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) - assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'\ufffd' in r.text, repr(r.text) + assert u'PREFIX' in r.text, repr(r.text) + assert u'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', \ body=b'\xf0value') - assert u'value' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse - #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') - #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') + #assert u'\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page" From 6ed08d23329bc33142804ea78320993d0c680175 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 27 Jan 2016 11:53:29 +0100 Subject: [PATCH 0184/3444] Add note for DEPTH_PRIORITY --- docs/topics/settings.rst | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 116a10f83..052be4429 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -276,6 +276,8 @@ DEPTH_LIMIT Default: ``0`` +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` + The maximum depth that will be allowed to crawl for any site. If zero, no limit will be imposed. @@ -286,9 +288,20 @@ DEPTH_PRIORITY Default: ``0`` -An integer that is used to adjust the request priority based on its depth. +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -If zero, no priority adjustment is made from depth. +An integer that is used to adjust the request priority based on its depth: + +- **a positive value will decrease the priority** +- a negative value will increase priority + +If zero (default), no priority adjustment is made from depth. + +.. note:: + + This setting adjusts priority **in the opposite way** compared to + other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` + and :setting:`RETRY_PRIORITY_ADJUST`. .. setting:: DEPTH_STATS @@ -297,6 +310,8 @@ DEPTH_STATS Default: ``True`` +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` + Whether to collect maximum depth stats. .. setting:: DEPTH_STATS_VERBOSE @@ -306,6 +321,8 @@ DEPTH_STATS_VERBOSE Default: ``False`` +Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` + Whether to collect verbose depth stats. If this is enabled, the number of requests for each depth is collected in the stats. From d999e3f7a704c5009999573d9514a5a51bd8be13 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 27 Jan 2016 12:57:03 +0100 Subject: [PATCH 0185/3444] More explicit description of DEPTH_PRIORITY --- docs/faq.rst | 4 +++- docs/topics/settings.rst | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 3d2bd8d4d..b3412211a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -45,7 +45,7 @@ Did Scrapy "steal" X from Django? Probably, but we don't like that word. We think Django_ is a great open source project and an example to follow, so we've used it as an inspiration for -Scrapy. +Scrapy. We believe that, if something is already done well, there's no need to reinvent it. This concept, besides being one of the foundations for open source and free @@ -85,6 +85,8 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. +.. _faq-bfo-dfo: + Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 052be4429..725345f2a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -292,10 +292,14 @@ Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` An integer that is used to adjust the request priority based on its depth: -- **a positive value will decrease the priority** -- a negative value will increase priority +- if zero (default), no priority adjustment is made from depth +- **a positive value will decrease the priority, i.e. higher depth + requests will be processed later** ; this is commonly used when doing + breadth-first crawls (BFO) +- a negative value will increase priority, i.e., higher depth requests + will be processed sooner (DFO) -If zero (default), no priority adjustment is made from depth. +See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. .. note:: From e0f48c486e4e658e80a4fea22f0fee58c3f01a0c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 27 Jan 2016 13:04:08 +0100 Subject: [PATCH 0186/3444] Add link to CoC mardown file on Github --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8a7d2c71d..3e050bb1e 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ Contributing ============ Please note that this project is released with a Contributor Code of Conduct -(see CODE_OF_CONDUCT.md). +(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@scrapinghub.com. From 7ca9ae19765d2c49c0e838ebbfc1596d0fbcd7d9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jan 2016 17:54:28 +0500 Subject: [PATCH 0187/3444] DOC typo fix --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 2e92961a9..82e674cee 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -447,7 +447,7 @@ Response objects The body of this Response. Keep in mind that Response.body is always a bytes object. If you want the unicode version use - :attr:`TextResponse.txt` (only available in :class:`TextResponse` + :attr:`TextResponse.text` (only available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use From dc8701ea429d4ded2f66d6b7c8fbce0bbcd0041a Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 12:56:42 -0200 Subject: [PATCH 0188/3444] Add test for already failed deferreds when downloading page in robots.txt middleware. --- tests/test_downloadermiddleware_robotstxt.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 5f45dcb82..f2e94e171 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) return deferred + def test_robotstxt_immediate_error(self): + self.crawler.settings.set('ROBOTSTXT_OBEY', True) + err = error.DNSLookupError('Robotstxt address not found') + def immediate_failure(request, spider): + deferred = Deferred() + deferred.errback(failure.Failure(err)) + return deferred + self.crawler.engine.download.side_effect = immediate_failure + + middleware = RobotsTxtMiddleware(self.crawler) + return self.assertNotIgnored(Request('http://site.local'), middleware) + def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) def ignore_request(request, spider): From b2beb3e85d2e82977d259eea71402809d00d197e Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 13:09:08 -0200 Subject: [PATCH 0189/3444] Fix handling of already failed deferreds when downloading page in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..7f6f0d012 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,7 +8,9 @@ import logging from six.moves.urllib import robotparser +from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + # engine.download() can return an already-called deferred, e.g. if a + # middleware returns a response in process_request(). Using + # deferLater() ensures that the error callback isn't called + # immediately upon being added, so that it doesn't remove the key + # before we check for it. + dfd = deferLater(reactor, 0, self.crawler.engine.download, + robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) From f1d971a5c0cdfe0f4fe5619146cd6818324fc98e Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 14:34:46 -0200 Subject: [PATCH 0190/3444] fix PythonItemExporter for non-string types --- scrapy/exporters.py | 8 ++++---- tests/test_exporters.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 145468dbe..c7c78d054 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -273,10 +273,10 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_dict(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if self.binary: - return to_bytes(value, encoding=self.encoding) - else: - return to_unicode(value, encoding=self.encoding) + encode_func = to_bytes if self.binary else to_unicode + if isinstance(value, (six.text_type, bytes)): + return encode_func(value, encoding=self.encoding) + return value def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1633e1039..662f8ec5c 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -134,6 +134,19 @@ class PythonItemExporterTest(BaseItemExporterTest): expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) + def test_other_python_types_item(self): + from datetime import datetime + now = datetime.now() + item = { + 'boolean': False, + 'number': 22, + 'time': now, + 'float': 3.14, + } + ie = self._get_exporter() + exported = ie.export_item(item) + self.assertEqual(exported, item) + class PprintItemExporterTest(BaseItemExporterTest): From c55ff110a34d39be27bbd3d03fbf52caa271b4c9 Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 15:43:17 -0200 Subject: [PATCH 0191/3444] Fix CSV exporter for non string Python types. --- scrapy/exporters.py | 2 +- tests/test_exporters.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c7c78d054..55d74332b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter): try: yield to_native_str(s) except TypeError: - yield to_native_str(repr(s)) + yield to_native_str(str(s)) def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 662f8ec5c..97c09a495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -271,6 +271,21 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"[4, 8]",John\r\n', ) + def test_other_python_types_item(self): + from datetime import datetime + now = datetime(2015, 1, 1, 1, 1, 1) + item = { + 'boolean': False, + 'number': 22, + 'time': now, + 'float': 3.14, + } + self.assertExportResult( + item=item, + include_headers_line=False, + expected='22,False,3.14,2015-01-01 01:01:01\r\n' + ) + class XmlItemExporterTest(BaseItemExporterTest): From 27758f60ada4791c044bfe8bc86d267aa930c744 Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 16:28:01 -0200 Subject: [PATCH 0192/3444] Changes fallback for CSVItemExporter, avoiding to call to_native_str(str()). --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 55d74332b..35f50838b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter): try: yield to_native_str(s) except TypeError: - yield to_native_str(str(s)) + yield s def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: From 3e080c3c52720535519ba1be7dee472258b19647 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Jan 2016 00:59:27 +0500 Subject: [PATCH 0193/3444] call .text from .body_as_unicode() and not the other way around --- scrapy/http/response/text.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 9c667ab7e..afa430329 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -59,7 +59,12 @@ class TextResponse(Response): def body_as_unicode(self): """Return body as unicode""" - # check for self.encoding before _cached_ubody just in + return self.text + + @property + def text(self): + """ Body as unicode """ + # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: @@ -67,11 +72,6 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody - @property - def text(self): - """ Body as unicode """ - return self.body_as_unicode() - def urljoin(self, url): """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" From 90e3ae1c580875e4e68c9d7238d0fb4642306bf9 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 21:00:35 -0200 Subject: [PATCH 0194/3444] Do not forget failed requests in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f6f0d012..6fdba90cc 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,9 +8,7 @@ import logging from six.moves.urllib import robotparser -from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred -from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -59,13 +57,7 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - # engine.download() can return an already-called deferred, e.g. if a - # middleware returns a response in process_request(). Using - # deferLater() ensures that the error callback isn't called - # immediately upon being added, so that it doesn't remove the key - # before we check for it. - dfd = deferLater(reactor, 0, self.crawler.engine.download, - robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) @@ -109,4 +101,6 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): - self._parsers.pop(netloc).callback(None) + rp_dfd = self._parsers[netloc] + self._parsers[netloc] = None + rp_dfd.callback(None) From cae268402d13a6d419c56024b3051b1cad3d82e1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 28 Jan 2016 13:42:04 +0100 Subject: [PATCH 0195/3444] Move guess_scheme() to scrapy.utils.url --- scrapy/commands/shell.py | 29 +---------------------------- scrapy/utils/url.py | 24 ++++++++++++++++++++++++ tests/test_command_shell.py | 2 +- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 5201feb42..7be7f7256 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,40 +3,13 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -import re -from six.moves.urllib.parse import urlparse from threading import Thread -from w3lib.url import any_to_uri from scrapy.commands import ScrapyCommand from scrapy.shell import Shell from scrapy.http import Request -from scrapy.utils.url import add_http_if_no_scheme from scrapy.utils.spider import spidercls_for_request, DefaultSpider - - -def guess_scheme(url): - """Given an URL as string, - returns a FileURI if it looks like a file path, - otherwise returns an HTTP URL - """ - parts = urlparse(url) - if parts.scheme: - return url - # Note: this does not match Windows filepath - if re.match(r'''^ # start with... - ( - \. # ...a single dot, - ( - \. | [^/\.]+ # optionally followed by - )? # either a second dot or some characters - )? # optional match of ".", ".." or ".blabla" - / # at least one "/" for a file path, - . # and something after the "/" - ''', parts.path, flags=re.VERBOSE): - return any_to_uri(url) - else: - return add_http_if_no_scheme(url) +from scrapy.utils.url import guess_scheme class Command(ScrapyCommand): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 0acbbb6ab..4b47566e5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -122,3 +122,27 @@ def add_http_if_no_scheme(url): url = scheme + url return url + + +def guess_scheme(url): + """Given an URL as string, + returns a FileURI if it looks like a file path, + otherwise returns an HTTP URL + """ + parts = urlparse(url) + if parts.scheme: + return url + # Note: this does not match Windows filepath + if re.match(r'''^ # start with... + ( + \. # ...a single dot, + ( + \. | [^/\.]+ # optionally followed by + )? # either a second dot or some characters + )? # optional match of ".", ".." or ".blabla" + / # at least one "/" for a file path, + . # and something after the "/" + ''', parts.path, flags=re.VERBOSE): + return any_to_uri(url) + else: + return add_http_if_no_scheme(url) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index a61d520fa..35a5fa21a 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -3,9 +3,9 @@ from os.path import join from twisted.trial import unittest from twisted.internet import defer -from scrapy.commands.shell import guess_scheme from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from scrapy.utils.url import guess_scheme from tests import tests_datadir From 481e251775a089a8e82c480a83181146d1bb6847 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 28 Jan 2016 13:51:50 +0100 Subject: [PATCH 0196/3444] Move guess_scheme() tests to relevant test module --- tests/test_command_shell.py | 58 --------------------------------- tests/test_utils_url.py | 65 +++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 35a5fa21a..9d0965902 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -10,64 +10,6 @@ from scrapy.utils.url import guess_scheme from tests import tests_datadir -class ShellURLTest(unittest.TestCase): - pass - -def create_guess_scheme_t(args): - def do_expected(self): - url = guess_scheme(args[0]) - assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( - args[0], url, args[1]) - return do_expected - -def create_skipped_scheme_t(args): - def do_expected(self): - raise unittest.SkipTest(args[2]) - url = guess_scheme(args[0]) - assert url.startswith(args[1]) - return do_expected - -for k, args in enumerate ([ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), - - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), - - # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), - - ], start=1): - t_method = create_guess_scheme_t(args) - t_method.__name__ = 'test_uri_%03d' % k - setattr (ShellURLTest, t_method.__name__, t_method) - -# TODO: the following tests do not pass with current implementation -for k, args in enumerate ([ - ('C:\absolute\path\to\a\file.html', 'file://', - 'Windows filepath are not supported for scrapy shell'), - ], start=1): - t_method = create_skipped_scheme_t(args) - t_method.__name__ = 'test_uri_skipped_%03d' % k - setattr (ShellURLTest, t_method.__name__, t_method) - - class ShellTest(ProcessTest, SiteTest, unittest.TestCase): command = 'shell' diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 314ccd30f..73ad11f8a 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,7 +4,8 @@ import unittest import six from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - canonicalize_url, add_http_if_no_scheme) + canonicalize_url, add_http_if_no_scheme, + guess_scheme) __doctests__ = ['scrapy.utils.url'] @@ -188,7 +189,7 @@ class CanonicalizeUrlTest(unittest.TestCase): class AddHttpIfNoScheme(unittest.TestCase): - + def test_add_scheme(self): self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') @@ -216,7 +217,7 @@ class AddHttpIfNoScheme(unittest.TestCase): def test_username_password(self): self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'), 'http://username:password@www.example.com') - + def test_complete_url(self): self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') @@ -294,5 +295,63 @@ class AddHttpIfNoScheme(unittest.TestCase): 'ftp://www.example.com') +class GuessSchemeTest(unittest.TestCase): + pass + +def create_guess_scheme_t(args): + def do_expected(self): + url = guess_scheme(args[0]) + assert url.startswith(args[1]), \ + 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( + args[0], url, args[1]) + return do_expected + +def create_skipped_scheme_t(args): + def do_expected(self): + raise unittest.SkipTest(args[2]) + url = guess_scheme(args[0]) + assert url.startswith(args[1]) + return do_expected + +for k, args in enumerate ([ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), + + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), + + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), + + ], start=1): + t_method = create_guess_scheme_t(args) + t_method.__name__ = 'test_uri_%03d' % k + setattr (GuessSchemeTest, t_method.__name__, t_method) + +# TODO: the following tests do not pass with current implementation +for k, args in enumerate ([ + ('C:\absolute\path\to\a\file.html', 'file://', + 'Windows filepath are not supported for scrapy shell'), + ], start=1): + t_method = create_skipped_scheme_t(args) + t_method.__name__ = 'test_uri_skipped_%03d' % k + setattr (GuessSchemeTest, t_method.__name__, t_method) + + if __name__ == "__main__": unittest.main() From e9f6b98816c220129fb02e6c09f488ecc8d686bd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 28 Jan 2016 14:39:19 +0100 Subject: [PATCH 0197/3444] Amend guess_scheme() docstring --- scrapy/utils/url.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4b47566e5..adef4a800 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -125,10 +125,7 @@ def add_http_if_no_scheme(url): def guess_scheme(url): - """Given an URL as string, - returns a FileURI if it looks like a file path, - otherwise returns an HTTP URL - """ + """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" parts = urlparse(url) if parts.scheme: return url From cf2ebb0687b9198f017cab836806bae5a5f3009c Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 19:36:16 -0200 Subject: [PATCH 0198/3444] Include tests for exporters: JSON, JSON-Lines, Pickle and Marshal. --- tests/test_exporters.py | 70 ++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 97c09a495..7ba5a0af6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -5,6 +5,7 @@ import marshal import tempfile import unittest from io import BytesIO +from datetime import datetime from six.moves import cPickle as pickle import lxml.etree @@ -42,6 +43,14 @@ class BaseItemExporterTest(unittest.TestCase): exported_dict[k] = to_unicode(v) self.assertEqual(self.i, exported_dict) + def _get_nonstring_types_item(self): + return { + 'boolean': False, + 'number': 22, + 'time': datetime(2015, 1, 1, 1, 1, 1), + 'float': 3.14, + } + def assertItemExportWorks(self, item): self.ie.start_exporting() try: @@ -134,15 +143,8 @@ class PythonItemExporterTest(BaseItemExporterTest): expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) - def test_other_python_types_item(self): - from datetime import datetime - now = datetime.now() - item = { - 'boolean': False, - 'number': 22, - 'time': now, - 'float': 3.14, - } + def test_nonstring_types_item(self): + item = self._get_nonstring_types_item() ie = self._get_exporter() exported = ie.export_item(item) self.assertEqual(exported, item) @@ -178,6 +180,15 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i1) self.assertEqual(pickle.load(f), i2) + def test_nonstring_types_item(self): + item = self._get_nonstring_types_item() + fp = BytesIO() + ie = PickleItemExporter(fp) + ie.start_exporting() + ie.export_item(item) + ie.finish_exporting() + self.assertEqual(pickle.loads(fp.getvalue()), item) + class MarshalItemExporterTest(BaseItemExporterTest): @@ -189,6 +200,17 @@ class MarshalItemExporterTest(BaseItemExporterTest): self.output.seek(0) self._assert_expected_item(marshal.load(self.output)) + def test_nonstring_types_item(self): + item = self._get_nonstring_types_item() + item.pop('time') # datetime is not marshallable + fp = tempfile.TemporaryFile() + ie = MarshalItemExporter(fp) + ie.start_exporting() + ie.export_item(item) + ie.finish_exporting() + fp.seek(0) + self.assertEqual(marshal.load(fp), item) + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -271,17 +293,9 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"[4, 8]",John\r\n', ) - def test_other_python_types_item(self): - from datetime import datetime - now = datetime(2015, 1, 1, 1, 1, 1) - item = { - 'boolean': False, - 'number': 22, - 'time': now, - 'float': 3.14, - } + def test_nonstring_types_item(self): self.assertExportResult( - item=item, + item=self._get_nonstring_types_item(), include_headers_line=False, expected='22,False,3.14,2015-01-01 01:01:01\r\n' ) @@ -390,6 +404,15 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self._check_output() self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True) + def test_nonstring_types_item(self): + item = self._get_nonstring_types_item() + self.ie.start_exporting() + self.ie.export_item(item) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue())) + item['time'] = str(item['time']) + self.assertEqual(exported, item) + class JsonItemExporterTest(JsonLinesItemExporterTest): @@ -438,6 +461,15 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) + def test_nonstring_types_item(self): + item = self._get_nonstring_types_item() + self.ie.start_exporting() + self.ie.export_item(item) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue())) + item['time'] = str(item['time']) + self.assertEqual(exported, [item]) + class CustomItemExporterTest(unittest.TestCase): From 78f00401cd284fd7bdcbb525ee94ea0bbedaa7cd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 16:56:05 +0100 Subject: [PATCH 0199/3444] Remove unused import in tests --- tests/test_command_shell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 9d0965902..c532fc0d8 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -5,7 +5,6 @@ from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from scrapy.utils.url import guess_scheme from tests import tests_datadir From a1ebff83d39e65cf4ed34281a1e81ea6cd108fe0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 18:39:34 +0100 Subject: [PATCH 0200/3444] Remove __str__ and __repr__ from settings, introduce copy_to_dict() instead Settings instances as dict's are easier to print or pretty print in the shell Fixes #1732 --- scrapy/settings/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 342d2585e..918bfc1e5 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -368,11 +368,25 @@ class BaseSettings(MutableMapping): def __len__(self): return len(self.attributes) - def __str__(self): - return str(self.attributes) + def _to_dict(self): + return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + for k, v in six.iteritems(self)} - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.attributes) + def copy_to_dict(self): + """ + Make a copy of current settings and convert to a dict. + + This method returns a new dict populated with the same values + and their priorities as the current settings. + + Modifications to the returned dict won't be reflected on the original + settings. + + This method can be useful for example for printing settings + in Scrapy shell. + """ + settings = self.copy() + return settings._to_dict() @property def overrides(self): From aa78758bc744b3264c330cc18e58a8b15315517d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 18:59:12 +0100 Subject: [PATCH 0201/3444] Update tests for settings copy_to_dict() --- tests/test_settings/__init__.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 44b9b6df3..4acf22cba 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -302,6 +302,21 @@ class BaseSettingsTest(unittest.TestCase): self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0], ['first_one', 'first_two']) + def test_copy_to_dict(self): + s = BaseSettings({'TEST_STRING': 'a string', + 'TEST_LIST': [1, 2], + 'TEST_BOOLEAN': False, + 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), + 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), + 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) + self.assertDictEqual(s.copy_to_dict(), + {'HASNOBASE': {3: 3000}, + 'TEST': {1: 10, 3: 30}, + 'TEST_BASE': {1: 1, 2: 2}, + 'TEST_BOOLEAN': False, + 'TEST_LIST': [1, 2], + 'TEST_STRING': 'a string'}) + def test_freeze(self): self.settings.freeze() with self.assertRaises(TypeError) as cm: @@ -343,14 +358,6 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(self.settings.defaults.get('BAR'), 'foo') self.assertIn('BAR', self.settings.defaults) - def test_repr(self): - settings = BaseSettings() - self.assertEqual(repr(settings), "") - attr = SettingsAttribute('testval', 15) - settings['testkey'] = attr - self.assertEqual(repr(settings), - "" % repr(attr)) - class SettingsTest(unittest.TestCase): From d843a0aae862a84b54f18f9d43ae957c182197b6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 21:12:03 +0100 Subject: [PATCH 0202/3444] Amend "settings" command to output JSON for dict settings --- scrapy/commands/settings.py | 9 ++++++++- tests/test_cmdline/__init__.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 0e73f4f58..bce4e6086 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,8 @@ from __future__ import print_function +import json + from scrapy.commands import ScrapyCommand +from scrapy.settings import BaseSettings class Command(ScrapyCommand): @@ -28,7 +31,11 @@ class Command(ScrapyCommand): def run(self, args, opts): settings = self.crawler_process.settings if opts.get: - print(settings.get(opts.get)) + s = settings.get(opts.get) + if isinstance(s, BaseSettings): + print(json.dumps(s.copy_to_dict())) + else: + print(s) elif opts.getbool: print(settings.getbool(opts.getbool)) elif opts.getint: diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index c2de4fbc8..7733e7180 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) - self.assertIn('value=200', settingsdict[EXT_PATH]) + self.assertEquals(200, settingsdict[EXT_PATH]) From f9dc02e23ae35e5ba9dce74dcc2571d02e90eb22 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Jan 2016 02:59:23 +0500 Subject: [PATCH 0203/3444] PY3 fix downloader slots GC --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 8beb08159..d835e65f7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -194,6 +194,6 @@ class Downloader(object): def _slot_gc(self, age=60): mintime = time() - age - for key, slot in self.slots.items(): + for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: self.slots.pop(key).close() From bb2cf7c0d7199fffe0aa100e5c8a51c6b4b82fc2 Mon Sep 17 00:00:00 2001 From: stummjr Date: Fri, 29 Jan 2016 19:23:26 -0200 Subject: [PATCH 0204/3444] Fixed bug on XMLItemExporter with non-string fields in items --- scrapy/exporters.py | 4 +++- tests/test_exporters.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 35f50838b..360007c0f 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -144,8 +144,10 @@ class XmlItemExporter(BaseItemExporter): elif is_listlike(serialized_value): for value in serialized_value: self._export_xml_field('value', value) - else: + elif isinstance(serialized_value, six.text_type): self._xg_characters(serialized_value) + else: + self._xg_characters(str(serialized_value)) self.xg.endElement(name) # Workaround for http://bugs.python.org/issue17606 diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 7ba5a0af6..cd72c661a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -376,6 +376,20 @@ class XmlItemExporterTest(BaseItemExporterTest): b'' ) + def test_nonstring_types_item(self): + item = self._get_nonstring_types_item() + self.assertExportResult(item, + b'\n' + b'' + b'' + b'3.14' + b'False' + b'22' + b'' + b'' + b'' + ) + class JsonLinesItemExporterTest(BaseItemExporterTest): From 268e912273dcb7bacc5a7102d5a3f90a868f035f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 1 Feb 2016 12:43:27 +0100 Subject: [PATCH 0205/3444] Add pretty-printting of settings as dict if using IPython shell Suggested by @digenis see http://ipython.readthedocs.org/en/stable/api/generated/IPython.lib.pretty.html?#extending --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 918bfc1e5..7b7808959 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,6 +4,7 @@ import copy import warnings from collections import MutableMapping from importlib import import_module +from pprint import pformat from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning @@ -388,6 +389,12 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() + def _repr_pretty_(self, p, cycle): + if cycle: + p.text(repr(self)) + else: + p.text(pformat(self.copy_to_dict())) + @property def overrides(self): warnings.warn("`Settings.overrides` attribute is deprecated and won't " From 65fb67f2dbdb36763832b7b332bdbff0fd2bd9db Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 3 Feb 2016 01:01:16 +0500 Subject: [PATCH 0206/3444] PY3 fixed CrawlerRunner.stop --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index bdcfa9d0c..ef99c243a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -189,7 +189,7 @@ class CrawlerRunner(object): Returns a deferred that is fired when they all have ended. """ - return defer.DeferredList([c.stop() for c in self.crawlers]) + return defer.DeferredList([c.stop() for c in list(self.crawlers)]) @defer.inlineCallbacks def join(self): From c6591b5c9f46271731ee711a23da952037a15388 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 3 Feb 2016 05:37:40 +0500 Subject: [PATCH 0207/3444] more efficient ExecutionEngine.spider_is_idle --- scrapy/core/engine.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index ef4403106..3c4bc662c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -177,12 +177,23 @@ class ExecutionEngine(object): return d def spider_is_idle(self, spider): - scraper_idle = self.scraper.slot.is_idle() - pending = self.slot.scheduler.has_pending_requests() - downloading = bool(self.downloader.active) - pending_start_requests = self.slot.start_requests is not None - idle = scraper_idle and not (pending or downloading or pending_start_requests) - return idle + if not self.scraper.slot.is_idle(): + # scraper is not idle + return False + + if self.downloader.active: + # downloader has pending requests + return False + + if self.slot.start_requests is not None: + # not all start requests are handled + return False + + if self.slot.scheduler.has_pending_requests(): + # scheduler has pending requests + return False + + return True @property def open_spiders(self): From db0697bc0654e490a2cc8773b8c65ad70f88a019 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 3 Feb 2016 12:32:40 +0100 Subject: [PATCH 0208/3444] Add 1.1 release notes (draft) --- docs/news.rst | 332 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 331 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 4d7dc4d41..6a83a6163 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,336 @@ Release notes ============= +1.1.0 (unreleased) +------------------ + +Python 3 Support (basic) +~~~~~~~~~~~~~~~~~~~~~~~~ + +We have been hard at work to make Scrapy work on Python 3. Some features +are still missing (and may never be ported to Python 3, see below), +but you can now run spiders on Python 3.3, 3.4 and 3.5. + +Almost all of addons/middleware should work, but here are the current +limitations we know of: + +- s3 downloads are not supported (see :issue:`1718`) +- sending emails is not supported +- FTP download handler is not supported (non-Python 3 ported Twisted dependency) +- telnet is not supported (non-Python 3 ported Twisted dependency) +- there are problems with non-ASCII URLs in Python 3 +- reported problems with HTTP cache created in Python 2.x which can't be used in 3.x (to be checked) +- there is also a nasty issue with cryptography library: + recent versions don't work well on OS X + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690), + downgrading to an older version helps + + +New Features and Enhancements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Command line tool has completion for zsh (:issue:`934`) +- ``scrapy shell`` works with local files again ; this was a regression + identified in 1.0+ releases (:issue:`1710`, :issue:`1550`) +- ``scrapy shell`` now also checks a new ``SCRAPY_PYTHON_SHELL`` environment + variable to launch the interactive shell of your choice ; + ``bpython`` is a newly supported option too (:issue:`1444`) +- Autothrottle has gotten a code cleanup and better docs ; + there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which + allows to send more than 1 concurrent request on average (:issue:`1324`) +- Memory usage extension has a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS`` + setting to change default check interval (:issue:`1282`) +- HTTP caching follows RFC2616 more closely (TODO: link to docs); + 2 new settings can be used to control level of compliancy: + ``HTTPCACHE_ALWAYS_STORE`` and ``HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS`` + (:issue:`1151`) +- Scheme Download handlers are now lazy-loaded on first request using + that scheme (``http(s)://``, ``ftp://``, ``file://``, ``s3://``) + (:issue:`1390`, :issue:`1421`) +- RedirectMiddleware now skips status codes in ``handle_httpstatus_list``, + set either as spider attribute or ``Request``'s ``meta`` key + (:issue:`1334`, :issue:`1364`, :issue:`1447`) + + +- Form submit button plain #1469 (https://github.com/scrapy/scrapy/commit/b876755f1cee619d8c421357777d223037d5289c) + Fixes: Form submit button (https://github.com/scrapy/scrapy/issues/1354) +- Implement FormRequest.from_response CSS support #1382 (https://github.com/scrapy/scrapy/commit/a6e5c848feb672c117f3380976077b6d0f42e3a6) + + Fix version number to appear new feature #1706 + +- Incomplete submit button #1472 (https://github.com/scrapy/scrapy/commit/bc499cb552dad362494b86082e47d1f732095874) + +- dont retry 400 #1289 (https://github.com/scrapy/scrapy/milestones/Scrapy%201.1) + + DOC fix docs after GH-1289. #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980) + WARNING: BACKWARDS INCOMPATIBLE! +- DOC fix docs after GH-1289. #1530 (https://github.com/scrapy/scrapy/commit/451318ef7a4e8ee7837b83e73b158da98f579980) + +- Support for returning deferreds in middlewares #1473 (https://github.com/scrapy/scrapy/commit/dd473145f2e1ae2d3c9462c489f3289a96e447f4) + Adds support for returning deferreds in middlewares, and makes use of this to fix a limitation in RobotsTxtMiddleware. + Fixes #1471 +- add support for a nested loaders #1467 (https://github.com/scrapy/scrapy/commit/3c596dcf4606315e4eb88608e3ecde430fe18c29) + Closes: https://github.com/scrapy/scrapy/pull/818 + Adds a nested_xpath()/nested_css() methods to ItemLoader. (TODO: add links to docs) + +- add_scheme_if_missing for `scrapy shell` command #1498 (https://github.com/scrapy/scrapy/commit/fe15f93e533be36e81e0385691fe5571c88b0b31) + Fixes: #1487 + Warning: backward incompatible + + + see: https://github.com/scrapy/scrapy/issues/1550, https://github.com/scrapy/scrapy/pull/1710 +- Per-key priorities for dict-like settings by promoting dicts to Settings instances #1149 (https://github.com/scrapy/scrapy/commit/dd9f777ba725d7a7dbb192302cc52a120005ad64) + + Backwards compatible per key priorities #1586 (https://github.com/scrapy/scrapy/commit/54216d7afe9d545031c57b5821f2c821faa2ccc3) + + Fixes: Per-key priorities for dictionary-like settings #1135 + Obsoletes: Settings.updatedict() method to update dictionary-like settings #1110 +- Support anonymous S3DownloadHandler (boto) connections #1358 (https://github.com/scrapy/scrapy/commit/5ec4319885e4be87b0248cb80b5213f68829129e) + + optional_features has been removed #1699 + +- Enable robots.txt handling by default for new projects. #1724 (https://github.com/scrapy/scrapy/commit/0d368c5d6fd468aed301ed5967f8bfe9d5e86101) + WARNING: backwards incompatible + +- Disable CloseSpider extension if no CLOSPIDER_* setting set #1723 (https://github.com/scrapy/scrapy/commit/2246280bb6f71d7d52e24aca5b4ce955b3aa1363) +- Disable SpiderState extension if no JOBDIR set #1725 + +- Add Code of Conduct Version 1.3.0 from http://contributor-covenant.org/ #1681 + + +API changes +~~~~~~~~~~~ + +- Update form.py to improve existing capability PR #1137 (https://github.com/scrapy/scrapy/commit/786f62664b41f264bf4213a1ee3805774d82ed69) + Adds "formid" parameter for Form from_response() + +- Add ExecutionEngine.close() method #1423 (https://github.com/scrapy/scrapy/commit/caf2080b8095acd11de6018911025076ead23585) + Adds a new method as a single entry point for shutting down the engine + and integrates it into Crawler.crawl() for graceful error handling during the crawling process. + + TODO: explain what this does +- public Crawler.create_crawler method #1528 (https://github.com/scrapy/scrapy/commit/57f87b95d4d705f8afdd8fb9f7551033a7d88ee2) + Note: this is a Core API change + Note: this is CrawlerRunner.create_crawler(), not Crawler.create_crawler + http://doc.scrapy.org/en/master/topics/api.html?#scrapy.crawler.CrawlerRunner.create_crawler + + Return a Crawler object. + + If crawler_or_spidercls is a Crawler, it is returned as-is. + If crawler_or_spidercls is a Spider subclass, a new Crawler is constructed for it. + If crawler_or_spidercls is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. + +- API CHANGE: response.text #1730 + micro-optimize response.text #1740 + New `.text` attribute on TextResponses + Response body, as unicode. + + +Deprecations and Removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- drop deprecated "optional_features" set #1359 (https://github.com/scrapy/scrapy/commit/7d187735ffecb0f49cffce1a9058961146212f59) +- Remove --lsprof command-line option. #1689 (https://github.com/scrapy/scrapy/commit/56b69d2ea85ccdebfa5ec7945f1ed1df54b4b87f) + WARNING: backward incompatible, but doesnt break user code + +- deprecated unused and untested code in scrapy.utils.datatypes #1720 + DEPRECATION: these will be removed in next releases + scrapy.utils.datatypes.MultiValueDictKeyError + scrapy.utils.datatypes.MultiValueDict + scrapy.utils.datatypes.SiteNode + + +Relocations +~~~~~~~~~~~ + +- Migrating selectors to use parsel #1409 (https://github.com/scrapy/scrapy/commit/15c1300d35e4764ea343d98c133bc83f7c90c2d6) + + Replace usage of deprecated class by its parsel\'s counterpart #1431 (https://github.com/scrapy/scrapy/commit/12bebb61725272cdd977ce914d18a4b18ec0cb77) + closes Scrapy.selector Enhancement Proposal (https://github.com/scrapy/scrapy/issues/906) +- Relocate telnetconsole to extensions/ #1524 (https://github.com/scrapy/scrapy/commit/72eeead6db7a5fdbce49a59102bb6a7125d56bc1) + Fixes: Move scrapy.telnet to scrapy.extensions.telnet #1520 + + See discussion on disabling telnet by default: (still open) https://github.com/scrapy/scrapy/issues/1572 + Note that telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595) + + +Documentation +~~~~~~~~~~~~~ + +- DOC SignalManager docstrings. See GH-713. #1291 (https://github.com/scrapy/scrapy/commit/5bd0395be4dc6d8315ad2726f1dbbd9c0b57b143) +- Improvements for docs on how to access settings #1302 (https://github.com/scrapy/scrapy/commit/8b3ca4f250b4d831403c7fcfa72efe7ecdfa5247) + (closes: https://github.com/scrapy/scrapy/issues/1300) +- Make Sphinx autodoc use local, not system-wide Scrapy PR #1335 (https://github.com/scrapy/scrapy/commit/b6eb3404a287508949ddb215e3f553a10fe43b8c) +- DOCS: Update deprecated examples #1660 (https://github.com/scrapy/scrapy/commit/95e8ff8ba1dff3ec045dce931b6ea4314e887399) +- DOCS: Update Stats Collection documentation for @master #1683 (https://github.com/scrapy/scrapy/commit/3f1f15bc4d3ee81612bce00fa0106ed16a7f72e5) +- DOCS: DOC: Update MetaRefreshMiddlware's setting variables #1642 (https://github.com/scrapy/scrapy/commit/b1e44436bc4629773388d25ad9ab7b8ecf43d15e) + + REDIRECT_MAX_METAREFRESH_DELAY has been deprecated and was renamed to METAREFRESH_MAXDELAY. + Merge duplicate documents about METAREFRESH_MAXDELAY appeared both in the settings page and the downloader-middlewares page. + + Leftover from https://github.com/scrapy/scrapy/commit/defc4f89b542b756276f0920921dc00fe3ec4675 +- DOCS;TESTS: tests+doc for subdomains in offsite middleware #1721 +- DOCS: Clarify priority adjust settings docs #1727 + + +Bugfixes +~~~~~~~~ + +- Support empty password for http_proxy config #1313 (https://github.com/scrapy/scrapy/commit/07f4f12e8b5417fe3e9f70560f7b60bc488570e8) + Fixes #1274 HTTP_PROXY variable with username and empty password not supported +- interpreting application/x-json as TextResponse #1333 (https://github.com/scrapy/scrapy/commit/2a7dc31f4cab7b13aacb632bdc78c50af754e76f) +- Support link rel attribute with multiple values #1214 (https://github.com/scrapy/scrapy/commit/aa31811cfdc85eda07ddab25178d5003155523ec) + Fixes: nofollow doesnt work correcly when there multiple values in rel attribute #1201 +- BUG FIX: for Incorrectly picked URL in `scrapy.http.FormRequest.from_response` when there is a `` tag #1562 + PR #1563 (https://github.com/scrapy/scrapy/commit/9548691fdd47077a53f85daace091ef4af599cb9) +- Startproject templates override #1575 (https://github.com/scrapy/scrapy/commit/3881eaff456d0d2704aa126f7c389080580d8f6c) + Closes: Override of TEMPLATES_DIR does not work for "startproject" command (https://github.com/scrapy/scrapy/issues/671) +- BUG FIX: Various FormRequest tests+fixes #1597 (https://github.com/scrapy/scrapy/commit/dc6502639556efbd06d45319efa8320e84e88fde) + Fixes: FormRequest should consider input type values case-insensitive #1595 + Fixes: FormRequest doesn't handle input elements without type attribute #1596 +- BUG FIX: for Incorrectly picked URL in `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `` tag. #1564 + PR #1565 (https://github.com/scrapy/scrapy/commit/17aba44f169fc3a86b6a1f46f30cf5fe29500db1) +- BUG FIX: BF: robustify _monkeypatches check for twisted - str() name first (Closes #1634) #1644 (https://github.com/scrapy/scrapy/commit/57f99fc34ebc7cb8a2a84371b89552e6623c9e9d) + Fixes: https://github.com/scrapy/scrapy/issues/1634 +- Fix bug on XMLItemExporter with non-string fields in items #1747 + Fixes: AttributeError when exporting non-string types through XMLFeedExporter #1738 +- change os.mknod() for open() #1657 + Fixes: Test for startproject command fails in OS X #1635 +- BUG FIX: Fix PythonItemExporter and CSVExporter for non-string item types #1737 + + +Python 3 porting effort +~~~~~~~~~~~~~~~~~~~~~~~ + +- Python 3: PY3 port scrapy.utils.python PR #1379 +- Python 3: In-progress Python 3 port PR #1384 + TODO: worth describing? +- Python 3: fix form requests tests on py3 (https://github.com/scrapy/scrapy/commit/de6e013b9a8080cf759096e793272f6814e3617d) +- Python 3: Port scrapy/responsetypes.py https://github.com/scrapy/scrapy/commit/d05cf6e0af8c26863cbb1edc7a8199165eaeeb5d +- Python 3: remove scrapy.utils.testsite from PY3 ignores #1397 +- Python 3: PY3 port scrapy.utils.response #1396 +- Python 3: PY3 port http cookies handling #1398 (https://github.com/scrapy/scrapy/commit/95e6bd2f8da9c0ed79c3667ae0619d35541de346) + +- Python 3: PY3 port scrapy.utils.reqser #1408 (https://github.com/scrapy/scrapy/commit/311293ffdc63892bd5ab8494310529a6da0f5b62) + +- Python 3: nyov's PY3 changes #1415 + Various files: + requirements-py3.txt + scrapy/cmdline.py + scrapy/core/downloader/handlers/s3.py + scrapy/core/downloader/middleware.py + scrapy/core/spidermw.py + scrapy/linkextractors/htmlparser.py + scrapy/pipelines/files.py + scrapy/pipelines/images.py + scrapy/utils/testproc.py + tests/py3-ignores.txt + tests/requirements-py3.txt + tests/test_cmdline/__init__.py + tests/test_command_version.py + tests/test_crawl.py + tests/test_loader.py + tests/test_pipeline_files.py + tests/test_pipeline_images.py + tests/test_selector_csstranslator.py + tests/test_selector_lxmldocument.py + tests/test_utils_iterators.py + tests/test_utils_reqser.py + tox.ini +- Python 3: py3: port dictionary itervalues call (666ebfa1d97264bc4e6adb78fe4ce1a9ea15cc1f) +- Python 3: PY3: port scrapy.utils.trackref #1420 (https://github.com/scrapy/scrapy/commit/fa3d84b0504e25f7478f7fac723a45) +- Python 3: Small Python 3 fixes #1456 (https://github.com/scrapy/scrapy/commit/026a1caffb9f0bafbefba4f56af61a7347750f20) +- Python 3: enable console tests in PY3 (8ecc4544b3747eb9be33153483b62c6441bd7c56) +- Python 3: assorted Python 3 porting #1461 (https://github.com/scrapy/scrapy/commit/0018caf0b61e4f10857e61cddb347c3854bacc4b) + Port LxmlLinkExtractor and leave other link extractors Python 2.x - only. + + refactor test_linkextractors + move tests for deprecated link extractors to another file and ignore it in Python 3 + port LxmlLinkExtractor to Python 3 + + scrapy.spiders and a couple more things + +- port some downloader middlewares to Python 3 #1470 (https://github.com/scrapy/scrapy/commit/3919ad64c5873d360aa1a412bee5270aad121760) + scrapy/downloadermiddlewares/httpauth.py + scrapy/downloadermiddlewares/useragent.py +- Python 3: PY3 redirect downloader mware #1488 (https://github.com/scrapy/scrapy/commit/4d1c5c3d32591c37e37f879f0e77e50db7124603) +- PY3 port bench, startproject, genspider, list and runspider commands #1535 (https://github.com/scrapy/scrapy/commit/411174cf38ebda00422529637b427a591c114eff) + Fixes: PY3 enable test_commands.ParseCommandTest #1536 +- Python 3: + - py3: fix webclient #1676 (https://github.com/scrapy/scrapy/commit/49fe631d8946f87e783c59e44a498f3d43083e2e) + - Py3: port http downloaders #1678 (https://github.com/scrapy/scrapy/commit/b4fb9d35342bc41a0149b74ecca38c056beaa220) + - Raise minimal twisted version for py3 #1694 (https://github.com/scrapy/scrapy/commit/d59d3f1e296795116704baa01780ff11870257f1) + - Cleanup http11 tunneling connection after #1678 #1701 + - Py3: port downloader cache and compression middlewares #1680 + - Add Python 3.5 tox env + Python 3.5 tests in Travis #1674 (https://github.com/scrapy/scrapy/commit/8fb9a6f8191dc0bf2dfb39ef01b1eb63e49bc23b) + - Py3: port test_engine #1691 + - Py3: port commands fetch and shell #1693 + - py3 fix HttpProxy and Retry Middlewares #1637 + - PY3 fixed scrapy bench command #1708 + - Py3: port test crawl #1692 + - PY3 enable tests for scrapy parse command #1711 + - py3: fix test_mail #1715 + - py3: reviewed passing test_spidermiddleware_httperror.py #1717 + - py3: test_pipeline_files and test_pipeline_images #1716 + - PY3 exporters #1499 + - PY3 fix downloader slots GC #1741 +- Python 3: PY3: port utils/iterators #1661 (https://github.com/scrapy/scrapy/commit/f01fd076420f0e58a1a165be31ec505eeb561ef4) + + +Tests, CI and Deploys +~~~~~~~~~~~~~~~~~~~~~ + +- BF: fail if docs failed to build #1319 +- Run on new travis-ci infra (https://github.com/scrapy/scrapy/commit/805a491647fabfed58acb9d2) + no more travis workarounds (removed .travis-workarounds.sh) +- Unset environment proxies for tests #1353 (https://github.com/scrapy/scrapy/commit/cbfb24dbeb82c791e82f1d9249685aa4d75fed3e) +- Coverage and reports at codecov.io and coveralls.io #1433 (https://github.com/scrapy/scrapy/commit/9adb5c31c06bc22d1b5243a04633a) +- drop coveralls support #1537 (https://github.com/scrapy/scrapy/commit/65f4ba349cb341736b67c0307074cef2cf0bd12e) +- Add some missing tests for scrapy.settings #1570 (https://github.com/scrapy/scrapy/commit/9424ca0fdbdd492f3049fe08be8848f92e84fde3) +- DOCS;TESTS: tests+doc for subdomains in offsite middleware #1721 +- TESTS: Include tests for non-string items to Exporters #1742 + + +Logging +~~~~~~~ + +- Ignore ScrapyDeprecationWarning warnings properly. #1294 (https://github.com/scrapy/scrapy/commit/64466526350820bdb424dc70968b4e015fd13641) +- Do not fail representing non-http requests #1419 (https://github.com/scrapy/scrapy/commit/bdcc78b4ddf47b6161b962b9d9fc8851b11f0117) +- Make list of enabled middlewares more readable #1263 (https://github.com/scrapy/scrapy/commit/a7787628ff53322e295be315e5595c555eb8e057) +- added more verbosity for log and for exception when download is cancelled because of a size limit #1624 (https://github.com/scrapy/scrapy/commit/fdc3c9d561ad87e417447fcee9adcc8cd6dbc594) +- LOGGING: show download warnsize once #1654 (https://github.com/scrapy/scrapy/commit/6827eab2c59e93d8ec46ef308bc751c6c00f32fd) +- LOGGING: Fix logging of enabled middlewares #1722 + Use long classes names for enabled middlewares in startup logs #1726 + + +Code refactoring +~~~~~~~~~~~~~~~~ + +- Avoid creation of temporary list object in iflatten #1476 (https://github.com/scrapy/scrapy/commit/6ae8963256f52bcc26ea8b4edc938743b07b6b2c) +- equal_attributes function optimization #1477 (https://github.com/scrapy/scrapy/commit/6490cb534e8e9a9068a8e298a8c6edb6be9725c5) +- Optimization - avoid temporary list objects, unnecessary function call #1481 (https://github.com/scrapy/scrapy/commit/3e13740a5765152e1b8241ad4db91efac5c746d7) +- Small downloader slots cleanup #1315 (https://github.com/scrapy/scrapy/commit/8a140b6ba1cf89e4a3bb74f8afb6e81c283e298b) + downloader.Slot becomes unaware of Scrapy settings; + it got __str__ and __repr__ methods useful in manhole; + unused import is dropped; + absolute_imports future import is added (I like adding it everywhere). +- extract CrawlerRunner._crawl method which always expects Crawler #1290 (https://github.com/scrapy/scrapy/commit/5bcda9b7d13b9c3b486c2b247fd6d87a7b59df1a) + Provides an extension point where crawler instance is available; + makes it easier to write alternative CrawlerRunner.crawl implementations. + User can override CrawlerRunner._crawl method and connect signals there. + + +Other changes +~~~~~~~~~~~~~ + + +- Extend regex for tags that deploy to PyPI to support new release cycle (:commit:`26f50d3`) +- rename str_to_unicode and unicode_to_str functions (ISSUE #778) (https://github.com/scrapy/scrapy/commit/61cd27e5c7b777a54) + +- fix utils.template.render_templatefile() bug +test #1212 (https://github.com/scrapy/scrapy/commit/71bd79e70fb10ed4899b15ca3ffa9aaa16567727) + +- style fixes for settings.py created by `scrapy startproject` #1496 (https://github.com/scrapy/scrapy/commit/5279da9916c00c7a6679cfc555f9a2b1863b4821) + Adds AUTOTHROTTLE_TARGET_CONCURRENCY to settings.py + +- (MINOR) Simplify if statement #1686 (https://github.com/scrapy/scrapy/commit/9ef25d7b68fe90c5e6b94bd3e81755089e743080) + Note: in conftest.py + +- (MINOR) fix indentation #1687 (https://github.com/scrapy/scrapy/commit/66f41aba3cbfa642b37354e8419e3d1437b88348) + Note: in scrapy/downloadermiddlewares/retry.py +- (MINOR) fixed typo You -> you #1698 (https://github.com/scrapy/scrapy/commit/e8b26e2ab25ac7ec15c03d3c0b766c7aa8f48cce) + Fixes DOWNLOAD_WARNSIZE is too verbose #1303 + + 1.0.4 (2015-12-30) ------------------ @@ -590,7 +920,7 @@ Enhancements - Document `request_scheduled` signal (:issue:`746`) - Add a note about reporting security issues (:issue:`697`) - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`) -- Sort spider list output of `scrapy list` command (:issue:`742`) +- Sort spider list output of `scrapy list` command (:issue:`742`) - Multiple documentation enhancemens and fixes (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`, :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`, From 142aa21737647864d8c9cdfff7e086be71041834 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 3 Feb 2016 12:33:10 +0100 Subject: [PATCH 0209/3444] Add AUTOTHROTTLE_TARGET_CONCURRENCY versionadded note --- docs/topics/autothrottle.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 0d664cf67..b83946a58 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -127,6 +127,8 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: 1.1 + Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote From 2b033eebcee5feb18bd96de19a843edb67fd7803 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 3 Feb 2016 12:34:44 +0100 Subject: [PATCH 0210/3444] Fix recently added HTTPCACHE_ settings versionadded notes --- docs/topics/downloader-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a97d5a696..4215cf69c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -582,7 +582,7 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 0.25 +.. versionadded:: 1.0 Default: ``False`` @@ -594,7 +594,7 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.25 +.. versionadded:: 1.1 Default: ``False`` @@ -614,7 +614,7 @@ responses you feedto the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.25 +.. versionadded:: 1.1 Default: ``[]`` From 44d8df2060dade244f40c70f1417236ef6e3cc50 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 3 Feb 2016 12:35:26 +0100 Subject: [PATCH 0211/3444] Add versionadded note for MEMUSAGE_CHECK_INTERVAL_SECONDS --- docs/topics/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 725345f2a..f8f35b5e3 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -766,6 +766,8 @@ See :ref:`topics-extensions-ref-memusage`. MEMUSAGE_CHECK_INTERVAL_SECONDS ------------------------------- +.. versionadded:: 1.1 + Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` From a21c90bee7658f3889a5093cda1693217c9b2e14 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Jr Date: Wed, 3 Feb 2016 11:54:46 -0200 Subject: [PATCH 0212/3444] edits on Py3 Support and New features sections --- docs/news.rst | 76 +++++++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 51 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 6a83a6163..f710192e8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,34 +9,31 @@ Release notes Python 3 Support (basic) ~~~~~~~~~~~~~~~~~~~~~~~~ -We have been hard at work to make Scrapy work on Python 3. Some features -are still missing (and may never be ported to Python 3, see below), -but you can now run spiders on Python 3.3, 3.4 and 3.5. +We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5, although some features are still missing (some of them may never be ported to Python 3). -Almost all of addons/middleware should work, but here are the current -limitations we know of: +Almost all addons/middlewares are expected to work. However, we are aware of some limitations: - s3 downloads are not supported (see :issue:`1718`) - sending emails is not supported - FTP download handler is not supported (non-Python 3 ported Twisted dependency) - telnet is not supported (non-Python 3 ported Twisted dependency) - there are problems with non-ASCII URLs in Python 3 -- reported problems with HTTP cache created in Python 2.x which can't be used in 3.x (to be checked) -- there is also a nasty issue with cryptography library: - recent versions don't work well on OS X + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690), - downgrading to an older version helps - +- reported problems with HTTP caches created by Scrapy in Python 2.x which can't be reused in Scrapy in Python 3.x (to be checked) +- there is also a nasty issue with `cryptography` library: recent versions don't work well on OS X + Python 3.5 (see https://github.com/pyca/cryptography/issues/2690). As a workaround, you can downgrade the library to an older version. New Features and Enhancements ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Command line tool has completion for zsh (:issue:`934`) -- ``scrapy shell`` works with local files again ; this was a regression +- Command line tool completion for zsh (:issue:`934`) +- ``scrapy shell`` works with local files again; this was a regression identified in 1.0+ releases (:issue:`1710`, :issue:`1550`) - ``scrapy shell`` now also checks a new ``SCRAPY_PYTHON_SHELL`` environment - variable to launch the interactive shell of your choice ; + variable to launch the interactive shell of your choice; ``bpython`` is a newly supported option too (:issue:`1444`) -- Autothrottle has gotten a code cleanup and better docs ; +- Scrapy shell now have `http` as the default schema for URLs. Now, you can + start it by: `scrapy shell scrapy.org` #1498 (https://github.com/scrapy/scrapy/commit/fe15f93e533be36e81e0385691fe5571c88b0b31). Fixes: #1487 **Warning: backwards incompatible!** + + see: https://github.com/scrapy/scrapy/issues/1550, https://github.com/scrapy/scrapy/pull/1710 +- Autothrottle code has been cleaned up and its docs have been improved; there's also a new ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting which allows to send more than 1 concurrent request on average (:issue:`1324`) - Memory usage extension has a new ``MEMUSAGE_CHECK_INTERVAL_SECONDS`` @@ -48,50 +45,27 @@ New Features and Enhancements - Scheme Download handlers are now lazy-loaded on first request using that scheme (``http(s)://``, ``ftp://``, ``file://``, ``s3://``) (:issue:`1390`, :issue:`1421`) -- RedirectMiddleware now skips status codes in ``handle_httpstatus_list``, - set either as spider attribute or ``Request``'s ``meta`` key +- RedirectMiddleware now skips the status codes from ``handle_httpstatus_list``. You can set it either as spider attribute or ``Request``'s ``meta`` key (:issue:`1334`, :issue:`1364`, :issue:`1447`) - - -- Form submit button plain #1469 (https://github.com/scrapy/scrapy/commit/b876755f1cee619d8c421357777d223037d5289c) - Fixes: Form submit button (https://github.com/scrapy/scrapy/issues/1354) -- Implement FormRequest.from_response CSS support #1382 (https://github.com/scrapy/scrapy/commit/a6e5c848feb672c117f3380976077b6d0f42e3a6) - + Fix version number to appear new feature #1706 - +- Form submission now works with `
`` elements +* ``//td``: selects all the ```` elements from the whole document. + Equivalent CSS selector: ``td``. * ``//div[@class="mine"]``: selects all ``div`` elements which contain an - attribute ``class="mine"`` + attribute ``class="mine"``. Equivalent CSS selector: ``div.mine``. These are just a couple of simple examples of what you can do with XPath, but XPath expressions are indeed much more powerful. To learn more about XPath, we @@ -220,7 +220,7 @@ to think in XPath" `_. Because of this, we encourage you to learn about XPath even if you already know how to construct CSS selectors. -For working with CSS and XPath expressions, Scrapy provides +For working with CSS and XPath expressions, Scrapy provides the :class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid instantiating selectors yourself every time you need to select something from a response. @@ -255,7 +255,7 @@ installed on your system. To start a shell, you must go to the project's top level directory and run:: - scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/" + scrapy shell "http://quotes.toscrape.com" .. note:: @@ -267,20 +267,20 @@ This is what the shell looks like:: [ ... Scrapy log here ... ] - 2014-01-23 17:11:42-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-01 18:14:39 [scrapy] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: - [s] crawler + [s] crawler [s] item {} - [s] request - [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] settings - [s] spider + [s] request + [s] response <200 http://quotes.toscrape.com> + [s] settings + [s] spider [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - - In [1]: + + >>> After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body @@ -297,19 +297,19 @@ or ``response.css()`` which map directly to ``response.selector.xpath()`` and So let's try it:: In [1]: response.xpath('//title') - Out[1]: [Open Directory - Computers: Progr'>] - + Out[1]: [Quotes to Scrape'>] + In [2]: response.xpath('//title').extract() - Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - + Out[2]: [u'Quotes to Scrape'] + In [3]: response.xpath('//title/text()') - Out[3]: [] - + Out[3]: [] + In [4]: response.xpath('//title/text()').extract() - Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - - In [5]: response.xpath('//title/text()').re('(\w+):') - Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] + Out[4]: [u'Quotes to Scrape'] + + In [11]: response.xpath('//title/text()').re('(\w+)') + Out[11]: [u'Quotes', u'to', u'Scrape'] Extracting the data ^^^^^^^^^^^^^^^^^^^ @@ -322,35 +322,42 @@ there could become a very tedious task. To make it easier, you can use Firefox Developer Tools or some Firefox extensions like Firebug. For more information see :ref:`topics-firebug` and :ref:`topics-firefox`. -After inspecting the page source, you'll find that the web site's information -is inside a ``