From 35b655d2f84d652440c393166f6e19d7384b4f1c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 12:23:22 +0100 Subject: [PATCH 1/9] Handle redirects transparently by default in shell and fetch Adds --no-status-aware command line option to have previous behaviour --- scrapy/commands/fetch.py | 5 ++++- scrapy/commands/shell.py | 4 +++- scrapy/shell.py | 12 ++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index f09a873c1..a157b19f8 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,6 +27,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not handle status codes like redirects and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -50,7 +52,8 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + if opts.no_status_aware: + request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7be7f7256..bc0203d89 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,6 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not transparently handle status codes like redirects") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -68,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url) + shell.start(url=url, handle_statuses=opts.no_status_aware) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 183ee1f70..966003f17 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -40,11 +40,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None): + def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider) + self.fetch(url, spider, handle_statuses=handle_statuses) elif request: self.fetch(request, spider) elif response: @@ -98,14 +98,14 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None): + def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): if isinstance(request_or_url, Request): request = request_or_url - url = request.url else: url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + request = Request(url, dont_filter=True, **kwargs) + if handle_statuses: + request.meta['handle_httpstatus_all'] = True response = None try: response, spider = threads.blockingCallFromThread( From 9aefc0a886ca571a49f087e5349e2557fd78d943 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 13:41:51 +0100 Subject: [PATCH 2/9] Add test for fetch command with redirections disabled --- scrapy/utils/testsite.py | 8 ++++++++ tests/test_command_fetch.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ad0375443..e50a989b3 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -20,12 +20,20 @@ class SiteTest(object): return urljoin(self.baseurl, path) +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request): + content = util.Redirect.render(self, request) + return content.replace(b'http-equiv=\"refresh\"', + b'http-no-equiv=\"do-not-refresh-me\"') + + def test_site(): r = resource.Resource() r.putChild(b"text", static.Data(b"Works", "text/plain")) r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 4843a9a2f..45d03a129 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/text')]) self.assertEqual(out.strip(), b'Works') + @defer.inlineCallbacks + def test_redirect_default(self): + _, out, _ = yield self.execute([self.url('/redirect')]) + self.assertEqual(out.strip(), b'Redirected here') + + @defer.inlineCallbacks + def test_redirect_disabled(self): + _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')]) + err = err.strip() + self.assertIn(b'downloader/response_status_count/302', err, err) + self.assertNotIn(b'downloader/response_status_count/200', err, err) + @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) From 778bed07bf771dd3942ea8cd51b7944065f4e2cd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 17:56:13 +0100 Subject: [PATCH 3/9] Let framework handle only HTTP redirects by default for fetch and shell commands --- scrapy/commands/fetch.py | 11 ++++++++--- scrapy/commands/shell.py | 6 +++--- scrapy/shell.py | 12 ++++++++---- scrapy/utils/datatypes.py | 10 ++++++++++ tests/test_command_fetch.py | 2 +- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a157b19f8..6fe6d73b9 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -27,8 +28,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") - parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ - default=False, help="do not handle status codes like redirects and print response as-is") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -52,7 +53,11 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - if opts.no_status_aware: + # by default, let the framework handle redirects, + # i.e. command handles all codes expect 3xx + if not opts.no_redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400)) + else: request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index bc0203d89..40a58d94a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,8 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ - default=False, help="do not transparently handle status codes like redirects") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -70,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url, handle_statuses=opts.no_status_aware) + shell.start(url=url, redirect=not opts.no_redirect) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 966003f17..6c78722be 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,6 +7,7 @@ from __future__ import print_function import os import signal +from six.moves import range import warnings from twisted.internet import reactor, threads, defer @@ -20,6 +21,7 @@ from scrapy.item import BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.console import start_python_console +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.conf import get_config @@ -40,11 +42,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): + def start(self, url=None, request=None, response=None, spider=None, redirect=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider, handle_statuses=handle_statuses) + self.fetch(url, spider, redirect=redirect) elif request: self.fetch(request, spider) elif response: @@ -98,13 +100,15 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): + def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): if isinstance(request_or_url, Request): request = request_or_url else: url = any_to_uri(request_or_url) request = Request(url, dont_filter=True, **kwargs) - if handle_statuses: + if redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: request.meta['handle_httpstatus_all'] = True response = None try: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d04b43176..e516185bd 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -304,3 +304,13 @@ class LocalCache(OrderedDict): while len(self) >= self.limit: self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) + + +class SequenceExclude(object): + """Object to test if an item is NOT within some sequence.""" + + def __init__(self, seq): + self.seq = seq + + def __contains__(self, item): + return item not in self.seq diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 45d03a129..3fa3ed930 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -21,7 +21,7 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_redirect_disabled(self): - _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')]) + _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')]) err = err.strip() self.assertIn(b'downloader/response_status_count/302', err, err) self.assertNotIn(b'downloader/response_status_count/200', err, err) From 7e54de24550df658690277839cbee99c9afe4bc8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 18:41:24 +0100 Subject: [PATCH 4/9] Add tests for shell command with and without --no-redirect --- tests/test_command_shell.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7bb7439d6..ee6e8ad8e 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -49,6 +49,16 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) assert out.strip().endswith(b'/redirected') + @defer.inlineCallbacks + def test_redirect_follow_302(self): + _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'200') + + @defer.inlineCallbacks + def test_redirect_not_follow_302(self): + _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'302') + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') From 2cd579a7748d0c37eac557216b857c38fbcf80df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 7 Dec 2016 19:07:32 +0100 Subject: [PATCH 5/9] Add test for fetch(url) within shell with and without redirect --- tests/test_command_shell.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index ee6e8ad8e..3e27d6abd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -59,6 +59,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) assert out.strip().endswith(b'302') + @defer.inlineCallbacks + def test_fetch_redirect_follow_302(self): + """Test that calling `fetch(url)` follows HTTP redirects by default.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}')" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Redirecting (302)' in errout + assert b'Crawled (200)' in errout + + @defer.inlineCallbacks + def test_fetch_redirect_not_follow_302(self): + """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Crawled (302)' in errout + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') From 7d1783603251923bb549d34a64d43952fe03b3bc Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 8 Dec 2016 17:27:25 +0100 Subject: [PATCH 6/9] Update documentation about --no-redirect option --- docs/topics/commands.rst | 28 ++++++++++++++++++ docs/topics/shell.rst | 62 ++++++++++++++++++++++++++-------------- scrapy/shell.py | 9 ++++-- 3 files changed, 75 insertions(+), 24 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 32669104c..3a26b19ae 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page. If used outside a project, no particular per-spider behaviour would be applied and it will just use the default Scrapy downloader settings. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--headers``: print the response's HTTP headers instead of the response's body + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html @@ -368,11 +376,31 @@ given. Also supports UNIX-style local file paths, either relative with ``./`` or ``../`` prefixes or absolute file paths. See :ref:`topics-shell` for more info. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``-c code``: evaluate the code in the shell, print the result and exit + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy shell http://www.example.com/some/page.html [ ... scrapy shell starts ... ] + $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)' + (200, 'http://www.example.com/') + + # shell follows HTTP redirects by default + $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (200, 'http://example.com/') + + # you can disable this with --no-redirect + $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') + + .. command:: parse parse diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..6eb81a71f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -97,8 +97,12 @@ Available Shortcuts * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``fetch(request_or_url)`` - fetch a new response from the given request or - URL and update all related objects accordingly. + * ``fetch(url[, redirect=True])`` - fetch a new response from the given + URL and update all related objects accordingly. You can optionaly ask for + HTTP 3xx redirections to not be followed by passing ``redirect=False`` + + * ``fetch(request)`` - fetch a new response from the given request and + update all related objects accordingly. * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order @@ -157,36 +161,28 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available Scrapy objects: - [s] crawler + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler [s] item {} [s] request - [s] response <200 http://scrapy.org> - [s] settings - [s] spider + [s] response <200 https://scrapy.org/> + [s] settings + [s] spider [s] Useful shortcuts: + [s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed) + [s] fetch(req) Fetch a scrapy.Request and update local objects [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> + After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") - [s] Available Scrapy objects: - [s] crawler - [s] item {} - [s] request - [s] response <200 https://www.reddit.com/> - [s] settings - [s] spider - [s] Useful shortcuts: - [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects - [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() [u'reddit: the front page of the internet'] @@ -194,12 +190,36 @@ After that, we can start playing with the objects:: >>> request = request.replace(method="POST") >>> fetch(request) - [s] Available Scrapy objects: - [s] crawler - ... + >>> response.status + 404 + + >>> from pprint import pprint + + >>> pprint(response.headers) + {'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} >>> + .. _topics-shell-inspect-response: Invoking the shell from spiders to inspect responses diff --git a/scrapy/shell.py b/scrapy/shell.py index 6c78722be..babc267c7 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -148,10 +148,13 @@ class Shell(object): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) b.append("Useful shortcuts:") - b.append(" shelp() Shell help (print this help)") if self.inthread: - b.append(" fetch(req_or_url) Fetch request (or URL) and " - "update local objects") + b.append(" fetch(url[, redirect=True]) " + "Fetch URL and update local objects " + "(by default, redirects are followed)") + b.append(" fetch(req) " + "Fetch a scrapy.Request and update local objects ") + b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") return "\n".join("[s] %s" % l for l in b) From f7e4081414d318ddb5297fb98a120e57044dc622 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Dec 2016 22:37:53 +0100 Subject: [PATCH 7/9] Add tests for SequenceExclude container --- tests/test_utils_datatypes.py | 63 ++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b31d2179c..80f797227 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,7 @@ import copy import unittest -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaselessDict, SequenceExclude __doctests__ = ['scrapy.utils.datatypes'] @@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase): assert isinstance(h2, CaselessDict) +class SequenceExcludeTest(unittest.TestCase): + + def test_list(self): + seq = [1, 2, 3] + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn(4, d) + self.assertNotIn(2, d) + + def test_range(self): + seq = range(10, 20) + d = SequenceExclude(seq) + self.assertIn(5, d) + self.assertIn(20, d) + self.assertNotIn(15, d) + + def test_six_range(self): + import six.moves + seq = six.moves.range(10**3, 10**6) + d = SequenceExclude(seq) + self.assertIn(10**2, d) + self.assertIn(10**7, d) + self.assertNotIn(10**4, d) + + def test_range_step(self): + seq = range(10, 20, 3) + d = SequenceExclude(seq) + are_not_in = [v for v in range(10, 20, 3) if v in d] + self.assertEquals([], are_not_in) + + are_not_in = [v for v in range(10, 20) if v in d] + self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + + def test_string_seq(self): + seq = "cde" + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_stringset_seq(self): + seq = set("cde") + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_set(self): + """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" + seq = set([-3, "test", 1.1]) + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn("foo", d) + self.assertIn(3.14, d) + self.assertIn(set("bar"), d) + + # supplied sequence is a set, so checking for list (non)inclusion fails + self.assertRaises(TypeError, (0, 1, 2) in d) + self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c']) + + for v in [-3, "test", 1.1]: + self.assertNotIn(v, d) + if __name__ == "__main__": unittest.main() From 70a69d2199c3c08a7e16f33ea5d35fd4066eb14b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Dec 2016 22:40:48 +0100 Subject: [PATCH 8/9] Use built-in range() --- scrapy/commands/fetch.py | 2 +- scrapy/shell.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 6fe6d73b9..7d4840529 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -56,7 +56,7 @@ class Command(ScrapyCommand): # by default, let the framework handle redirects, # i.e. command handles all codes expect 3xx if not opts.no_redirect: - request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400)) + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) else: request.meta['handle_httpstatus_all'] = True diff --git a/scrapy/shell.py b/scrapy/shell.py index babc267c7..6f94635a1 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,7 +7,6 @@ from __future__ import print_function import os import signal -from six.moves import range import warnings from twisted.internet import reactor, threads, defer From 140a57d7b00f0b044598715ab640bd7e96e32318 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 19 Dec 2016 17:51:30 +0100 Subject: [PATCH 9/9] Amend note on --no-redirect option for shell tool --- docs/topics/commands.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3a26b19ae..6636c30cb 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -382,7 +382,9 @@ Supported options: * ``-c code``: evaluate the code in the shell, print the result and exit -* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them); + this only affects the URL you may pass as argument on the command line; + once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default. Usage example:: @@ -397,6 +399,7 @@ Usage example:: (200, 'http://example.com/') # you can disable this with --no-redirect + # (only for the URL passed as command line argument) $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F')