diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index f0d435379..58bdb9156 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.exceptions import UsageError + def sanitize_module_name(module_name): """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start @@ -21,6 +22,7 @@ def sanitize_module_name(module_name): module_name = "a" + module_name return module_name + class Command(ScrapyCommand): requires_project = True @@ -52,7 +54,8 @@ class Command(ScrapyCommand): if opts.dump: template_file = self._find_template(opts.dump) if template_file: - print(open(template_file, 'r').read()) + with open(template_file, "r") as f: + print(f.read()) return if len(args) != 2: raise UsageError() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index f90f7cdbc..0185bcabd 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -32,33 +32,34 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, \ + parser.add_option("--spider", dest="spider", default=None, help="use this spider without looking for one") - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") - parser.add_option("--pipelines", action="store_true", \ + parser.add_option("--pipelines", action="store_true", help="process items through pipelines") - parser.add_option("--nolinks", dest="nolinks", action="store_true", \ + parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show links to follow (extracted requests)") - parser.add_option("--noitems", dest="noitems", action="store_true", \ + parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", \ + parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output") - parser.add_option("-r", "--rules", dest="rules", action="store_true", \ + parser.add_option("-r", "--rules", dest="rules", action="store_true", help="use CrawlSpider rules to discover the callback") - parser.add_option("-c", "--callback", dest="callback", \ + parser.add_option("-c", "--callback", dest="callback", help="use this callback for parsing, instead looking for a callback") - parser.add_option("-d", "--depth", dest="depth", type="int", default=1, \ + parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \ + parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") @property def max_level(self): - levels = self.items.keys() + self.requests.keys() - if levels: return max(levels) - else: return 0 + levels = list(self.items.keys()) + list(self.requests.keys()) + if not levels: + return 0 + return max(levels) def add_items(self, lvl, new_items): old_items = self.items.get(lvl, []) @@ -79,7 +80,7 @@ class Command(ScrapyCommand): def print_requests(self, lvl=None, colour=True): if lvl is None: - levels = self.requests.keys() + levels = list(self.requests.keys()) if levels: requests = self.requests[max(levels)] else: @@ -94,7 +95,7 @@ class Command(ScrapyCommand): colour = not opts.nocolour if opts.verbose: - for level in xrange(1, self.max_level+1): + for level in range(1, self.max_level+1): print('\n>>> DEPTH LEVEL: %s <<<' % level) if not opts.noitems: self.print_items(level, colour) @@ -107,7 +108,6 @@ class Command(ScrapyCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, cb): items, requests = [], [] @@ -146,7 +146,6 @@ class Command(ScrapyCommand): _start_requests = lambda s: [self.prepare_request(s, request, opts)] self.spidercls.start_requests = _start_requests - def start_parsing(self, url, opts): self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 90aa8f705..1d7bd006c 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -4,18 +4,20 @@ import os import re import string + def render_templatefile(path, **kwargs): - with open(path, 'rb') as file: - raw = file.read() + with open(path, 'rb') as fp: + raw = fp.read().decode('utf8') content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path - with open(render_path, 'wb') as file: - file.write(content) + with open(render_path, 'wb') as fp: + fp.write(content.encode('utf8')) if path.endswith('.tmpl'): os.remove(path) + CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]') def string_camelcase(string): """ Convert a word to its CamelCase version and remove invalid chars diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index e40b4c73c..55ed75c92 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,7 +1,6 @@ tests/test_closespider.py tests/test_command_fetch.py tests/test_command_shell.py -tests/test_commands.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py @@ -43,5 +42,4 @@ scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py -scrapy/commands/bench.py scrapy/mail.py diff --git a/tests/test_commands.py b/tests/test_commands.py index 7c10faf0c..e0c0648ca 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,10 +6,12 @@ from time import sleep from os.path import exists, join, abspath from shutil import rmtree from tempfile import mkdtemp +import six from twisted.trial import unittest from twisted.internet import defer +from scrapy.utils.python import to_native_str from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest @@ -94,11 +96,11 @@ class GenspiderCommandTest(CommandTest): args = ['--template=%s' % tplname] if tplname else [] spname = 'test_spider' p = self.proc('genspider', spname, 'test.com', *args) - out = retry_on_eintr(p.stdout.read) + out = to_native_str(retry_on_eintr(p.stdout.read)) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) p = self.proc('genspider', spname, 'test.com', *args) - out = retry_on_eintr(p.stdout.read) + out = to_native_str(retry_on_eintr(p.stdout.read)) self.assertIn("Spider %r already exists in module" % spname, out) def test_template_basic(self): @@ -146,7 +148,7 @@ class MySpider(scrapy.Spider): return [] """) p = self.proc('runspider', fname) - log = p.stderr.read() + log = to_native_str(p.stderr.read()) self.assertIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) self.assertIn("INFO: Closing spider (finished)", log) @@ -161,12 +163,12 @@ class MySpider(scrapy.Spider): from scrapy.spiders import Spider """) p = self.proc('runspider', fname) - log = p.stderr.read() + log = to_native_str(p.stderr.read()) self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): p = self.proc('runspider', 'some_non_existent_file') - log = p.stderr.read() + log = to_native_str(p.stderr.read()) self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): @@ -176,11 +178,12 @@ from scrapy.spiders import Spider with open(fname, 'w') as f: f.write("") p = self.proc('runspider', fname) - log = p.stderr.read() + log = to_native_str(p.stderr.read()) self.assertIn("Unable to load", log) class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + skip = not six.PY2 command = 'parse' @@ -226,7 +229,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '-a', 'test_arg=1', '-c', 'parse', self.url('/html')]) - self.assertIn("DEBUG: It Works!", stderr) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) @defer.inlineCallbacks def test_pipelines(self): @@ -234,14 +237,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--pipelines', '-c', 'parse', self.url('/html')]) - self.assertIn("INFO: It Works!", stderr) + self.assertIn("INFO: It Works!", to_native_str(stderr)) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", out) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) @@ -250,5 +253,5 @@ class BenchCommandTest(CommandTest): def test_run(self): p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', '-s', 'CLOSESPIDER_TIMEOUT=0.01') - log = p.stderr.read() + log = to_native_str(p.stderr.read()) self.assertIn('INFO: Crawled', log)