RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PY3 port bench, startproject, genspider, list and runspider commands

This commit is contained in:
Mikhail Korobov 2015-10-12 17:59:42 +05:00
parent 0117c811c9
commit 215905bdb6
5 changed files with 39 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/commands/genspider.py
View File

@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
def sanitize_module_name(module_name):
"""Sanitize the given module name, by replacing dashes and points
with underscores and prefixing it with a letter if it doesn't start
@ -21,6 +22,7 @@ def sanitize_module_name(module_name):
module_name = "a" + module_name
return module_name
class Command(ScrapyCommand):
requires_project = True
@ -52,7 +54,8 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
print(open(template_file, 'r').read())
with open(template_file, "r") as f:
print(f.read())
return
if len(args) != 2:
raise UsageError()

33
scrapy/commands/parse.py
View File

@ -32,33 +32,34 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None, \
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true", \
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true", \
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true", \
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true", \
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true", \
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback", \
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1, \
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@property
def max_level(self):
levels = self.items.keys() + self.requests.keys()
if levels: return max(levels)
else: return 0
levels = list(self.items.keys()) + list(self.requests.keys())
if not levels:
return 0
return max(levels)
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
@ -79,7 +80,7 @@ class Command(ScrapyCommand):
def print_requests(self, lvl=None, colour=True):
if lvl is None:
levels = self.requests.keys()
levels = list(self.requests.keys())
if levels:
requests = self.requests[max(levels)]
else:
@ -94,7 +95,7 @@ class Command(ScrapyCommand):
colour = not opts.nocolour
if opts.verbose:
for level in xrange(1, self.max_level+1):
for level in range(1, self.max_level+1):
print('\n>>> DEPTH LEVEL: %s <<<' % level)
if not opts.noitems:
self.print_items(level, colour)
@ -107,7 +108,6 @@ class Command(ScrapyCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
def run_callback(self, response, cb):
items, requests = [], []
@ -146,7 +146,6 @@ class Command(ScrapyCommand):
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
self.crawler_process.crawl(self.spidercls, **opts.spargs)
self.pcrawler = list(self.crawler_process.crawlers)[0]

10
scrapy/utils/template.py
View File

@ -4,18 +4,20 @@ import os
import re
import string
def render_templatefile(path, **kwargs):
with open(path, 'rb') as file:
raw = file.read()
with open(path, 'rb') as fp:
raw = fp.read().decode('utf8')
content = string.Template(raw).substitute(**kwargs)
render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
with open(render_path, 'wb') as file:
file.write(content)
with open(render_path, 'wb') as fp:
fp.write(content.encode('utf8'))
if path.endswith('.tmpl'):
os.remove(path)
CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]')
def string_camelcase(string):
""" Convert a word to its CamelCase version and remove invalid chars

2
tests/py3-ignores.txt
View File

@ -1,7 +1,6 @@
tests/test_closespider.py
tests/test_command_fetch.py
tests/test_command_shell.py
tests/test_commands.py
tests/test_exporters.py
tests/test_linkextractors_deprecated.py
tests/test_crawl.py
@ -43,5 +42,4 @@ scrapy/downloadermiddlewares/httpproxy.py
scrapy/downloadermiddlewares/cookies.py
scrapy/extensions/statsmailer.py
scrapy/extensions/memusage.py
scrapy/commands/bench.py
scrapy/mail.py

23
tests/test_commands.py
View File

@ -6,10 +6,12 @@ from time import sleep
from os.path import exists, join, abspath
from shutil import rmtree
from tempfile import mkdtemp
import six
from twisted.trial import unittest
from twisted.internet import defer
from scrapy.utils.python import to_native_str
from scrapy.utils.python import retry_on_eintr
from scrapy.utils.test import get_testenv
from scrapy.utils.testsite import SiteTest
@ -94,11 +96,11 @@ class GenspiderCommandTest(CommandTest):
args = ['--template=%s' % tplname] if tplname else []
spname = 'test_spider'
p = self.proc('genspider', spname, 'test.com', *args)
out = retry_on_eintr(p.stdout.read)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
p = self.proc('genspider', spname, 'test.com', *args)
out = retry_on_eintr(p.stdout.read)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("Spider %r already exists in module" % spname, out)
def test_template_basic(self):
@ -146,7 +148,7 @@ class MySpider(scrapy.Spider):
return []
""")
p = self.proc('runspider', fname)
log = p.stderr.read()
log = to_native_str(p.stderr.read())
self.assertIn("DEBUG: It Works!", log)
self.assertIn("INFO: Spider opened", log)
self.assertIn("INFO: Closing spider (finished)", log)
@ -161,12 +163,12 @@ class MySpider(scrapy.Spider):
from scrapy.spiders import Spider
""")
p = self.proc('runspider', fname)
log = p.stderr.read()
log = to_native_str(p.stderr.read())
self.assertIn("No spider found in file", log)
def test_runspider_file_not_found(self):
p = self.proc('runspider', 'some_non_existent_file')
log = p.stderr.read()
log = to_native_str(p.stderr.read())
self.assertIn("File not found: some_non_existent_file", log)
def test_runspider_unable_to_load(self):
@ -176,11 +178,12 @@ from scrapy.spiders import Spider
with open(fname, 'w') as f:
f.write("")
p = self.proc('runspider', fname)
log = p.stderr.read()
log = to_native_str(p.stderr.read())
self.assertIn("Unable to load", log)
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
skip = not six.PY2
command = 'parse'
@ -226,7 +229,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'-a', 'test_arg=1',
'-c', 'parse',
self.url('/html')])
self.assertIn("DEBUG: It Works!", stderr)
self.assertIn("DEBUG: It Works!", to_native_str(stderr))
@defer.inlineCallbacks
def test_pipelines(self):
@ -234,14 +237,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
'--pipelines',
'-c', 'parse',
self.url('/html')])
self.assertIn("INFO: It Works!", stderr)
self.assertIn("INFO: It Works!", to_native_str(stderr))
@defer.inlineCallbacks
def test_parse_items(self):
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", out)
self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
@ -250,5 +253,5 @@ class BenchCommandTest(CommandTest):
def test_run(self):
p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
log = p.stderr.read()
log = to_native_str(p.stderr.read())
self.assertIn('INFO: Crawled', log)