mirror of https://github.com/scrapy/scrapy.git
cleanup and refactor of parse & fetch commands
* removed scrapy.utils.fetch * each command schedule requests and start scrapy engine * fetch command instance BaseSpider if given url does not match any spider or match more than one * parse command schedule url if one spider matches * parse and fetch doesn't support multiple urls as parameter * force spider behavior --spider moved from BaseCommand to only commands: fetch, parse, crawl
This commit is contained in:
Rolando Espinoza La fuente
parent
dd477914db
commit
35a7059636
|
|
@ -20,6 +20,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None, \
|
||||
help="always use this spider when arguments are urls")
|
||||
parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
|
||||
help="don't follow links (for use with URLs only)")
|
||||
|
||||
|
|
@ -29,12 +31,41 @@ class Command(ScrapyCommand):
|
|||
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.spider:
|
||||
spider = spiders.create(opts.spider)
|
||||
else:
|
||||
spider = None
|
||||
|
||||
# aggregate urls and domains
|
||||
urls, domains = self._split_urls_and_domains(args)
|
||||
for dom in domains:
|
||||
scrapymanager.crawl_domain(dom)
|
||||
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = spiders.create(opts.spider)
|
||||
for url in urls:
|
||||
scrapymanager.crawl_url(url, spider)
|
||||
except KeyError:
|
||||
log.msg('Could not find spider: %s' % opts.spider, log.ERROR)
|
||||
else:
|
||||
for name, urls in self._group_urls_by_spider(urls):
|
||||
spider = spiders.create(name)
|
||||
for url in urls:
|
||||
scrapymanager.crawl_url(url, spider)
|
||||
|
||||
scrapymanager.start()
|
||||
|
||||
def _group_urls_by_spider(self, urls):
|
||||
spider_urls = defaultdict(list)
|
||||
for url in urls:
|
||||
spider_names = spiders.find_by_request(Request(url))
|
||||
if not spider_names:
|
||||
log.msg('Could not find spider for url: %s' % url,
|
||||
log.ERROR)
|
||||
elif len(spider_names) > 1:
|
||||
log.msg('More than one spider found for url: %s' % url,
|
||||
log.ERROR)
|
||||
else:
|
||||
spider_urls[spider_names[0]].append(url)
|
||||
return spider_urls.items()
|
||||
|
||||
def _split_urls_and_domains(self, args):
|
||||
urls = []
|
||||
domains = []
|
||||
for arg in args:
|
||||
|
|
@ -42,36 +73,4 @@ class Command(ScrapyCommand):
|
|||
urls.append(arg)
|
||||
else:
|
||||
domains.append(arg)
|
||||
|
||||
# schedule first domains
|
||||
for dom in domains:
|
||||
scrapymanager.crawl_domain(dom)
|
||||
|
||||
# if forced spider schedule urls directly
|
||||
if spider:
|
||||
for url in urls:
|
||||
scrapymanager.crawl_url(url, spider)
|
||||
else:
|
||||
# group urls by spider
|
||||
spider_urls = defaultdict(list)
|
||||
find_by_url = lambda url: spiders.find_by_request(Request(url))
|
||||
for url in urls:
|
||||
spider_names = find_by_url(url)
|
||||
if not spider_names:
|
||||
log.msg('Could not find spider for url: %s' % url,
|
||||
log.ERROR)
|
||||
elif len(spider_names) > 1:
|
||||
log.msg('More than one spider found for url: %s' % url,
|
||||
log.ERROR)
|
||||
else:
|
||||
spider_urls[spider_names[0]].append(url)
|
||||
|
||||
# schedule grouped urls with same spider
|
||||
for name, urls in spider_urls.iteritems():
|
||||
# instance spider for each url-list
|
||||
spider = spiders.create(name)
|
||||
for url in urls:
|
||||
scrapymanager.crawl_url(url, spider)
|
||||
|
||||
# crawl just scheduled arguments without keeping idle
|
||||
scrapymanager.start()
|
||||
return urls, domains
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import pprint
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.utils.fetch import fetch
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import BaseSpider, spiders
|
||||
from scrapy.utils.url import is_url
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
|
|
@ -19,17 +23,33 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true", \
|
||||
help="print response HTTP headers instead of body")
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1:
|
||||
print "One URL is required"
|
||||
return
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
return False
|
||||
responses = [] # to collect downloaded responses
|
||||
request = Request(args[0], callback=responses.append, dont_filter=True)
|
||||
|
||||
responses = fetch(args)
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = spiders.create(opts.spider)
|
||||
except KeyError:
|
||||
log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
|
||||
else:
|
||||
spider = scrapymanager._create_spider_for_request(request, \
|
||||
BaseSpider('default'))
|
||||
|
||||
scrapymanager.crawl_request(request, spider)
|
||||
scrapymanager.start()
|
||||
|
||||
# display response
|
||||
if responses:
|
||||
if opts.headers:
|
||||
pprint.pprint(responses[0].headers)
|
||||
else:
|
||||
print responses[0].body
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.utils.fetch import fetch
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.utils import display
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
from scrapy.utils.url import is_url
|
||||
from scrapy import log
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
|
|
@ -18,6 +22,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None, \
|
||||
help="always use this spider")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true", \
|
||||
help="don't show extracted links")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true", \
|
||||
|
|
@ -37,25 +43,13 @@ class Command(ScrapyCommand):
|
|||
return item
|
||||
|
||||
def run_callback(self, spider, response, callback, args, opts):
|
||||
spider_names = spiders.find_by_request(response.request)
|
||||
if not spider_names:
|
||||
log.msg('Cannot find spider for url: %s' % response.url,
|
||||
level=log.ERROR)
|
||||
return (), ()
|
||||
elif len(spider_names) > 1:
|
||||
log.msg('More than one spider found for url: %s' % response.url,
|
||||
level=log.ERROR)
|
||||
return (), ()
|
||||
else:
|
||||
spider = spiders.create(spider_names[0])
|
||||
|
||||
if callback:
|
||||
callback_fcn = callback if callable(callback) else getattr(spider, callback, None)
|
||||
if not callback_fcn:
|
||||
log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name))
|
||||
return (), ()
|
||||
|
||||
result = callback_fcn(response)
|
||||
result = iterate_spider_output(callback_fcn(response))
|
||||
links = [i for i in result if isinstance(i, Request)]
|
||||
items = [self.pipeline_process(i, spider, opts) for i in result if \
|
||||
isinstance(i, BaseItem)]
|
||||
|
|
@ -78,36 +72,68 @@ class Command(ScrapyCommand):
|
|||
display.pprint(list(links))
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "An URL is required"
|
||||
if not len(args) == 1 or not is_url(args[0]):
|
||||
return False
|
||||
|
||||
request = Request(args[0])
|
||||
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = spiders.create(opts.spider)
|
||||
except KeyError:
|
||||
log.msg('Could not find spider: %s' % opts.spider, log.ERROR)
|
||||
return
|
||||
else:
|
||||
spider = scrapymanager._create_spider_for_request(request, \
|
||||
log_none=True, log_multiple=True)
|
||||
|
||||
if not spider:
|
||||
return
|
||||
|
||||
for response in fetch(args):
|
||||
spider = spiders.fromurl(response.url)
|
||||
if not spider:
|
||||
log.msg('Cannot find spider for "%s"' % response.url)
|
||||
continue
|
||||
responses = [] # to collect downloaded responses
|
||||
request = request.replace(callback=responses.append)
|
||||
|
||||
if self.callbacks:
|
||||
for callback in self.callbacks:
|
||||
items, links = self.run_callback(spider, response, callback, args, opts)
|
||||
self.print_results(items, links, callback, opts)
|
||||
scrapymanager.crawl_request(request, spider)
|
||||
scrapymanager.start()
|
||||
|
||||
elif opts.rules:
|
||||
rules = getattr(spider, 'rules', None)
|
||||
if rules:
|
||||
items, links = [], []
|
||||
for rule in rules:
|
||||
if rule.callback and rule.link_extractor.matches(response.url):
|
||||
items, links = self.run_callback(spider, response, rule.callback, args, opts)
|
||||
self.print_results(items, links, rule.callback, opts)
|
||||
break
|
||||
else:
|
||||
log.msg('No rules found for spider "%s", please specify a callback for parsing' \
|
||||
% spider.domain_name)
|
||||
continue
|
||||
if not responses:
|
||||
log.msg('No response returned', log.ERROR, spider=spider)
|
||||
return
|
||||
|
||||
# now process response
|
||||
# - if callbacks defined then call each one print results
|
||||
# - if --rules option given search for matching spider's rule
|
||||
# - default print result using default 'parse' spider's callback
|
||||
response = responses[0]
|
||||
|
||||
if self.callbacks:
|
||||
# apply each callback
|
||||
for callback in self.callbacks:
|
||||
items, links = self.run_callback(spider, response,
|
||||
callback, args, opts)
|
||||
self.print_results(items, links, callback, opts)
|
||||
elif opts.rules:
|
||||
# search for matching spider's rule
|
||||
if hasattr(spider, 'rules') and spider.rules:
|
||||
items, links = [], []
|
||||
for rule in spider.rules:
|
||||
if rule.link_extractor.matches(response.url) \
|
||||
and rule.callback:
|
||||
|
||||
items, links = self.run_callback(spider,
|
||||
response, rule.callback,
|
||||
args, opts)
|
||||
self.print_results(items, links,
|
||||
rule.callback, opts)
|
||||
# first-match rule breaks rules loop
|
||||
break
|
||||
else:
|
||||
items, links = self.run_callback(spider, response, 'parse', args, opts)
|
||||
self.print_results(items, links, 'parse', opts)
|
||||
log.msg('No rules found for spider "%s", ' \
|
||||
'please specify a callback for parsing' \
|
||||
% spider.domain_name, log.ERROR)
|
||||
else:
|
||||
# default callback 'parse'
|
||||
items, links = self.run_callback(spider, response,
|
||||
'parse', args, opts)
|
||||
self.print_results(items, links, 'parse', opts)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,8 +57,6 @@ class ScrapyCommand(object):
|
|||
help="log level (default: %s)" % settings['LOGLEVEL'])
|
||||
group.add_option("--nolog", action="store_true", dest="nolog", \
|
||||
help="disable logging completely")
|
||||
group.add_option("--spider", dest="spider", default=None, \
|
||||
help="always use this spider when arguments are urls")
|
||||
group.add_option("--profile", dest="profile", metavar="FILE", default=None, \
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
from scrapy.http import Request
|
||||
from scrapy.core.manager import scrapymanager
|
||||
|
||||
def fetch(urls):
|
||||
"""Fetch a list of urls and return a list of the downloaded Scrapy
|
||||
Responses.
|
||||
|
||||
This is a blocking function not suitable for calling from spiders. Instead,
|
||||
it is indended to be called from outside the framework such as Scrapy
|
||||
commands or standalone scripts.
|
||||
"""
|
||||
responses = []
|
||||
for url in urls:
|
||||
req = Request(url, callback=responses.append, dont_filter=True)
|
||||
# @@@ request will require a suitable spider.
|
||||
# If not will not be schedule
|
||||
scrapymanager.crawl_request(req)
|
||||
scrapymanager.start()
|
||||
return responses
|
||||
|
||||
|
|
@ -4,4 +4,3 @@ from scrapy.utils.misc import arg_to_iter
|
|||
|
||||
def iterate_spider_output(result):
|
||||
return [result] if isinstance(result, BaseItem) else arg_to_iter(result)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue