diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 97f8311de..6644d65e4 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -461,6 +461,9 @@ Supported options: * ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' +* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback. + This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 69418a478..2486f3f23 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,12 +51,13 @@ class Command(ScrapyCommand): help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_option("--cb_kwargs", dest="cb_kwargs", + help="inject extra cb_kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") - @property def max_level(self): levels = list(self.items.keys()) + list(self.requests.keys()) @@ -111,10 +112,11 @@ class Command(ScrapyCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, cb): + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} items, requests = [], [] - for x in iterate_spider_output(cb(response)): + for x in iterate_spider_output(callback(response, **cb_kwargs)): if isinstance(x, (BaseItem, dict)): items.append(x) elif isinstance(x, Request): @@ -142,8 +144,7 @@ class Command(ScrapyCommand): else: self.spidercls = spidercls_for_request(spider_loader, Request(url)) if not self.spidercls: - logger.error('Unable to find spider for: %(url)s', - {'url': url}) + logger.error('Unable to find spider for: %(url)s', {'url': url}) # Request requires callback argument as callable or None, not string request = Request(url, None) @@ -160,7 +161,7 @@ class Command(ScrapyCommand): {'url': url}) def prepare_request(self, spider, request, opts): - def callback(response): + def callback(response, **cb_kwargs): # memorize first request if not self.first_response: self.first_response = response @@ -175,7 +176,7 @@ class Command(ScrapyCommand): if not cb: logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', - {'url': response.url, 'spider': spider.name}) + {'url': response.url, 'spider': spider.name}) return else: cb = 'parse' @@ -192,7 +193,7 @@ class Command(ScrapyCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb) + items, requests = self.run_callback(response, cb, cb_kwargs) if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc for item in items: @@ -207,10 +208,14 @@ class Command(ScrapyCommand): req.callback = callback return requests - #update request meta if any extra meta was passed through the --meta/-m opts. + # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: request.meta.update(opts.meta) + # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option. + if opts.cb_kwargs: + request.cb_kwargs.update(opts.cb_kwargs) + request.meta['_depth'] = 1 request.meta['_callback'] = request.callback request.callback = callback @@ -221,23 +226,29 @@ class Command(ScrapyCommand): self.process_spider_arguments(opts) self.process_request_meta(opts) + self.process_request_cb_kwargs(opts) def process_spider_arguments(self, opts): - try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) def process_request_meta(self, opts): - if opts.meta: try: opts.meta = json.loads(opts.meta) except ValueError: - raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \ - "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " + "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + def process_request_cb_kwargs(self, opts): + if opts.cb_kwargs: + try: + opts.cb_kwargs = json.loads(opts.cb_kwargs) + except ValueError: + raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. " + "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False) def run(self, args, opts): # parse arguments diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 02037b866..1404005fb 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -43,6 +43,12 @@ class MySpider(scrapy.Spider): else: self.logger.debug('It Works!') + def parse_request_with_cb_kwargs(self, response, foo=None, key=None): + if foo == 'bar' and key == 'value': + self.logger.debug('It Works!') + else: + self.logger.debug('It Does Not Work :(') + def parse_request_without_meta(self, response): foo = response.meta.get('foo', 'bar') @@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_request_with_cb_kwargs(self): + raw_json_string = '{"foo" : "bar", "key": "value"}' + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--cb_kwargs', raw_json_string, + '-c', 'parse_request_with_cb_kwargs', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_without_meta(self):