diff --git a/debian/scrapy-files/scrapy.1 b/debian/scrapy-files/scrapy.1 deleted file mode 100644 index e73ff5459..000000000 --- a/debian/scrapy-files/scrapy.1 +++ /dev/null @@ -1,75 +0,0 @@ -.TH SCRAPY 1 "October 17, 2009" -.SH NAME -scrapy \- the Scrapy command-line tool -.SH SYNOPSIS -.B scrapy -[\fIcommand\fR] [\fIOPTIONS\fR] ... -.SH DESCRIPTION -.PP -Scrapy is controlled through the \fBscrapy\fR command-line tool. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options. -.SH OPTIONS -.SS fetch\fR [\fIOPTION\fR] \fIURL\fR -.TP -Fetch a URL using the Scrapy downloader -.TP -.I --headers -Print response HTTP headers instead of body - -.SS runspider\fR [\fIOPTION\fR] \fIspiderfile\fR -Run a spider -.TP -.I --output=FILE -Store scraped items to FILE in XML format - -.SS settings [\fIOPTION\fR] -Query Scrapy settings -.TP -.I --get=SETTING -Print raw setting value -.TP -.I --getbool=SETTING -Print setting value, intepreted as a boolean -.TP -.I --getint=SETTING -Print setting value, intepreted as an integer -.TP -.I --getfloat=SETTING -Print setting value, intepreted as an float -.TP -.I --getlist=SETTING -Print setting value, intepreted as an float -.TP -.I --init -Print initial setting value (before loading extensions and spiders) - -.SS shell\fR \fIURL\fR | \fIfile\fR -Launch the interactive scraping console - -.SS startproject\fR \fIprojectname\fR -Create new project with an initial project template - -.SS --help, -h -Print command help and options -.SS --logfile=FILE -Log file. if omitted stderr will be used -.SS --loglevel=LEVEL, -L LEVEL -Log level (default: None) -.SS --nolog -Disable logging completely -.SS --spider=SPIDER -Always use this spider when arguments are urls -.SS --profile=FILE -Write python cProfile stats to FILE -.SS --lsprof=FILE -Write lsprof profiling stats to FILE -.SS --pidfile=FILE -Write process ID to FILE -.SS --set=SET -Set/override setting (may be repeated) - -.SH AUTHOR -Scrapy was written by the Scrapy Developers -. -.PP -This manual page was written by Ignace Mouzannar , -for the Debian project (but may be used by others). diff --git a/debian/scrapy.manpages b/debian/scrapy.manpages index baa38f044..4818e9c92 100644 --- a/debian/scrapy.manpages +++ b/debian/scrapy.manpages @@ -1 +1 @@ -debian/scrapy-files/scrapy.1 +extras/scrapy.1 diff --git a/docs/faq.rst b/docs/faq.rst index 67c5241be..0cd058284 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -194,15 +194,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider --set FEED_URI=items.json --set FEED_FORMAT=json + scrapy crawl myspider -s FEED_URI=items.json -s FEED_FORMAT=json To dump into a CSV file:: - scrapy crawl myspider --set FEED_URI=items.csv --set FEED_FORMAT=csv + scrapy crawl myspider -s FEED_URI=items.csv -s FEED_FORMAT=csv To dump into a XML file:: - scrapy crawl myspider --set FEED_URI=items.xml --set FEED_FORMAT=xml + scrapy crawl myspider -s FEED_URI=items.xml -s FEED_FORMAT=xml For more information see :ref:`topics-feed-exports` diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 46fb29bce..db2cafe2d 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -161,7 +161,7 @@ Run the spider to extract the data Finally, we'll run the spider to crawl the site an output file ``scraped_data.json`` with the scraped data in JSON format:: - scrapy crawl mininova.org --set FEED_URI=scraped_data.json --set FEED_FORMAT=json + scrapy crawl mininova.org -s FEED_URI=scraped_data.json -s FEED_FORMAT=json This uses :ref:`feed exports ` to generate the JSON file. You can easily change the export format (XML or CSV, for example) or the diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a1b015bf1..a47b187f0 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -420,7 +420,7 @@ Storing the scraped data The simplest way to store the scraped data is by using the :ref:`Feed exports `, with the following command:: - scrapy crawl dmoz --set FEED_URI=items.json --set FEED_FORMAT=json + scrapy crawl dmoz -s FEED_URI=items.json -s FEED_FORMAT=json That will generate a ``items.json`` file containing all scraped items, serialized in `JSON`_. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 61f1d0b21..e19185a4c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -56,13 +56,13 @@ Example:: >>> settings.overrides['LOG_ENABLED'] = True You can also override one (or more) settings from command line using the -``--set`` command line argument. +``-s`` (or ``--set``) command line option. .. highlight:: sh Example:: - scrapy crawl domain.com --set LOG_FILE=scrapy.log + scrapy crawl domain.com -s LOG_FILE=scrapy.log 2. Project settings module -------------------------- diff --git a/extras/scrapy.1 b/extras/scrapy.1 index e73ff5459..433778be7 100644 --- a/extras/scrapy.1 +++ b/extras/scrapy.1 @@ -64,7 +64,7 @@ Write python cProfile stats to FILE Write lsprof profiling stats to FILE .SS --pidfile=FILE Write process ID to FILE -.SS --set=SET +.SS --set=NAME=VALUE, -s NAME=VALUE Set/override setting (may be repeated) .SH AUTHOR diff --git a/scrapy/command.py b/scrapy/command.py index 37fd50e3e..999bead4d 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -85,7 +85,7 @@ class ScrapyCommand(object): try: settings.overrides.update(arglist_to_dict(opts.set)) except ValueError: - raise UsageError("Invalid --set value, use --set NAME=VALUE", print_help=False) + raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) if opts.logfile: settings.overrides['LOG_ENABLED'] = True diff --git a/scrapy/tests/test_cmdline/__init__.py b/scrapy/tests/test_cmdline/__init__.py index 2fb08e119..9327e262f 100644 --- a/scrapy/tests/test_cmdline/__init__.py +++ b/scrapy/tests/test_cmdline/__init__.py @@ -22,7 +22,7 @@ class CmdlineTest(unittest.TestCase): 'default + loaded + started') def test_override_settings_using_set_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '--set', 'TEST1=override'), \ + self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), \ 'override + loaded + started') def test_override_settings_using_envvar(self): diff --git a/scrapyd/tests/test_utils.py b/scrapyd/tests/test_utils.py index 97652e000..e01311eeb 100644 --- a/scrapyd/tests/test_utils.py +++ b/scrapyd/tests/test_utils.py @@ -25,7 +25,7 @@ class UtilsTest(unittest.TestCase): def test_get_crawl_args_with_settings(self): msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1', 'settings': {'ONE': 'two'}} cargs = get_crawl_args(msg) - self.assertEqual(cargs, ['lala', '-a', 'arg1=val1', '--set', 'ONE=two']) + self.assertEqual(cargs, ['lala', '-a', 'arg1=val1', '-s', 'ONE=two']) assert all(isinstance(x, str) for x in cargs), cargs class GetSpiderListTest(unittest.TestCase): diff --git a/scrapyd/utils.py b/scrapyd/utils.py index 188424d10..4ec5b94da 100644 --- a/scrapyd/utils.py +++ b/scrapyd/utils.py @@ -45,7 +45,7 @@ def get_crawl_args(message): args += ['-a'] args += ['%s=%s' % (k, v)] for k, v in stringify_dict(settings, keys_only=False).items(): - args += ['--set'] + args += ['-s'] args += ['%s=%s' % (k, v)] return args