diff --git a/docs/faq.rst b/docs/faq.rst index 3d01ebe25..79197d21d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -234,7 +234,7 @@ higher) in your spider: class MySpider(CrawlSpider): - name = "myspider" + name = "my_spider" download_delay = 2 @@ -257,15 +257,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -O items.json + scrapy crawl my_spider -O items.json To dump into a CSV file:: - scrapy crawl myspider -O items.csv + scrapy crawl my_spider -O items.csv To dump into an XML file:: - scrapy crawl myspider -O items.xml + scrapy crawl my_spider -O items.xml For more information see :ref:`topics-feed-exports` diff --git a/docs/news.rst b/docs/news.rst index aa404d760..edfa3b904 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5104,7 +5104,7 @@ New features ~~~~~~~~~~~~ - Support ``'True'`` and ``'False'`` string values for boolean settings (:issue:`2519`); - you can now do something like ``scrapy crawl myspider -s REDIRECT_ENABLED=False``. + you can now do something like ``scrapy crawl my_spider -s REDIRECT_ENABLED=False``. - Support kwargs with ``response.xpath()`` to use :ref:`XPath variables ` and ad-hoc namespaces declarations ; this requires at least Parsel v1.1 (:issue:`2457`). diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6ffb8ae93..80c0108f2 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -280,14 +280,14 @@ Supported options: Usage examples:: - $ scrapy crawl myspider - [ ... myspider starts crawling ... ] + $ scrapy crawl my_spider + [ ... my_spider starts crawling ... ] - $ scrapy crawl -o myfile:csv myspider - [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] + $ scrapy crawl -o myfile:csv my_spider + [ ... my_spider starts crawling and appends the result to the file myfile in csv format ... ] - $ scrapy crawl -O myfile:json myspider - [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] + $ scrapy crawl -O myfile:json my_spider + [ ... my_spider starts crawling and saves the result in myfile in json format overwriting the original content... ] .. command:: check @@ -563,7 +563,7 @@ project. Example usage:: - $ scrapy runspider myspider.py + $ scrapy runspider my_spider.py [ ... spider starts crawling ... ] .. command:: version diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 988e37bbd..de243adb5 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -15,7 +15,7 @@ Consider the following Scrapy spider below: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" start_urls = ( "http://example.com/page1", "http://example.com/page2", @@ -60,7 +60,7 @@ simple to use, but does not allow debugging code inside a method. In order to see the item scraped from a specific url:: - $ scrapy parse --spider=myspider -c parse_item -d 2 + $ scrapy parse --spider=my_spider -c parse_item -d 2 [ ... scrapy log lines crawling example.com spider ... ] >>> STATUS DEPTH LEVEL 2 <<< @@ -72,7 +72,7 @@ In order to see the item scraped from a specific url:: Using the ``--verbose`` or ``-v`` option we can see the status at each depth level:: - $ scrapy parse --spider=myspider -c parse_item -d 2 -v + $ scrapy parse --spider=my_spider -c parse_item -d 2 -v [ ... scrapy log lines crawling example.com spider ... ] >>> DEPTH LEVEL: 1 <<< @@ -93,7 +93,7 @@ Using the ``--verbose`` or ``-v`` option we can see the status at each depth lev Checking items scraped from a single start_url, can also be easily achieved using:: - $ scrapy parse --spider=myspider -d 3 'http://example.com/page1' + $ scrapy parse --spider=my_spider -d 3 'http://example.com/page1' .. skip: end diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a398d6c83..fee20b516 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -117,7 +117,7 @@ instance, which can be accessed and used like this: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" start_urls = ["https://scrapy.org"] def parse(self, response): @@ -135,7 +135,7 @@ Python logger you want. For example: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" start_urls = ["https://scrapy.org"] def parse(self, response): diff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rst index 62bf2c835..36976f64c 100644 --- a/docs/topics/optimize.rst +++ b/docs/topics/optimize.rst @@ -88,6 +88,39 @@ Lowering memory usage :attr:`~scrapy.Request.callback` cannot yield additional requests. + For example, the following spider uses a higher priority (1) for book + requests than for pagination requests: + + .. code-block:: python + + from scrapy import Spider + + + class BooksToScrapeComSpider(Spider): + name = "books_toscrape_com" + start_urls = [ + "http://books.toscrape.com/catalogue/category/books/mystery_3/index.html" + ] + + def parse(self, response): + next_page_links = response.css(".next a") + yield from response.follow_all(next_page_links) + book_links = response.css("article a") + yield from response.follow_all(book_links, callback=self.parse_book, priority=1) + + def parse_book(self, response): + yield { + "name": response.css("h1::text").get(), + "price": response.css(".price_color::text").re_first("£(.*)"), + "url": response.url, + } + + .. note:: If the number of request-yielding, low-priority requests + scheduled at any given time is lower than concurrency settings + (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or + :setting:`CONCURRENT_REQUESTS`), as in the example above, this can + slow down your crawl by turning those requests into a bottleneck. + - If you have multiple :ref:`start requests `, consider :ref:`lazy ` or :ref:`idle ` scheduling. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0407e4083..452d4b209 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -58,7 +58,7 @@ You can explicitly override one or more settings using the ``-s`` (or Example:: - scrapy crawl myspider -s LOG_LEVEL=INFO -s LOG_FILE=scrapy.log + scrapy crawl my_spider -s LOG_LEVEL=INFO -s LOG_FILE=scrapy.log .. _spider-settings: @@ -82,7 +82,7 @@ attribute: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" custom_settings = { "SOME_SETTING": "some value", @@ -97,7 +97,7 @@ and settings set there should use the ``"spider"`` priority explicitly: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" @classmethod def update_settings(cls, settings): @@ -116,7 +116,7 @@ arguments ` or other logic: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler, *args, **kwargs): @@ -209,7 +209,7 @@ In a spider, settings are available through ``self.settings``: .. code-block:: python class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" start_urls = ["http://example.com"] def parse(self, response): diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4898843e4..41cda4486 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -252,7 +252,7 @@ Here's an example of how you would call it from your spider: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" start_urls = [ "http://example.com", "http://example.org", diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d166bdda3..107fd935a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -176,7 +176,7 @@ scrapy.Spider class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" custom_feed = { "/home/user/documents/items.json": { "format": "json", @@ -301,7 +301,7 @@ functionality of the spider. Spider arguments are passed through the :command:`crawl` command using the ``-a`` option. For example:: - scrapy crawl myspider -a category=electronics + scrapy crawl my_spider -a category=electronics Spiders can access arguments in their `__init__` methods: @@ -311,7 +311,7 @@ Spiders can access arguments in their `__init__` methods: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" def __init__(self, category=None, *args, **kwargs): super(MySpider, self).__init__(*args, **kwargs) @@ -328,7 +328,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): - name = "myspider" + name = "my_spider" async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") @@ -359,7 +359,7 @@ used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` or the user agent used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: - scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot + scrapy crawl my_spider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. @@ -529,7 +529,7 @@ versions, you must define both methods. For example: class MySpider(Spider): - name = "myspider" + name = "my_spider" def start_requests(self): yield Request("https://toscrape.com", headers={"Foo": "Bar"}) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 2a228a3c8..7f355f421 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -93,7 +93,7 @@ class Spider(object_ref): class MySpider(Spider): - name = "myspider" + name = "my_spider" async def start(self): yield Request("https://toscrape.com/") diff --git a/tests/test_addons.py b/tests/test_addons.py index b4294c815..343c5daa2 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -187,7 +187,7 @@ class TestAddonManager(unittest.TestCase): @inlineCallbacks def test_enable_addon_in_spider(self): class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler, *args, **kwargs): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9e66d319c..f391937a2 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -25,7 +25,7 @@ class TestParseCommand(ProcessTest, SiteTest, TestCommandBase): def setUp(self): super().setUp() self.spider_name = "parse_spider" - (self.proj_mod_path / "spiders" / "myspider.py").write_text( + (self.proj_mod_path / "spiders" / "my_spider.py").write_text( f""" import scrapy from scrapy.linkextractors import LinkExtractor diff --git a/tests/test_commands.py b/tests/test_commands.py index 16af97842..9c1523a7e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -662,13 +662,13 @@ Unknown command: abc class TestRunSpiderCommand(TestCommandBase): - spider_filename = "myspider.py" + spider_filename = "my_spider.py" debug_log_spider = """ import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug("It Works!") @@ -740,7 +740,7 @@ class BadSpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' start_urls = ['http://localhost:12345'] def parse(self, response): @@ -752,12 +752,12 @@ class MySpider(scrapy.Spider): def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - assert "[myspider] DEBUG: It Works!" in log1 + assert "[my_spider] DEBUG: It Works!" in log1 assert "[scrapy]" in log1 assert "[scrapy.core.engine]" not in log1 log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - assert "[myspider] DEBUG: It Works!" in log2 + assert "[my_spider] DEBUG: It Works!" in log2 assert "[scrapy]" not in log2 assert "[scrapy.core.engine]" in log2 @@ -770,7 +770,7 @@ class MySpider(scrapy.Spider): assert "File not found: some_non_existent_file" in log def test_runspider_unable_to_load(self): - log = self.get_log("", name="myspider.txt") + log = self.get_log("", name="my_spider.txt") assert "Unable to load" in log def test_start_errors(self): @@ -846,7 +846,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) @@ -855,7 +855,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + assert "[my_spider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log def test_overwrite_output(self): spider_code = """ @@ -863,7 +863,7 @@ import json import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug( @@ -878,7 +878,7 @@ class MySpider(scrapy.Spider): args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + '[my_spider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' in log ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: @@ -890,7 +890,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): return @@ -907,7 +907,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) @@ -916,7 +916,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log + assert "[my_spider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log @skipIf(platform.system() == "Windows", reason="Linux only") def test_absolute_path_linux(self): @@ -924,7 +924,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' start_urls = ["data:,"] @@ -953,7 +953,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' start_urls = ["data:,"] @@ -981,7 +981,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' @classmethod def from_crawler(cls, crawler, *args, **kwargs): @@ -1001,7 +1001,7 @@ class MySpider(scrapy.Spider): class TestWindowsRunSpiderCommand(TestRunSpiderCommand): - spider_filename = "myspider.pyw" + spider_filename = "my_spider.pyw" def setUp(self): if platform.system() != "Windows": @@ -1044,10 +1044,10 @@ class TestViewCommand(TestCommandBase): class TestCrawlCommand(TestCommandBase): def crawl(self, code, args=()): - Path(self.proj_mod_path, "spiders", "myspider.py").write_text( + Path(self.proj_mod_path, "spiders", "my_spider.py").write_text( code, encoding="utf-8" ) - return self.proc("crawl", "myspider", *args) + return self.proc("crawl", "my_spider", *args) def get_log(self, code, args=()): _, _, stderr = self.crawl(code, args=args) @@ -1058,7 +1058,7 @@ class TestCrawlCommand(TestCommandBase): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug('It works!') @@ -1066,14 +1066,14 @@ class MySpider(scrapy.Spider): yield """ log = self.get_log(spider_code) - assert "[myspider] DEBUG: It works!" in log + assert "[my_spider] DEBUG: It works!" in log def test_output(self): spider_code = """ import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) @@ -1082,7 +1082,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + assert "[my_spider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log def test_overwrite_output(self): spider_code = """ @@ -1090,7 +1090,7 @@ import json import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): self.logger.debug( @@ -1105,7 +1105,7 @@ class MySpider(scrapy.Spider): args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + '[my_spider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' in log ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: @@ -1117,7 +1117,7 @@ class MySpider(scrapy.Spider): import scrapy class MySpider(scrapy.Spider): - name = 'myspider' + name = 'my_spider' async def start(self): return diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 42950ea94..f2eb5efcb 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -143,7 +143,7 @@ class TestCrawler(TestBaseCrawler): TrackingDownloaderMiddleware.instances.append(self) class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -197,7 +197,7 @@ class TestCrawler(TestBaseCrawler): @inlineCallbacks def test_get_downloader_middleware_no_engine(self): class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -223,7 +223,7 @@ class TestCrawler(TestBaseCrawler): TrackingExtension.instances.append(self) class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -277,7 +277,7 @@ class TestCrawler(TestBaseCrawler): @inlineCallbacks def test_get_extension_no_engine(self): class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -303,7 +303,7 @@ class TestCrawler(TestBaseCrawler): TrackingItemPipeline.instances.append(self) class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -357,7 +357,7 @@ class TestCrawler(TestBaseCrawler): @inlineCallbacks def test_get_item_pipeline_no_engine(self): class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -383,7 +383,7 @@ class TestCrawler(TestBaseCrawler): TrackingSpiderMiddleware.instances.append(self) class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): @@ -437,7 +437,7 @@ class TestCrawler(TestBaseCrawler): @inlineCallbacks def test_get_spider_middleware_no_engine(self): class MySpider(Spider): - name = "myspider" + name = "my_spider" @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 43e603f6c..893192518 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -5,11 +5,11 @@ from scrapy.utils.spider import iter_spider_classes, iterate_spider_output class MySpider1(Spider): - name = "myspider1" + name = "my_spider1" class MySpider2(Spider): - name = "myspider2" + name = "my_spider2" class TestUtilsSpiders: