From 97a52665a0b96c8e2b93303dd5b10276ffac1ed2 Mon Sep 17 00:00:00 2001 From: Demelziraptor Date: Fri, 18 Sep 2015 17:16:43 +0900 Subject: [PATCH 01/53] interpreting json-amazonui-streaming as TextResponse --- scrapy/responsetypes.py | 1 + tests/test_responsetypes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 4880cc7b9..319656648 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -25,6 +25,7 @@ class ResponseTypes(object): 'application/xml': 'scrapy.http.XmlResponse', 'application/json': 'scrapy.http.TextResponse', 'application/x-json': 'scrapy.http.TextResponse', + 'application/json-amazonui-streaming': 'scrapy.http.TextResponse', 'application/javascript': 'scrapy.http.TextResponse', 'application/x-javascript': 'scrapy.http.TextResponse', 'text/xml': 'scrapy.http.XmlResponse', diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 2374d518f..b34147b74 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -36,6 +36,7 @@ class ResponseTypesTest(unittest.TestCase): ('application/xml; charset=UTF-8', XmlResponse), ('application/octet-stream', Response), ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse), + ('application/json-amazonui-streaming;charset=UTF-8', TextResponse), ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) From 495d3226912bb84651b5c86183a4adad58c509cc Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Aug 2016 20:16:22 +0500 Subject: [PATCH 02/53] DOC move Data Flow below the picture; add links to components --- docs/topics/architecture.rst | 98 +++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ba0e2c61c..39e54ee99 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -16,20 +16,71 @@ components and an outline of the data flow that takes place inside the system below with links for more detailed information about them. The data flow is also described below. +.. _data-flow: + +Data flow +========= + .. image:: _images/scrapy_architecture_02.png :width: 700 :height: 470 :alt: Scrapy architecture +The data flow in Scrapy is controlled by the execution engine, and goes like +this: + +1. The :ref:`Engine ` gets the first URLs to crawl from the + :ref:`Spider `. + +2. The :ref:`Engine ` schedules the URLs in the + :ref:`Scheduler ` as Requests and asks for the + next URLs to crawl. + +3. The :ref:`Scheduler ` returns the next URLs to crawl + to the :ref:`Engine `. + +4. The :ref:`Engine ` sends the URLs to the + :ref:`Downloader `, passing through the + :ref:`Downloader Middleware ` + (request direction). + +5. Once the page finishes downloading the + :ref:`Downloader ` generates a Response (with + that page) and sends it to the Engine, passing through the + :ref:`Downloader Middleware ` + (response direction). + +6. The :ref:`Engine ` receives the Response from the + :ref:`Downloader ` and sends it to the + :ref:`Spider ` for processing, passing + through the :ref:`Spider Middleware ` + (input direction). + +7. The :ref:`Spider ` processes the Response and returns + scraped items and new Requests (to follow) to the + :ref:`Engine `, passing through the + :ref:`Spider Middleware ` (output direction). + +8. The :ref:`Engine ` sends processed items to + :ref:`Item Pipelines ` and processed Requests to + the :ref:`Scheduler `. + +9. The process repeats (from step 1) until there are no more requests from the + :ref:`Scheduler `. + Components ========== +.. _component-engine: + Scrapy Engine ------------- The engine is responsible for controlling the data flow between all components -of the system, and triggering events when certain actions occur. See the Data -Flow section below for more details. +of the system, and triggering events when certain actions occur. See the +:ref:`Data Flow ` section above for more details. + +.. _component-scheduler: Scheduler --------- @@ -37,12 +88,16 @@ Scheduler The Scheduler receives requests from the engine and enqueues them for feeding them later (also to the engine) when the engine requests them. +.. _component-downloader: + Downloader ---------- The Downloader is responsible for fetching web pages and feeding them to the engine which, in turn, feeds them to the spiders. +.. _component-spiders: + Spiders ------- @@ -50,6 +105,8 @@ Spiders are custom classes written by Scrapy users to parse responses and extract items (aka scraped items) from them or additional URLs (requests) to follow. For more information see :ref:`topics-spiders`. +.. _component-pipelines: + Item Pipeline ------------- @@ -58,6 +115,8 @@ extracted (or scraped) by the spiders. Typical tasks include cleansing, validation and persistence (like storing the item in a database). For more information see :ref:`topics-item-pipeline`. +.. _component-downloader-middleware: + Downloader middlewares ---------------------- @@ -76,6 +135,8 @@ Use a Downloader middleware if you need to do one of the following: For more information see :ref:`topics-downloader-middleware`. +.. _component-spider-middleware: + Spider middlewares ------------------ @@ -93,39 +154,6 @@ Use a Spider middleware if you need to For more information see :ref:`topics-spider-middleware`. -Data flow -========= - -The data flow in Scrapy is controlled by the execution engine, and goes like -this: - -1. The Engine gets the first URLs to crawl from the Spider. - -2. The Engine schedules the URLs in the Scheduler as Requests and asks for the - next URLs to crawl. - -3. The Scheduler returns the next URLs to crawl to the Engine. - -4. The Engine sends the URLs to the Downloader, passing through the - Downloader Middleware (request direction). - -5. Once the page finishes downloading the Downloader generates a Response (with - that page) and sends it to the Engine, passing through the Downloader - Middleware (response direction). - -6. The Engine receives the Response from the Downloader and sends it to the - Spider for processing, passing through the Spider Middleware (input direction). - -7. The Spider processes the Response and returns scraped items and new Requests - (to follow) to the Engine, passing through the Spider Middleware - (output direction). - -8. The Engine sends processed items to Item Pipelines and processed Requests to - the Scheduler. - -9. The process repeats (from step 1) until there are no more requests from the - Scheduler. - Event-driven networking ======================= From 22e870e955995bf789290504292f123f760f5f1b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Sep 2016 10:19:49 +0200 Subject: [PATCH 03/53] Add Debian Jessie test env --- tox.ini | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tox.ini b/tox.ini index f6de64b27..812302b4c 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,20 @@ deps = zope.interface==3.6.1 -rtests/requirements.txt +[testenv:jessie] +# https://packages.debian.org/en/jessie/python/ +# https://packages.debian.org/en/jessie/zope/ +basepython = python2.7 +deps = + pyOpenSSL==0.14 + lxml==3.4.0 + Twisted==14.0.2 + boto==2.34.0 + Pillow==2.6.1 + cssselect==0.9.1 + zope.interface==4.1.1 + -rtests/requirements.txt + [testenv:trunk] basepython = python2.7 commands = From 2b2bfcea88a3b5c98a47adbd5c8d3979e6a5626d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Sep 2016 10:20:49 +0200 Subject: [PATCH 04/53] Add "jessie" build to Travis-CI config --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c58ab39a5..59657b82e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ branches: env: - TOXENV=py27 - TOXENV=precise + - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 - TOXENV=docs From 58cd7bf895321c39493cfd77feb4c07b7e614259 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Sep 2016 11:17:53 +0200 Subject: [PATCH 05/53] Remove "precise" test env from Travis-CI config --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 59657b82e..506f3779b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ branches: - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ env: - TOXENV=py27 - - TOXENV=precise - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 From 9cea6f07308fdbc041601aa35c097d9f6b045d73 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Fri, 2 Sep 2016 14:51:07 -0300 Subject: [PATCH 06/53] Add Segment Analytics to Documentation --- docs/_templates/layout.html | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..a6f6cbda8 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,16 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + +{% endblock %} From 960b1bc8f0335f34db9595532a93d2bef32df9a1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 7 Sep 2016 04:54:32 +0500 Subject: [PATCH 07/53] typo fix in HttpProxyMiddleware --- scrapy/downloadermiddlewares/httpproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index b01bab76d..98c87aa9c 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -43,7 +43,7 @@ class HttpProxyMiddleware(object): return creds, proxy_url def process_request(self, request, spider): - # ignore if proxy is already seted + # ignore if proxy is already set if 'proxy' in request.meta: return From 0ef570f6f0a152d5dc525868b1eb6fef3084823d Mon Sep 17 00:00:00 2001 From: Matti Remes Date: Sat, 3 Sep 2016 17:36:47 +0300 Subject: [PATCH 08/53] Update exceptions.rst Added the missing dot. (+1 squashed commit) Squashed commits: [2198972] Update exceptions.rst There are namely no constructors in classes in Python but an ``__init__`` method instead. --- docs/topics/exceptions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 9f8d16d84..cc02369d4 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -60,7 +60,7 @@ remain disabled. Those components include: * Downloader middlewares * Spider middlewares -The exception must be raised in the component constructor. +The exception must be raised in the component's ``__init__`` method. NotSupported ------------ From 743a0aa422ae2f515cbea69153e60a02c39da98d Mon Sep 17 00:00:00 2001 From: Joakim Uddholm Date: Thu, 8 Sep 2016 21:52:14 +0200 Subject: [PATCH 09/53] Two fixes for when using the parse command and the '-r' flag (rules). 1. Use default "parse" as callback when the matching rule has no callback. 2. Log error and return when no rule matches the parsed url. --- scrapy/commands/parse.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6a8978415..5264982b6 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -121,8 +121,8 @@ class Command(ScrapyCommand): def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): for rule in spider.rules: - if rule.link_extractor.matches(response.url) and rule.callback: - return rule.callback + if rule.link_extractor.matches(response.url): + return rule.callback or "parse" else: logger.error('No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', @@ -166,6 +166,11 @@ class Command(ScrapyCommand): if not cb: if opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) + + if not cb: + logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', + {'url': response.url, 'spider': spider.name}) + return else: cb = 'parse' From 80260824c65512edc3f60aa7eb8fead9818b1a90 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 12 Sep 2016 00:43:58 +0300 Subject: [PATCH 10/53] Fix completion in `scrapy shell` for new imports --- scrapy/utils/console.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index a712df30e..567fd51bc 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -13,7 +13,9 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - shell = InteractiveShellEmbed( + # Always use .instace() to ensure _instance propagation to all parents + # this is needed for completion works well for new imports + shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config) shell() return wrapper From fbb555929977f91eabfa5dc28f3ae3d68972371a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 12 Sep 2016 13:35:14 +0200 Subject: [PATCH 11/53] Add tests for crawl command non-default cases --- tests/test_commands.py | 82 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index d25045cb9..d13024922 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -257,6 +257,9 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): with open(fname, 'w') as f: f.write(""" import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + class MySpider(scrapy.Spider): name = '{0}' @@ -265,6 +268,33 @@ class MySpider(scrapy.Spider): if getattr(self, 'test_arg', None): self.logger.debug('It Works!') return [scrapy.Item(), dict(foo='bar')] + + +class MyGoodCrawlSpider(CrawlSpider): + name = 'goodcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + Rule(LinkExtractor(allow=r'/text'), follow=True), + ) + + def parse_item(self, response): + return [scrapy.Item(), dict(foo='bar')] + + def parse(self, response): + return [scrapy.Item(), dict(nomatch='default')] + + +class MyBadCrawlSpider(CrawlSpider): + '''Spider which doesn't define a parse_item callback while using it in a rule.''' + name = 'badcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + ) + + def parse(self, response): + return [scrapy.Item(), dict(foo='bar')] """.format(self.spider_name)) fname = abspath(join(self.proj_mod_path, 'pipelines.py')) @@ -309,6 +339,58 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} ) self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + @defer.inlineCallbacks + def test_parse_items_no_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_wrong_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_callback_set(self): + """If a rule matches the URL, use it's defined callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_default_callback(self): + """If a rule match but it has no callback set, use the 'parse' callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ) + self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_spider_with_no_rules_attribute(self): + """Using -r with a spider with no rule should not produce items.""" + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_missing_callback(self): + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_no_matching_rule(self): + """The requested URL has no matching rule, so no items should be scraped""" + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") class BenchCommandTest(CommandTest): From 10f8c52f5d3023a9f60fb440b7b1fca3274c29e2 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Fri, 9 Sep 2016 10:35:28 -0300 Subject: [PATCH 12/53] changed tutorial examples from dmoz to quotes.toscrape.com --- docs/intro/tutorial.rst | 301 +++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 160 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6ecd637c3..262d6b3a4 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,7 +7,7 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to use `Open directory project (dmoz) `_ as +We are going to use `quotes.toscrape.com `_ as our example domain to scrape. This tutorial will walk you through these tasks: @@ -16,8 +16,7 @@ This tutorial will walk you through these tasks: 2. Defining the Items you will extract 3. Writing a :ref:`spider ` to crawl a site and extract :ref:`Items ` -4. Writing an :ref:`Item Pipeline ` to store the - extracted Items +4. Exporting the scraped data using command line Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of @@ -54,7 +53,6 @@ This will create a ``tutorial`` directory with the following contents:: spiders/ # a directory where you'll later put your spiders __init__.py - ... Defining our Item @@ -72,16 +70,15 @@ its attributes as :class:`scrapy.Field ` objects, much like i easy task). We begin by modeling the item that we will use to hold the site's data obtained -from dmoz.org. As we want to capture the name, url and description of the -sites, we define fields for each of these three attributes. To do that, we edit +from quotes.toscrape.com. As we want to capture the text and author from each of +the quotes listed there, we define fields for each of these three attributes. To do that, we edit ``items.py``, found in the ``tutorial`` directory. Our Item class looks like this:: import scrapy - class DmozItem(scrapy.Item): - title = scrapy.Field() - link = scrapy.Field() - desc = scrapy.Field() + class QuoteItem(scrapy.Item): + text = scrapy.Field() + author = scrapy.Field() This may seem complicated at first, but defining an item class allows you to use other handy components and helpers within Scrapy. @@ -99,10 +96,11 @@ To create a Spider, you must subclass :class:`scrapy.Spider ` and define some attributes: * :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be - unique, that is, you can't set the same name for different Spiders. + unique within a project, that is, you can't set the same name for different + Spiders. * :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the - Spider will begin to crawl from. The first pages downloaded will be those + Spider will begin to crawl from. The first pages downloaded will be those listed here. The subsequent URLs will be generated successively from data contained in the start URLs. @@ -119,20 +117,20 @@ To create a Spider, you must subclass :class:`scrapy.Spider objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects). This is the code for our first Spider; save it in a file named -``dmoz_spider.py`` under the ``tutorial/spiders`` directory:: +``quotes_spider.py`` under the ``tutorial/spiders`` directory:: import scrapy - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', ] def parse(self, response): - filename = response.url.split("/")[-2] + '.html' + filename = 'quotes-' + response.url.split("/")[-2] + '.html' with open(filename, 'wb') as f: f.write(response.body) @@ -141,24 +139,25 @@ Crawling To put our spider to work, go to the project's top level directory and run:: - scrapy crawl dmoz + scrapy crawl quotes -This command runs the spider with name ``dmoz`` that we've just added, that -will send some requests for the ``dmoz.org`` domain. You will get an output +This command runs the spider with name ``quotes`` that we've just added, that +will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: - 2014-01-23 18:13:07-0400 [scrapy] INFO: Scrapy started (bot: tutorial) - 2014-01-23 18:13:07-0400 [scrapy] INFO: Optional features available: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Overridden settings: {} - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled extensions: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled downloader middlewares: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled spider middlewares: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled item pipelines: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Spider opened - 2014-01-23 18:13:08-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 18:13:09-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 18:13:09-0400 [scrapy] INFO: Closing spider (finished) + 2016-09-01 16:51:27 [scrapy] INFO: Scrapy started (bot: tutorial) + 2016-09-01 16:51:27 [scrapy] INFO: Overridden settings: {...} + 2016-09-01 16:51:27 [scrapy] INFO: Enabled extensions: ... + 2016-09-01 16:51:27 [scrapy] INFO: Enabled downloader middlewares: ... + 2016-09-01 16:51:27 [scrapy] INFO: Enabled spider middlewares: ... + 2016-09-01 16:51:27 [scrapy] INFO: Enabled item pipelines: ... + 2016-09-01 16:51:27 [scrapy] INFO: Spider opened + 2016-09-01 16:51:27 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (404) (referer: None) + 2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-01 16:51:29 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-01 16:51:29 [scrapy] INFO: Closing spider (finished) .. note:: At the end you can see a log line for each URL defined in ``start_urls``. @@ -166,7 +165,7 @@ similar to this:: shown at the end of the log line, where it says ``(referer: None)``. Now, check the files in the current directory. You should notice two new files -have been created: *Books.html* and *Resources.html*, with the content for the respective +have been created: *quotes-1.html* and *quotes-2.html*, with the content for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? @@ -197,15 +196,16 @@ mechanisms see the :ref:`Selectors documentation `. Here are some examples of XPath expressions and their meanings: * ``/html/head/title``: selects the ```` element, inside the ``<head>`` - element of an HTML document + element of an HTML document. Equivalent CSS selector: ``html > head > title``. * ``/html/head/title/text()``: selects the text inside the aforementioned - ``<title>`` element. + ``<title>`` element. Equivalent CSS selector: ``html > head > title ::text``. -* ``//td``: selects all the ``<td>`` elements +* ``//td``: selects all the ``<td>`` elements from the whole document. + Equivalent CSS selector: ``td``. * ``//div[@class="mine"]``: selects all ``div`` elements which contain an - attribute ``class="mine"`` + attribute ``class="mine"``. Equivalent CSS selector: ``div.mine``. These are just a couple of simple examples of what you can do with XPath, but XPath expressions are indeed much more powerful. To learn more about XPath, we @@ -220,7 +220,7 @@ to think in XPath" <http://plasmasturm.org/log/xpath101/>`_. Because of this, we encourage you to learn about XPath even if you already know how to construct CSS selectors. -For working with CSS and XPath expressions, Scrapy provides +For working with CSS and XPath expressions, Scrapy provides the :class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid instantiating selectors yourself every time you need to select something from a response. @@ -255,7 +255,7 @@ installed on your system. To start a shell, you must go to the project's top level directory and run:: - scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/" + scrapy shell "http://quotes.toscrape.com" .. note:: @@ -267,20 +267,20 @@ This is what the shell looks like:: [ ... Scrapy log here ... ] - 2014-01-23 17:11:42-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None) + 2016-09-01 18:14:39 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com> (referer: None) [s] Available Scrapy objects: - [s] crawler <scrapy.crawler.Crawler object at 0x3636b50> + [s] crawler <scrapy.crawler.Crawler object at 0x109001c90> [s] item {} - [s] request <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] settings <scrapy.settings.Settings object at 0x3fadc50> - [s] spider <Spider 'default' at 0x3cebf50> + [s] request <GET http://quotes.toscrape.com> + [s] response <200 http://quotes.toscrape.com> + [s] settings <scrapy.settings.Settings object at 0x109001610> + [s] spider <DefaultSpider 'default' at 0x1092808d0> [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - - In [1]: + + >>> After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body @@ -297,19 +297,19 @@ or ``response.css()`` which map directly to ``response.selector.xpath()`` and So let's try it:: In [1]: response.xpath('//title') - Out[1]: [<Selector xpath='//title' data=u'<title>Open Directory - Computers: Progr'>] - + Out[1]: [<Selector xpath='//title' data=u'<title>Quotes to Scrape'>] + In [2]: response.xpath('//title').extract() - Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - + Out[2]: [u'Quotes to Scrape'] + In [3]: response.xpath('//title/text()') - Out[3]: [] - + Out[3]: [] + In [4]: response.xpath('//title/text()').extract() - Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - - In [5]: response.xpath('//title/text()').re('(\w+):') - Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] + Out[4]: [u'Quotes to Scrape'] + + In [11]: response.xpath('//title/text()').re('(\w+)') + Out[11]: [u'Quotes', u'to', u'Scrape'] Extracting the data ^^^^^^^^^^^^^^^^^^^ @@ -322,35 +322,42 @@ there could become a very tedious task. To make it easier, you can use Firefox Developer Tools or some Firefox extensions like Firebug. For more information see :ref:`topics-firebug` and :ref:`topics-firefox`. -After inspecting the page source, you'll find that the web site's information -is inside a ``
    `` element, in fact the *second* ``
      `` element. +After inspecting the page source, you'll find that every quote in the website +is inside a separate ``
      `` element, such as: -So we can select each ``
    • `` element belonging to the site's list with this -code:: +
      + “We accept the love we think we deserve.” + by Stephen Chbosky +
      + Tags: + + inspirational + love +
      +
      - response.xpath('//ul/li') -And from them, the site's descriptions:: +So we can select each ``
      `` element belonging to the site's +list with this code:: - response.xpath('//ul/li/text()').extract() + response.xpath('//div[@class="quote"]') -The site's titles:: +From each quote block, we can select text with:: - response.xpath('//ul/li/a/text()').extract() + response.xpath('//div[@class="quote"]/span[@class="text"]/text()').extract() -And the site's links:: +The authors:: - response.xpath('//ul/li/a/@href').extract() + response.xpath('//div[@class="quote"]/span/small/text()').extract() As we've said before, each ``.xpath()`` call returns a list of selectors, so we can concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use that property here, so:: - for sel in response.xpath('//ul/li'): - title = sel.xpath('a/text()').extract() - link = sel.xpath('a/@href').extract() - desc = sel.xpath('text()').extract() - print title, link, desc + for quote in response.xpath('//div[@class="quote"]'): + text = quote.xpath('span[@class="text"]/text()').extract() + author = quote.xpath('span/small/text()').extract() + print('{}: {}'.format(author, text)) .. note:: @@ -362,26 +369,25 @@ that property here, so:: Let's add this code to our spider:: import scrapy - - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] - - def parse(self, response): - for sel in response.xpath('//ul/li'): - title = sel.xpath('a/text()').extract() - link = sel.xpath('a/@href').extract() - desc = sel.xpath('text()').extract() - print title, link, desc -Now try crawling dmoz.org again and you'll see sites being printed + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + + def parse(self, response): + for quote in response.xpath('//div[@class="quote"]'): + text = quote.xpath('span[@class="text"]/text()').extract() + author = quote.xpath('span/small/text()').extract() + print('{}: {}'.format(author, text)) + +Now try crawling quotes.toscrape.com again and you'll see sites being printed in your output. Run:: - scrapy crawl dmoz + scrapy crawl quotes Using our item -------------- @@ -389,91 +395,83 @@ Using our item :class:`~scrapy.item.Item` objects are custom Python dicts; you can access the values of their fields (attributes of the class we defined earlier) using the standard dict syntax like:: - - >>> item = DmozItem() - >>> item['title'] = 'Example title' + + >>> from tutorial.items import QuoteItem + >>> item = QuoteItem() + >>> item['text'] = 'Some random quote' >>> item['title'] - 'Example title' + 'Some random quote' So, in order to return the data we've scraped so far, the final code for our Spider would be like this:: import scrapy + from tutorial.items import QuoteItem - from tutorial.items import DmozItem - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', ] def parse(self, response): - for sel in response.xpath('//ul/li'): - item = DmozItem() - item['title'] = sel.xpath('a/text()').extract() - item['link'] = sel.xpath('a/@href').extract() - item['desc'] = sel.xpath('text()').extract() + for quote in response.xpath('//div[@class="quote"]'): + item = QuoteItem() + item['text'] = quote.xpath('span[@class="text"]/text()').extract() + item['author'] = quote.xpath('span/small/text()').extract() yield item -.. note:: You can find a fully-functional variant of this spider in the dirbot_ - project available at https://github.com/scrapy/dirbot -Now crawling dmoz.org yields ``DmozItem`` objects:: +Now crawling quotes.toscrape.com yields ``QuoteItem`` objects:: - [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n], - 'link': [u'http://gnosis.cx/TPiP/'], - 'title': [u'Text Processing in Python']} - [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], - 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], - 'title': [u'XML Processing with Python']} + 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> + {'author': ['Oscar Wilde'], + 'text': ['“We are all in the gutter, but some of us are looking at the ' + 'stars.”']} + 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> + {'author': ['Mark Twain'], + 'text': ['“The man who does not read has no advantage over the man who cannot ' + 'read.”']} Following links =============== -Let's say, instead of just scraping the stuff in *Books* and *Resources* pages, -you want everything that is under the `Python directory -`_. +Let's say, instead of just scraping the stuff from the first two pages +from quotes.toscrape.com, you want quotes from all the pages in the website. -Now that you know how to extract data from a page, why not extract the links -for the pages you are interested, follow them and then extract the data you +Now that you know how to extract data from a page, why not extract the +pagination links in each page, follow them and then extract the data you want for all of them? Here is a modification to our spider that does just that:: import scrapy + from tutorial.items import QuoteItem - from tutorial.items import DmozItem - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/", + 'http://quotes.toscrape.com/page/1/', ] def parse(self, response): - for href in response.css("ul.directory.dir-col > li > a::attr('href')"): - url = response.urljoin(href.extract()) - yield scrapy.Request(url, callback=self.parse_dir_contents) - - def parse_dir_contents(self, response): - for sel in response.xpath('//ul/li'): - item = DmozItem() - item['title'] = sel.xpath('a/text()').extract() - item['link'] = sel.xpath('a/@href').extract() - item['desc'] = sel.xpath('text()').extract() + for quote in response.xpath('//div[@class="quote"]'): + item = QuoteItem() + item['text'] = quote.xpath('span[@class="text"]/text()').extract() + item['author'] = quote.xpath('span/small/text()').extract() yield item + next_page = response.xpath('//li[@class="next"]/a/@href').extract_first() + if next_page: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Now the `parse()` method only extracts the interesting links from the page, +Now after extracting an item the `parse()` method looks for the link to the next page, builds a full absolute URL using the `response.urljoin` method (since the links can -be relative) and yields new requests to be sent later, registering as callback -the method `parse_dir_contents()` that will ultimately scrape the data we want. +be relative) and yields a new request to the next page, registering itself as callback to handle the data extraction for the next page and to keep the crawling going through all the pages. What you see here is Scrapy's mechanism of following links: when you yield a Request in a callback method, Scrapy will schedule that request to be sent @@ -483,25 +481,8 @@ Using this, you can build complex crawlers that follow links according to rules you define, and extract different kinds of data depending on the page it's visiting. -A common pattern is a callback method that extracts some items, looks for a link -to follow to the next page and then yields a `Request` with the same callback -for it:: - - def parse_articles_follow_next_page(self, response): - for article in response.xpath("//article"): - item = ArticleItem() - - ... extract article data here - - yield item - - next_page = response.css("ul.navigation > li.next-page > a::attr('href')") - if next_page: - url = response.urljoin(next_page[0].extract()) - yield scrapy.Request(url, self.parse_articles_follow_next_page) - -This creates a sort of loop, following all the links to the next page until it -doesn't find one -- handy for crawling blogs, forums and other sites with +In our example, it creates a sort of loop, following all the links to the next page +until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. Another common pattern is to build an item with data from more than one page, @@ -521,7 +502,7 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl dmoz -o items.json + scrapy crawl quotes -o items.json That will generate an ``items.json`` file containing all scraped items, serialized in `JSON`_. From 498a3725d18c5f8f5fd63a69ceffe9de9986aa28 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 14 Sep 2016 12:19:50 +0200 Subject: [PATCH 13/53] Add flush() method to StreamLogger Fixes GH-2125 --- scrapy/utils/log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cc2f0b164..a28002c08 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -145,6 +145,9 @@ class StreamLogger(object): for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) + def flush(self): + pass + class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" From bc67cd9edd825abb6c918292cbca25bc76f1e923 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Wed, 14 Sep 2016 12:39:29 -0300 Subject: [PATCH 14/53] fix indentation issue --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 262d6b3a4..f3f1bc645 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -323,7 +323,7 @@ use Firefox Developer Tools or some Firefox extensions like Firebug. For more information see :ref:`topics-firebug` and :ref:`topics-firefox`. After inspecting the page source, you'll find that every quote in the website -is inside a separate ``
      `` element, such as: +is inside a separate ``
      `` element, such as::
      “We accept the love we think we deserve.” From a9a96bed8f5ec9e0e00d678e9524c5c2baf308de Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Wed, 14 Sep 2016 18:09:39 -0300 Subject: [PATCH 15/53] updated tutorial as per review comments --- docs/intro/tutorial.rst | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f3f1bc645..f802c4e49 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -342,7 +342,7 @@ list with this code:: response.xpath('//div[@class="quote"]') -From each quote block, we can select text with:: +From the quote elements, we can select the texts with:: response.xpath('//div[@class="quote"]/span[@class="text"]/text()').extract() @@ -380,9 +380,12 @@ Let's add this code to our spider:: def parse(self, response): for quote in response.xpath('//div[@class="quote"]'): - text = quote.xpath('span[@class="text"]/text()').extract() - author = quote.xpath('span/small/text()').extract() - print('{}: {}'.format(author, text)) + text = quote.xpath('span[@class="text"]/text()').extract_first() + author = quote.xpath('span/small/text()').extract_first() + print(u'{}: {}'.format(author, text)) + +Note how we've changed to use the method ``.extract_first()``, which extracts +the first element from a selector list returned by ``.xpath()``. Now try crawling quotes.toscrape.com again and you'll see sites being printed in your output. Run:: @@ -419,21 +422,19 @@ Spider would be like this:: def parse(self, response): for quote in response.xpath('//div[@class="quote"]'): item = QuoteItem() - item['text'] = quote.xpath('span[@class="text"]/text()').extract() - item['author'] = quote.xpath('span/small/text()').extract() + item['text'] = quote.xpath('span[@class="text"]/text()').extract_first() + item['author'] = quote.xpath('span/small/text()').extract_first() yield item Now crawling quotes.toscrape.com yields ``QuoteItem`` objects:: 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> - {'author': ['Oscar Wilde'], - 'text': ['“We are all in the gutter, but some of us are looking at the ' - 'stars.”']} + {'author': 'Oscar Wilde', + 'text': '“We are all in the gutter, but some of us are looking at the stars.”'} 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> - {'author': ['Mark Twain'], - 'text': ['“The man who does not read has no advantage over the man who cannot ' - 'read.”']} + {'author': 'Mark Twain', + 'text': '“The man who does not read has no advantage over the man who cannot read.”'} Following links @@ -461,8 +462,8 @@ Here is a modification to our spider that does just that:: def parse(self, response): for quote in response.xpath('//div[@class="quote"]'): item = QuoteItem() - item['text'] = quote.xpath('span[@class="text"]/text()').extract() - item['author'] = quote.xpath('span/small/text()').extract() + item['text'] = quote.xpath('span[@class="text"]/text()').extract_first() + item['author'] = quote.xpath('span/small/text()').extract_first() yield item next_page = response.xpath('//li[@class="next"]/a/@href').extract_first() if next_page: From 7d882095437a82085ee62cf55b9a2e1003a92145 Mon Sep 17 00:00:00 2001 From: pawelmhm Date: Thu, 15 Sep 2016 09:30:09 +0200 Subject: [PATCH 16/53] [image & file pipeline] loading setting for user classes if user has some custom subclass of Image pipeline and no setting for this pipeline, he should get default settings defined for Image Pipeline. Fixes #2198 --- scrapy/pipelines/files.py | 3 ++- scrapy/pipelines/images.py | 3 ++- scrapy/pipelines/media.py | 9 ++++++--- tests/test_pipeline_files.py | 25 ++++++++++++++++++++++--- tests/test_pipeline_images.py | 27 ++++++++++++++++++++++----- 5 files changed, 54 insertions(+), 13 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8cdc548f6..843b4d3ec 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -233,7 +233,8 @@ class FilesPipeline(MediaPipeline): cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, - base_class_name=cls_name) + base_class_name=cls_name, + settings=settings) self.expires = settings.getint( resolve('FILES_EXPIRES'), self.EXPIRES ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index af5825c0b..5796bfb80 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -55,7 +55,8 @@ class ImagesPipeline(FilesPipeline): settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, - base_class_name="ImagesPipeline") + base_class_name="ImagesPipeline", + settings=settings) self.expires = settings.getint( resolve("IMAGES_EXPIRES"), self.EXPIRES ) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 82b4b462e..57f70499e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -28,7 +28,8 @@ class MediaPipeline(object): self.download_func = download_func - def _key_for_pipe(self, key, base_class_name=None): + def _key_for_pipe(self, key, base_class_name=None, + settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -38,9 +39,11 @@ class MediaPipeline(object): 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - if class_name == base_class_name or not base_class_name: + formatted_key = "{}_{}".format(class_name.upper(), key) + if class_name == base_class_name or not base_class_name \ + or (settings and not settings.get(formatted_key)): return key - return "{}_{}".format(class_name.upper(), key) + return formatted_key @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bda2a2199..157c21a89 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -255,16 +255,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If file settings are defined but they are not defined for subclass class attributes - should be preserved. + If file settings are defined but they are not defined for subclass + settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) + setting_value = settings.get(settings_attr) self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + self.assertEqual(value, setting_value) def test_no_custom_settings_for_subclasses(self): """ @@ -321,6 +322,24 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(pipeline.files_urls_field, "that") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of FilesPipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(FilesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), + expected_value) + + class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8286582de..6c1976b63 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -309,17 +309,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If image settings are defined but they are not defined for subclass class attributes - should be preserved. + If image settings are defined but they are not defined for subclass default + values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: - # Instance attribute (lowercase) must be equal to class attribute (uppercase). + # Instance attribute (lowercase) must be equal to + # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + setings_value = settings.get(settings_attr) + self.assertEqual(value, setings_value) def test_no_custom_settings_for_subclasses(self): """ @@ -370,11 +372,26 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir})) self.assertEqual(pipeline.images_result_field, "something_else") self.assertEqual(pipeline.images_urls_field, "something") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of ImagePipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(ImagesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr in self.img_cls_attribute_names: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), + expected_value) def _create_image(format, *a, **kw): buf = TemporaryFile() From b828facff499e97f8d2dfec0c2fd5a76a5152237 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 15 Sep 2016 19:25:20 +0200 Subject: [PATCH 17/53] Add shell test for using scrapy.Request() directly without importing scrapy --- tests/test_command_shell.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index c532fc0d8..7bb7439d6 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -56,6 +56,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, _ = yield self.execute(['-c', code.format(url)]) self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks + def test_scrapy_import(self): + url = self.url('/text') + code = "fetch(scrapy.Request('{0}'))" + errcode, out, _ = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks def test_local_file(self): filepath = join(tests_datadir, 'test_site/index.html') From 105163fece502c7afd93d253398cfa6061ef9609 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 15 Sep 2016 19:26:53 +0200 Subject: [PATCH 18/53] Make scrapy available in shell without explicit import statement --- scrapy/shell.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/shell.py b/scrapy/shell.py index 099e1af0a..4f9f06561 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -115,6 +115,9 @@ class Shell(object): self.populate_vars(response, request, spider) def populate_vars(self, response=None, request=None, spider=None): + import scrapy + + self.vars['scrapy'] = scrapy self.vars['crawler'] = self.crawler self.vars['item'] = self.item_class() self.vars['settings'] = self.crawler.settings From 18bd0b0886f6c09fcb03397b88ec5868fd6a9c03 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 15 Sep 2016 15:16:30 -0300 Subject: [PATCH 19/53] docs: update overview spider code to use toscrape.com and minor changes So, this will replace the spider example code from the overview that scrapes questions from StackOverflow by a spider scraping quotes (much like the one in the tutorial), and upates the text around it to be consistent. There are also minor wording changes plus a small Sphinx/reST syntax fix on the features list at the bottom (it was creating a definition list, causing one line to be bold). --- docs/intro/overview.rst | 96 +++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 52 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6c1291c1f..fb390d79d 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -19,74 +19,69 @@ Walk-through of an example spider In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. -So, here's the code for a spider that follows the links to the top -voted questions on StackOverflow and scrapes some data from each page:: +Here's the code for a spider that scrapes famous quotes from website +http://quotes.toscrape.com, following the pagination:: import scrapy - class StackOverflowSpider(scrapy.Spider): - name = 'stackoverflow' - start_urls = ['http://stackoverflow.com/questions?sort=votes'] + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/tag/humor/', + ] def parse(self, response): - for href in response.css('.question-summary h3 a::attr(href)'): - full_url = response.urljoin(href.extract()) - yield scrapy.Request(full_url, callback=self.parse_question) + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } - def parse_question(self, response): - yield { - 'title': response.css('h1 a::text').extract_first(), - 'votes': response.css('.question .vote-count-post::text').extract_first(), - 'body': response.css('.question .post-text').extract_first(), - 'tags': response.css('.question .post-tag::text').extract(), - 'link': response.url, - } + next_page = response.css('li.next a::attr("href")').extract_first() + if next_page: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Put this in a file, name it to something like ``stackoverflow_spider.py`` +Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json + scrapy runspider quotes_spider.py -o quotes.json -When this finishes you will have in the ``top-stackoverflow-questions.json`` file -a list of the most upvoted questions in StackOverflow in JSON format, containing the -title, link, number of upvotes, a list of the tags and the question content in HTML, -looking like this (reformatted for easier reading):: +When this finishes you will have in the ``quotes.json`` file a list of the +quotes in JSON format, containing text and author, looking like this (reformatted +here for better readability):: [{ - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array", - "tags": ["java", "c++", "performance", "optimization"], - "title": "Why is processing a sorted array faster than an unsorted array?", - "votes": "9924" + "author": "Jane Austen", + "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" }, { - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule", - "tags": ["git", "git-submodules"], - "title": "How do I remove a Git submodule?", - "votes": "1764" + "author": "Groucho Marx", + "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" + }, + { + "author": "Steve Martin", + "text": "\u201cA day without sunshine is like, you know, night.\u201d" }, ...] - What just happened? ------------------- -When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a +When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked for a Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for StackOverflow top questions page) +attribute (in this case, only the URL for quotes in *humor* category) and called the default callback method ``parse``, passing the response object as -an argument. In the ``parse`` callback we extract the links to the -question pages using a CSS Selector with a custom extension that allows to get -the value for an attribute. Then we yield a few more requests to be sent, -registering the method ``parse_question`` as the callback to be called for each -of them as they finish. +an argument. In the ``parse`` callback we loop through the quote elements +using a CSS Selector, yield a Python dict with the extracted quote text and author, +look for a link to the next page and schedules another request using the same +``parse`` method as callback. Here you notice one of the main advantages about Scrapy: requests are :ref:`scheduled and processed asynchronously `. This @@ -103,10 +98,6 @@ each request, limiting amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension ` that tries to figure out these automatically. -Finally, the ``parse_question`` callback scrapes the question data for each -page yielding a dict, which Scrapy then collects and writes to a JSON file as -requested in the command line. - .. note:: This is using :ref:`feed exports ` to generate the @@ -145,12 +136,13 @@ scraping easy and efficient, such as: :ref:`pipelines `). * Wide range of built-in extensions and middlewares for handling: - * cookies and session handling - * HTTP features like compression, authentication, caching - * user-agent spoofing - * robots.txt - * crawl depth restriction - * and more + + - cookies and session handling + - HTTP features like compression, authentication, caching + - user-agent spoofing + - robots.txt + - crawl depth restriction + - and more * A :ref:`Telnet console ` for hooking into a Python console running inside your Scrapy process, to introspect and debug your @@ -165,8 +157,8 @@ What's next? ============ The next steps for you are to :ref:`install Scrapy `, -:ref:`follow through the tutorial ` to learn how to organize -your code in Scrapy projects and `join the community`_. Thanks for your +:ref:`follow through the tutorial ` to learn how to create +a full-blown Scrapy project and `join the community`_. Thanks for your interest! .. _join the community: http://scrapy.org/community/ From 1d159ae6f978a4c172fe8e0aab0f8beb74615dc8 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 15 Sep 2016 15:37:03 -0300 Subject: [PATCH 20/53] minor grammar fix --- docs/intro/overview.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index fb390d79d..953fdc968 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -78,9 +78,9 @@ Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` attribute (in this case, only the URL for quotes in *humor* category) and called the default callback method ``parse``, passing the response object as -an argument. In the ``parse`` callback we loop through the quote elements +an argument. In the ``parse`` callback, we loop through the quote elements using a CSS Selector, yield a Python dict with the extracted quote text and author, -look for a link to the next page and schedules another request using the same +look for a link to the next page and schedule another request using the same ``parse`` method as callback. Here you notice one of the main advantages about Scrapy: requests are From effaab867e45d441b5ca99ea6b24fd423e256a33 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 15 Sep 2016 21:37:15 +0200 Subject: [PATCH 21/53] Update shell help with availability of scrapy module --- scrapy/shell.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/shell.py b/scrapy/shell.py index 4f9f06561..183ee1f70 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -139,6 +139,7 @@ class Shell(object): def get_help(self): b = [] b.append("Available Scrapy objects:") + b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) From 75531e409e5aaa00eb3146f4763f6a2a7fb1c59b Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 15 Sep 2016 16:56:13 -0300 Subject: [PATCH 22/53] use better condition in example spider --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 953fdc968..7195017ff 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -39,7 +39,7 @@ http://quotes.toscrape.com, following the pagination:: } next_page = response.css('li.next a::attr("href")').extract_first() - if next_page: + if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) From 2427791287d14f14b846c67dc1edff40a1d2b778 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 15 Sep 2016 17:46:31 -0300 Subject: [PATCH 23/53] tutorial: remove item class definition and present start_requests first This changes the tutorial, removing the step of creating an item class and also starts by presenting the start_requests method instead of start_urls. --- docs/intro/tutorial.rst | 139 +++++++++++++--------------------------- 1 file changed, 43 insertions(+), 96 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f802c4e49..0a3361799 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -13,10 +13,9 @@ our example domain to scrape. This tutorial will walk you through these tasks: 1. Creating a new Scrapy project -2. Defining the Items you will extract -3. Writing a :ref:`spider ` to crawl a site and extract +2. Writing a :ref:`spider ` to crawl a site and extract :ref:`Items ` -4. Exporting the scraped data using command line +3. Exporting the scraped data using command line Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of @@ -55,34 +54,6 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py -Defining our Item -================= - -`Items` are containers that will be loaded with the scraped data; they work -like simple Python dicts. While you can use plain Python dicts with Scrapy, -`Items` provide additional protection against populating undeclared fields, -preventing typos. They can also be used with :ref:`Item Loaders -`, a mechanism with helpers to conveniently populate `Items`. - -They are declared by creating a :class:`scrapy.Item ` class and defining -its attributes as :class:`scrapy.Field ` objects, much like in an ORM -(don't worry if you're not familiar with ORMs, you will see that this is an -easy task). - -We begin by modeling the item that we will use to hold the site's data obtained -from quotes.toscrape.com. As we want to capture the text and author from each of -the quotes listed there, we define fields for each of these three attributes. To do that, we edit -``items.py``, found in the ``tutorial`` directory. Our Item class looks like this:: - - import scrapy - - class QuoteItem(scrapy.Item): - text = scrapy.Field() - author = scrapy.Field() - -This may seem complicated at first, but defining an item class allows you to use other handy -components and helpers within Scrapy. - Our first Spider ================ @@ -93,20 +64,23 @@ They define an initial list of URLs to download, how to follow links, and how to parse the contents of pages to extract :ref:`items `. To create a Spider, you must subclass :class:`scrapy.Spider -` and define some attributes: +` and define some attributes and methods: * :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be unique within a project, that is, you can't set the same name for different Spiders. -* :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the - Spider will begin to crawl from. The first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data - contained in the start URLs. +* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list + of requests where the Spider will begin to crawl from. + Subsequent requests will be generated successively from these initial requests. + + As alternative to defining this method, you can define a class + attribute :attr:`~scrapy.spiders.Spider.start_urls`, which the default + implementation of this method will use to create the proper requests. * :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will be called with the downloaded :class:`~scrapy.http.Response` object of each - start URL. The response is passed to the method as the first and only + initial request. The response is passed to the method as the first and only argument. This method is responsible for parsing the response data and extracting @@ -124,13 +98,16 @@ This is the code for our first Spider; save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', - ] + + def start_requests(self): + base_url = 'http://quotes.toscrape.com' + for path in ['/page/1/', '/page/2/']: + yield scrapy.Request(url=base_url + path, + callback=self.parse) def parse(self, response): - filename = 'quotes-' + response.url.split("/")[-2] + '.html' + page = response.url.split("/")[-2] + filename = 'quotes-%s.html' % page with open(filename, 'wb') as f: f.write(response.body) @@ -171,13 +148,13 @@ URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy creates :class:`scrapy.Request ` objects -for each URL in the ``start_urls`` attribute of the Spider, and assigns -them the ``parse`` method of the spider as their callback function. +Scrapy will schedule the :class:`scrapy.Request ` objects +returned by the ``start_requests`` method of the Spider, and when receiving +a response for each one it will instantiate :class:`scrapy.http.Response` +objects and call the ``parse`` callback method passing the response as argument. -These Requests are scheduled, then executed, and :class:`scrapy.http.Response` -objects are returned and then fed back to the spider, through the -:meth:`~scrapy.spiders.Spider.parse` method. +.. TODO: add here an explanation about how this structure is so command that + we can do a short version of the spider w/ start_urls and default callback Extracting Items ---------------- @@ -355,9 +332,13 @@ concatenate further ``.xpath()`` calls to dig deeper into a node. We are going t that property here, so:: for quote in response.xpath('//div[@class="quote"]'): - text = quote.xpath('span[@class="text"]/text()').extract() - author = quote.xpath('span/small/text()').extract() - print('{}: {}'.format(author, text)) + text = quote.xpath('span[@class="text"]/text()').extract_first() + author = quote.xpath('span/small/text()').extract_first() + print({'text': text, 'author': author}) + +In the above snippet we've decided to use the method ``.extract_first()`` +instead of ``.extract()``, to extract the content from the first element from a +selector list returned by ``.xpath()``. .. note:: @@ -366,7 +347,11 @@ that property here, so:: :ref:`topics-selectors-relative-xpaths` in the :ref:`topics-selectors` documentation -Let's add this code to our spider:: +Knowing to use selectors, extracting data from a page is just a matter of +yield the Python dictionaries from the callback method instead of printing +them. + +Let's add the necessary code to our spider:: import scrapy @@ -380,54 +365,16 @@ Let's add this code to our spider:: def parse(self, response): for quote in response.xpath('//div[@class="quote"]'): - text = quote.xpath('span[@class="text"]/text()').extract_first() - author = quote.xpath('span/small/text()').extract_first() - print(u'{}: {}'.format(author, text)) + yield { + 'text': quote.xpath('span[@class="text"]/text()').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } -Note how we've changed to use the method ``.extract_first()``, which extracts -the first element from a selector list returned by ``.xpath()``. - -Now try crawling quotes.toscrape.com again and you'll see sites being printed -in your output. Run:: +Run:: scrapy crawl quotes -Using our item --------------- - -:class:`~scrapy.item.Item` objects are custom Python dicts; you can access the -values of their fields (attributes of the class we defined earlier) using the -standard dict syntax like:: - - >>> from tutorial.items import QuoteItem - >>> item = QuoteItem() - >>> item['text'] = 'Some random quote' - >>> item['title'] - 'Some random quote' - -So, in order to return the data we've scraped so far, the final code for our -Spider would be like this:: - - import scrapy - from tutorial.items import QuoteItem - - - class QuotesSpider(scrapy.Spider): - name = "quotes" - start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', - ] - - def parse(self, response): - for quote in response.xpath('//div[@class="quote"]'): - item = QuoteItem() - item['text'] = quote.xpath('span[@class="text"]/text()').extract_first() - item['author'] = quote.xpath('span/small/text()').extract_first() - yield item - - -Now crawling quotes.toscrape.com yields ``QuoteItem`` objects:: +Now crawling quotes.toscrape.com will show dictionary objects:: 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> {'author': 'Oscar Wilde', From c508f406892f9d38860fedf1caf8a41fc69bc184 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 15 Sep 2016 18:05:09 -0300 Subject: [PATCH 24/53] use harcoded URLs, remove item reference on second spider --- docs/intro/tutorial.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 0a3361799..d160bfc5c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -100,10 +100,12 @@ This is the code for our first Spider; save it in a file named name = "quotes" def start_requests(self): - base_url = 'http://quotes.toscrape.com' - for path in ['/page/1/', '/page/2/']: - yield scrapy.Request(url=base_url + path, - callback=self.parse) + urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): page = response.url.split("/")[-2] @@ -397,7 +399,6 @@ want for all of them? Here is a modification to our spider that does just that:: import scrapy - from tutorial.items import QuoteItem class QuotesSpider(scrapy.Spider): @@ -408,12 +409,13 @@ Here is a modification to our spider that does just that:: def parse(self, response): for quote in response.xpath('//div[@class="quote"]'): - item = QuoteItem() - item['text'] = quote.xpath('span[@class="text"]/text()').extract_first() - item['author'] = quote.xpath('span/small/text()').extract_first() - yield item + yield { + 'text': quote.xpath('span[@class="text"]/text()').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } + next_page = response.xpath('//li[@class="next"]/a/@href').extract_first() - if next_page: + if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) From 0da497cf7a5063accfabde3acaf2568bfd2b57e4 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Fri, 16 Sep 2016 11:55:23 -0300 Subject: [PATCH 25/53] updates on the first section (our first spider) --- docs/intro/tutorial.rst | 121 +++++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 51 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index d160bfc5c..9ab194865 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,8 +7,8 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to use `quotes.toscrape.com `_ as -our example domain to scrape. +We are going to scrape `quotes.toscrape.com `_, a website +that lists quotes from famous authors. This tutorial will walk you through these tasks: @@ -57,41 +57,13 @@ This will create a ``tutorial`` directory with the following contents:: Our first Spider ================ -Spiders are classes that you define and Scrapy uses to scrape information from a -domain (or group of domains). +Spiders are classes that you define and that Scrapy uses to scrape information +from a website (or group of websites). They define an initial list of URLs to +download, how to follow links, and how to parse the the downloaded page contents +to extract :ref:`items `. -They define an initial list of URLs to download, how to follow links, and how -to parse the contents of pages to extract :ref:`items `. - -To create a Spider, you must subclass :class:`scrapy.Spider -` and define some attributes and methods: - -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be - unique within a project, that is, you can't set the same name for different - Spiders. - -* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list - of requests where the Spider will begin to crawl from. - Subsequent requests will be generated successively from these initial requests. - - As alternative to defining this method, you can define a class - attribute :attr:`~scrapy.spiders.Spider.start_urls`, which the default - implementation of this method will use to create the proper requests. - -* :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will - be called with the downloaded :class:`~scrapy.http.Response` object of each - initial request. The response is passed to the method as the first and only - argument. - - This method is responsible for parsing the response data and extracting - scraped data (as scraped items) and more URLs to follow. - - The :meth:`~scrapy.spiders.Spider.parse` method is in charge of processing - the response and returning scraped data (as :class:`~scrapy.item.Item` - objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects). - -This is the code for our first Spider; save it in a file named -``quotes_spider.py`` under the ``tutorial/spiders`` directory:: +This is the code for our first Spider. Save it in a file named +``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: import scrapy @@ -113,8 +85,29 @@ This is the code for our first Spider; save it in a file named with open(filename, 'wb') as f: f.write(response.body) -Crawling --------- + +As you can see, our Spider subclasses :class:`scrapy.Spider ` +and defines some attributes and methods: + +* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be + unique within a project, that is, you can't set the same name for different + Spiders. + +* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list + of requests where the Spider will begin to crawl from. + Subsequent requests will be generated successively from these initial requests. + +* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle + the response downloaded for each of the requests made. The response parameter + is an instance of :class:`~scrapy.http.Response` that holds the page content and + has further helpful methods to handle it. + + The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + the scraped data as items (:class:`~scrapy.item.Item`) and also finding new URLs to + follow and creating new requests (:class:`~scrapy.http.Request`) from them. + +How to run your spider +---------------------- To put our spider to work, go to the project's top level directory and run:: @@ -138,25 +131,51 @@ similar to this:: 2016-09-01 16:51:29 [scrapy] DEBUG: Crawled (200) (referer: None) 2016-09-01 16:51:29 [scrapy] INFO: Closing spider (finished) -.. note:: - At the end you can see a log line for each URL defined in ``start_urls``. - Because these URLs are the starting ones, they have no referrers, which is - shown at the end of the log line, where it says ``(referer: None)``. +Now, check the files in the current directory. You should notice that two new +files have been created: *quotes-1.html* and *quotes-2.html*, with the content +for the respective URLs, as our ``parse`` method instructs. + +.. note:: If you are wondering why we haven't parsed the HTML yet, hold + on, we will cover that soon. -Now, check the files in the current directory. You should notice two new files -have been created: *quotes-1.html* and *quotes-2.html*, with the content for the respective -URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Scrapy schedules the :class:`scrapy.Request ` objects +returned by the ``start_requests`` method of the Spider. Upon receiving +a response for each one, it instantiates :class:`scrapy.http.Response` +objects and calls the ``parse`` callback method passing the response as +argument. -Scrapy will schedule the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider, and when receiving -a response for each one it will instantiate :class:`scrapy.http.Response` -objects and call the ``parse`` callback method passing the response as argument. -.. TODO: add here an explanation about how this structure is so command that - we can do a short version of the spider w/ start_urls and default callback +Simplifying your spider +----------------------- +Instead of defining the :meth:`~scrapy.spiders.Spider.start_requests` method +generating :class:`scrapy.Request ` +objects from URLs, you can just put those URLs in the +:attr:`~scrapy.spiders.Spider.start_urls` attribute:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + + def parse(self, response): + page = response.url.split("/")[-2] + filename = 'quotes-%s.html' % page + with open(filename, 'wb') as f: + f.write(response.body) + +The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle +each of the requests for those URLs, even though we haven't explicitely told +Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` +is Scrapy's default callback method. + Extracting Items ---------------- From 0cd9dfcc85433b5c09049630fe87f08e9ebd36da Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Fri, 16 Sep 2016 15:21:49 -0300 Subject: [PATCH 26/53] small fixes on tutorial --- docs/intro/tutorial.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 9ab194865..2bbf71573 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -13,8 +13,7 @@ that lists quotes from famous authors. This tutorial will walk you through these tasks: 1. Creating a new Scrapy project -2. Writing a :ref:`spider ` to crawl a site and extract - :ref:`Items ` +2. Writing a :ref:`spider ` to crawl a site and extract data 3. Exporting the scraped data using command line Scrapy is written in Python_. If you're new to the language you might want to @@ -58,9 +57,9 @@ Our first Spider ================ Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or group of websites). They define an initial list of URLs to -download, how to follow links, and how to parse the the downloaded page contents -to extract :ref:`items `. +from a website (or group of websites). They define the initial requests to make, +how to follow links in the pages, and how to parse the downloaded page content +to extract data. This is the code for our first Spider. Save it in a file named ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: @@ -103,7 +102,7 @@ and defines some attributes and methods: has further helpful methods to handle it. The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting - the scraped data as items (:class:`~scrapy.item.Item`) and also finding new URLs to + the scraped data as dicts and also finding new URLs to follow and creating new requests (:class:`~scrapy.http.Request`) from them. How to run your spider From b2a5cddbb01e3970ce1e38e821a76c7d10520845 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 16 Sep 2016 15:44:39 -0300 Subject: [PATCH 27/53] tutorial: update section about following links, expand examples adding an AuthorSpider to demonstrate further a different crawling arrangement. --- docs/intro/tutorial.rst | 78 ++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 13 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 2bbf71573..f3b933d28 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -408,13 +408,13 @@ Following links =============== Let's say, instead of just scraping the stuff from the first two pages -from quotes.toscrape.com, you want quotes from all the pages in the website. +from http://quotes.toscrape.com, you want quotes from all the pages in the website. -Now that you know how to extract data from a page, why not extract the -pagination links in each page, follow them and then extract the data you -want for all of them? +Now that you know how to extract data from pages, let's see how to follow links +from them. -Here is a modification to our spider that does just that:: +Here is a modification of our spider that recursively follows the link to the next +page, extracting data from it:: import scrapy @@ -426,18 +426,19 @@ Here is a modification to our spider that does just that:: ] def parse(self, response): - for quote in response.xpath('//div[@class="quote"]'): + for quote in response.css('div.quote'): yield { - 'text': quote.xpath('span[@class="text"]/text()').extract_first(), - 'author': quote.xpath('span/small/text()').extract_first(), + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), } - next_page = response.xpath('//li[@class="next"]/a/@href').extract_first() + next_page = response.css('li.next a::attr("href")').extract_first() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) -Now after extracting an item the `parse()` method looks for the link to the next page, + +Now, after extracting the data, the `parse()` method looks for the link to the next page, builds a full absolute URL using the `response.urljoin` method (since the links can be relative) and yields a new request to the next page, registering itself as callback to handle the data extraction for the next page and to keep the crawling going through all the pages. @@ -457,13 +458,64 @@ Another common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks `. +Another example: scraping authors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here is another spider that illustrates callbacks and following links, +this time for scraping author information:: + + + import scrapy + + + class AuthorSpider(scrapy.Spider): + name = 'author' + + start_urls = ['http://quotes.toscrape.com/'] + + def parse(self, response): + # follow links to author pages + for href in response.css('.author a::attr("href")').extract(): + yield scrapy.Request(response.urljoin(href), + callback=self.parse_author) + + # follow pagination links + next_page = response.css('li.next a::attr("href")').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) + + def parse_author(self, response): + def extract_with_css(query): + return response.css(query).extract_first().strip() + + yield { + 'name': extract_with_css('h3.author-title::text'), + 'birthdate': extract_with_css('.author-born-date::text'), + 'bio': extract_with_css('.author-description::text'), + } + +This spider will start from the main page, it will follow all the links to the +authors pages calling the ``parse_author`` callback for each of them, and also +the paginations links too with the ``parse`` callback as we saw before. + +The ``parse_author`` callback defines a helper function to extract and cleanup the +data from a CSS query and yields the Python dict with the author data. + +Another interesting this spider demonstrates is that, even if there are many +quotes from the same author, we don't need to worry about visiting the same +page multiple times because Scrapy by default filters out duplicated requests +to URLs already visited, avoiding the problem of hitting servers too much +because of a programming mistake. This can be configured by the setting +:setting:`DUPEFILTER_CLASS`. .. note:: - As an example spider that leverages this mechanism, check out the - :class:`~scrapy.spiders.CrawlSpider` class for a generic spider - that implements a small rules engine that you can use to write your + As another example spider that leverages the mechanism of following links, + check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic + spider that implements a small rules engine that you can use to write your crawlers on top of it. + Storing the scraped data ======================== From 21de617c77ca49d7ab09f8721676c449d793cdb7 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 16 Sep 2016 15:55:14 -0300 Subject: [PATCH 28/53] mention that spiders need to subclass scrapy.Spider --- docs/intro/tutorial.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f3b933d28..62304de2c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -57,9 +57,10 @@ Our first Spider ================ Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or group of websites). They define the initial requests to make, -how to follow links in the pages, and how to parse the downloaded page content -to extract data. +from a website (or group of websites). They must subclass +:class:`scrapy.Spider` and define the initial requests to make, how to follow +links in the pages, and how to parse the downloaded page content to extract +data. This is the code for our first Spider. Save it in a file named ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: From 31260cf02fa60efff42aba0af185f4afe5d18557 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 16 Sep 2016 16:05:36 -0300 Subject: [PATCH 29/53] mentions stackoverflow as help channel (fixes #2255) --- docs/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 4cb3eb741..b4272e47f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,13 +13,15 @@ Having trouble? We'd like to help! * Try the :doc:`FAQ ` -- it's got answers to some common questions. * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`. +* Ask or search questions in `StackOverflow using the scrapy tag`_, * Search for information in the `archives of the scrapy-users mailing list`_, or `post a question`_. -* Ask a question in the `#scrapy IRC channel`_. +* Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. .. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _post a question: https://groups.google.com/forum/#!forum/scrapy-users +.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues From 147e75602d52d66ea0d3f385dd34f9cedcab883e Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 16 Sep 2016 16:47:24 -0300 Subject: [PATCH 30/53] update after review comments (thanks @stummjr) --- docs/intro/tutorial.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 62304de2c..a3a0ab390 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -439,9 +439,11 @@ page, extracting data from it:: yield scrapy.Request(next_page, callback=self.parse) -Now, after extracting the data, the `parse()` method looks for the link to the next page, -builds a full absolute URL using the `response.urljoin` method (since the links can -be relative) and yields a new request to the next page, registering itself as callback to handle the data extraction for the next page and to keep the crawling going through all the pages. +Now, after extracting the data, the ``parse()`` method looks for the link to +the next page, builds a full absolute URL using the ``response.urljoin`` method +(since the links can be relative) and yields a new request to the next page, +registering itself as callback to handle the data extraction for the next page +and to keep the crawling going through all the pages. What you see here is Scrapy's mechanism of following links: when you yield a Request in a callback method, Scrapy will schedule that request to be sent @@ -498,16 +500,16 @@ this time for scraping author information:: This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also -the paginations links too with the ``parse`` callback as we saw before. +the pagination links too with the ``parse`` callback as we saw before. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. -Another interesting this spider demonstrates is that, even if there are many -quotes from the same author, we don't need to worry about visiting the same -page multiple times because Scrapy by default filters out duplicated requests -to URLs already visited, avoiding the problem of hitting servers too much -because of a programming mistake. This can be configured by the setting +Another interesting thing this spider demonstrates is that, even if there are +many quotes from the same author, we don't need to worry about visiting the +same author page multiple times. By default, Scrapy filters out duplicated +requests to URLs already visited, avoiding the problem of hitting servers too +much because of a programming mistake. This can be configured by the setting :setting:`DUPEFILTER_CLASS`. .. note:: From 31545a9f84785edf309eb8a1c7238f46d024cdc2 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Fri, 16 Sep 2016 17:13:24 -0300 Subject: [PATCH 31/53] tutorial: updating extracting data section to introduce CSS and XPath equally --- docs/intro/tutorial.rst | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a3a0ab390..34d33a9b3 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -177,8 +177,8 @@ Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's default callback method. -Extracting Items ----------------- +Extracting data +--------------- Introduction to Selectors ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -191,25 +191,31 @@ mechanisms see the :ref:`Selectors documentation `. .. _XPath: https://www.w3.org/TR/xpath .. _CSS: https://www.w3.org/TR/selectors -Here are some examples of XPath expressions and their meanings: +Here are some examples of XPath expressions, their meanings and CSS +equivalents: * ``/html/head/title``: selects the ```` element, inside the ``<head>`` - element of an HTML document. Equivalent CSS selector: ``html > head > title``. + element of an HTML document. Using CSS, the equivalent would be: ``html > + head > title``. * ``/html/head/title/text()``: selects the text inside the aforementioned - ``<title>`` element. Equivalent CSS selector: ``html > head > title ::text``. + ``<title>`` element. In Scrapy, you can do the same with CSS using ``html > + head > title::text``. The ``::text`` bit isn't really CSS, but is supported + by Scrapy for extracting purposes. * ``//td``: selects all the ``<td>`` elements from the whole document. - Equivalent CSS selector: ``td``. + The equivalent CSS selector: ``td``. -* ``//div[@class="mine"]``: selects all ``div`` elements which contain an - attribute ``class="mine"``. Equivalent CSS selector: ``div.mine``. +* ``//div[@id="mine"]``: selects all ``div`` elements which contain an + attribute ``id="mine"``. The equivalent CSS selector would be: ``div#mine``. -These are just a couple of simple examples of what you can do with XPath, but -XPath expressions are indeed much more powerful. To learn more about XPath, we -recommend `this tutorial to learn XPath through examples -<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how -to think in XPath" <http://plasmasturm.org/log/xpath101/>`_. +These are just a couple of simple examples of what you can do with XPath and +CSS. XPath expressions are very powerful, they're the foundation of Scrapy +selectors. In fact, CSS selectors are converted to XPath expressions +under-the-hood. To learn more about XPath, we recommend `this tutorial to learn +XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this +tutorial to learn "how to think in XPath" +<http://plasmasturm.org/log/xpath101/>`_. .. note:: **CSS vs XPath:** you can go a long way extracting data from web pages using only CSS selectors. However, XPath offers more power because besides From 233b98d642f3c20d47917b54f6944184ee61e0cf Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior <stummjr@gmail.com> Date: Fri, 16 Sep 2016 18:08:10 -0300 Subject: [PATCH 32/53] include section describing spider arguments --- docs/intro/tutorial.rst | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 34d33a9b3..79cf502a6 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -524,6 +524,46 @@ much because of a programming mistake. This can be configured by the setting spider that implements a small rules engine that you can use to write your crawlers on top of it. +Customizing behavior via spider arguments +========================================= +You can provide command line arguments to your spiders by using the ``-a`` +option when running them:: + + scrapy crawl quotes -o items.json -a tag=humor + +In this example, the value provided for the ``tag`` argument will be available +via a spider attribute. Using this, you could make your spider get only quotes +tagged with a specific tag, building the URL based on the argument:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + + def start_requests(self): + url = 'http://quotes.toscrape.com/' + tag = getattr(self, 'tag', None) + if tag is not None: + url = url + 'tag/' + tag + yield scrapy.Request(url) + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small a::text').extract_first(), + } + + next_page = response.css('li.next a::attr("href")').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) + + +If you pass the ``tag=humor`` argument to this spider, you'll notice that it +will only visit URLs from the ``humor`` tag, such as +``http://quotes.toscrape.com/tag/humor``. Storing the scraped data ======================== From 03ab0772491c61cb7f2198b2c019c7eb7672a331 Mon Sep 17 00:00:00 2001 From: Paul Tremberth <paul.tremberth@gmail.com> Date: Sat, 17 Sep 2016 01:36:56 +0200 Subject: [PATCH 33/53] Feed exporter: start exporting only on first item Fixes GH-872 --- scrapy/extensions/feedexport.py | 12 ++++++-- tests/test_feedexport.py | 49 ++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c3fc66de5..85d328528 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -170,6 +170,7 @@ class FeedExporter(object): if not self._exporter_supported(self.format): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') + self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,14 +189,18 @@ class FeedExporter(object): file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, encoding=self.export_encoding) - exporter.start_exporting() + if self.store_empty: + exporter.start_exporting() + self._exporting = True self.slot = SpiderSlot(file, exporter, storage, uri) def close_spider(self, spider): slot = self.slot if not slot.itemcount and not self.store_empty: return - slot.exporter.finish_exporting() + if self._exporting: + slot.exporter.finish_exporting() + self._exporting = False logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" log_args = {'format': self.format, 'itemcount': slot.itemcount, @@ -210,6 +215,9 @@ class FeedExporter(object): def item_scraped(self, item, spider): slot = self.slot + if not self._exporting: + slot.exporter.start_exporting() + self._exporting = True slot.exporter.export_item(item) slot.itemcount += 1 return item diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 353b21927..e93d2bafb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + class TestSpider(scrapy.Spider): + name = 'testspider' + start_urls = ['http://localhost:8998/'] + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, ordered=False) + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + formats = ('json', + 'jsonlines', + 'xml', + 'csv',) + + for fmt in formats: + settings = {'FEED_FORMAT': fmt} + data = yield self.exported_no_data(settings) + self.assertEqual(data, b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[\n\n]'), + ('jsonlines', b''), + ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + data = yield self.exported_no_data(settings) + self.assertEqual(data, expctd) + @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase): def test_export_encoding(self): items = [dict({'foo': u'Test\xd6'})] header = ['foo'] - + formats = { 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - + for format in formats: settings = {'FEED_FORMAT': format} data = yield self.exported_data(items, settings) self.assertEqual(formats[format], data) - + formats = { 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - + for format in formats: settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} data = yield self.exported_data(items, settings) From cc8497abb12df79693d7f2aa63f645e17367d3ba Mon Sep 17 00:00:00 2001 From: Wayne Lovely <wayne.lovely@gmail.com> Date: Sat, 17 Sep 2016 11:09:28 +0000 Subject: [PATCH 34/53] Fix a dict key in the tutorial --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f802c4e49..9233c828e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -402,7 +402,7 @@ standard dict syntax like:: >>> from tutorial.items import QuoteItem >>> item = QuoteItem() >>> item['text'] = 'Some random quote' - >>> item['title'] + >>> item['text'] 'Some random quote' So, in order to return the data we've scraped so far, the final code for our From 48f6a065b8d1ed1f9e45a78643b52874f09e2c30 Mon Sep 17 00:00:00 2001 From: Paul Tremberth <paul.tremberth@gmail.com> Date: Sat, 17 Sep 2016 15:25:45 +0200 Subject: [PATCH 35/53] Flush StreamLogger handlers --- scrapy/utils/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index a28002c08..51f303216 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -146,7 +146,8 @@ class StreamLogger(object): self.logger.log(self.log_level, line.rstrip()) def flush(self): - pass + for h in self.logger.handlers: + h.flush() class LogCounterHandler(logging.Handler): From 8c38dde4e8e60371a2599523aa793a4786dc904e Mon Sep 17 00:00:00 2001 From: Joakim Uddholm <joakim@uddholm.com> Date: Mon, 19 Sep 2016 05:33:05 +0200 Subject: [PATCH 36/53] Moved parse command tests to its own file. Added some checks to check for logged errors. --- tests/test_command_parse.py | 156 ++++++++++++++++++++++++++++++++++++ tests/test_commands.py | 147 --------------------------------- 2 files changed, 156 insertions(+), 147 deletions(-) create mode 100644 tests/test_command_parse.py diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py new file mode 100644 index 000000000..b6d6db9ee --- /dev/null +++ b/tests/test_command_parse.py @@ -0,0 +1,156 @@ +from os.path import join, abspath +from twisted.trial import unittest +from twisted.internet import defer +from scrapy.utils.testsite import SiteTest +from scrapy.utils.testproc import ProcessTest +from scrapy.utils.python import to_native_str +from tests.test_commands import CommandTest + + +class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + command = 'parse' + + def setUp(self): + super(ParseCommandTest, self).setUp() + self.spider_name = 'parse_spider' + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(""" +import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class MySpider(scrapy.Spider): + name = '{0}' + + def parse(self, response): + if getattr(self, 'test_arg', None): + self.logger.debug('It Works!') + return [scrapy.Item(), dict(foo='bar')] + + +class MyGoodCrawlSpider(CrawlSpider): + name = 'goodcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + Rule(LinkExtractor(allow=r'/text'), follow=True), + ) + + def parse_item(self, response): + return [scrapy.Item(), dict(foo='bar')] + + def parse(self, response): + return [scrapy.Item(), dict(nomatch='default')] + + +class MyBadCrawlSpider(CrawlSpider): + '''Spider which doesn't define a parse_item callback while using it in a rule.''' + name = 'badcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + ) + + def parse(self, response): + return [scrapy.Item(), dict(foo='bar')] +""".format(self.spider_name)) + + fname = abspath(join(self.proj_mod_path, 'pipelines.py')) + with open(fname, 'w') as f: + f.write(""" +import logging + +class MyPipeline(object): + component_name = 'my_pipeline' + + def process_item(self, item, spider): + logging.info('It Works!') + return item +""") + + fname = abspath(join(self.proj_mod_path, 'settings.py')) + with open(fname, 'a') as f: + f.write(""" +ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} +""" % self.project_name) + + @defer.inlineCallbacks + def test_spider_arguments(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-a', 'test_arg=1', + '-c', 'parse', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_pipelines(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--pipelines', + '-c', 'parse', + self.url('/html')]) + self.assertIn("INFO: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_parse_items_no_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_wrong_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_callback_set(self): + """If a rule matches the URL, use it's defined callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_default_callback(self): + """If a rule match but it has no callback set, use the 'parse' callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ) + self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_spider_with_no_rules_attribute(self): + """Using -r with a spider with no rule should not produce items.""" + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_missing_callback(self): + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_no_matching_rule(self): + """The requested URL has no matching rule, so no items should be scraped""" + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) diff --git a/tests/test_commands.py b/tests/test_commands.py index d13024922..b507c46bc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -246,153 +246,6 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' - - def setUp(self): - super(ParseCommandTest, self).setUp() - self.spider_name = 'parse_spider' - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(""" -import scrapy -from scrapy.linkextractors import LinkExtractor -from scrapy.spiders import CrawlSpider, Rule - - -class MySpider(scrapy.Spider): - name = '{0}' - - def parse(self, response): - if getattr(self, 'test_arg', None): - self.logger.debug('It Works!') - return [scrapy.Item(), dict(foo='bar')] - - -class MyGoodCrawlSpider(CrawlSpider): - name = 'goodcrawl{0}' - - rules = ( - Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), - Rule(LinkExtractor(allow=r'/text'), follow=True), - ) - - def parse_item(self, response): - return [scrapy.Item(), dict(foo='bar')] - - def parse(self, response): - return [scrapy.Item(), dict(nomatch='default')] - - -class MyBadCrawlSpider(CrawlSpider): - '''Spider which doesn't define a parse_item callback while using it in a rule.''' - name = 'badcrawl{0}' - - rules = ( - Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), - ) - - def parse(self, response): - return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) - - fname = abspath(join(self.proj_mod_path, 'pipelines.py')) - with open(fname, 'w') as f: - f.write(""" -import logging - -class MyPipeline(object): - component_name = 'my_pipeline' - - def process_item(self, item, spider): - logging.info('It Works!') - return item -""") - - fname = abspath(join(self.proj_mod_path, 'settings.py')) - with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) - - @defer.inlineCallbacks - def test_spider_arguments(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-a', 'test_arg=1', - '-c', 'parse', - self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_pipelines(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--pipelines', - '-c', 'parse', - self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_parse_items(self): - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] - ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) - - @defer.inlineCallbacks - def test_parse_items_no_callback_passed(self): - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, self.url('/html')] - ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) - - @defer.inlineCallbacks - def test_wrong_callback_passed(self): - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] - ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - - @defer.inlineCallbacks - def test_crawlspider_matching_rule_callback_set(self): - """If a rule matches the URL, use it's defined callback.""" - status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] - ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) - - @defer.inlineCallbacks - def test_crawlspider_matching_rule_default_callback(self): - """If a rule match but it has no callback set, use the 'parse' callback.""" - status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] - ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) - - @defer.inlineCallbacks - def test_spider_with_no_rules_attribute(self): - """Using -r with a spider with no rule should not produce items.""" - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-r', self.url('/html')] - ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - - @defer.inlineCallbacks - def test_crawlspider_missing_callback(self): - status, out, stderr = yield self.execute( - ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] - ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - - @defer.inlineCallbacks - def test_crawlspider_no_matching_rule(self): - """The requested URL has no matching rule, so no items should be scraped""" - status, out, stderr = yield self.execute( - ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] - ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - - class BenchCommandTest(CommandTest): def test_run(self): From 2a409d1d951d3c968231785cbc1c2c398445888a Mon Sep 17 00:00:00 2001 From: Elias Dorneles <eliasdorneles@gmail.com> Date: Mon, 19 Sep 2016 17:13:04 -0300 Subject: [PATCH 37/53] [wip] changing introduction to scraping with selectors --- docs/intro/tutorial.rst | 291 ++++++++++++---------------------------- 1 file changed, 88 insertions(+), 203 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 79cf502a6..f6aa6476c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -180,235 +180,120 @@ is Scrapy's default callback method. Extracting data --------------- -Introduction to Selectors -^^^^^^^^^^^^^^^^^^^^^^^^^ +The best way to learn how to extract data with Scrapy is trying selectors +using the shell :ref:`Scrapy shell <topics-shell>`. Run:: -There are several ways to extract data from web pages. Scrapy uses a mechanism -based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors -<topics-selectors>`. For more information about selectors and other extraction -mechanisms see the :ref:`Selectors documentation <topics-selectors>`. + scrapy crawl http://quotes.toscrape.com/page/1/ -.. _XPath: https://www.w3.org/TR/xpath -.. _CSS: https://www.w3.org/TR/selectors - -Here are some examples of XPath expressions, their meanings and CSS -equivalents: - -* ``/html/head/title``: selects the ``<title>`` element, inside the ``<head>`` - element of an HTML document. Using CSS, the equivalent would be: ``html > - head > title``. - -* ``/html/head/title/text()``: selects the text inside the aforementioned - ``<title>`` element. In Scrapy, you can do the same with CSS using ``html > - head > title::text``. The ``::text`` bit isn't really CSS, but is supported - by Scrapy for extracting purposes. - -* ``//td``: selects all the ``<td>`` elements from the whole document. - The equivalent CSS selector: ``td``. - -* ``//div[@id="mine"]``: selects all ``div`` elements which contain an - attribute ``id="mine"``. The equivalent CSS selector would be: ``div#mine``. - -These are just a couple of simple examples of what you can do with XPath and -CSS. XPath expressions are very powerful, they're the foundation of Scrapy -selectors. In fact, CSS selectors are converted to XPath expressions -under-the-hood. To learn more about XPath, we recommend `this tutorial to learn -XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this -tutorial to learn "how to think in XPath" -<http://plasmasturm.org/log/xpath101/>`_. - -.. note:: **CSS vs XPath:** you can go a long way extracting data from web pages - using only CSS selectors. However, XPath offers more power because besides - navigating the structure, it can also look at the content: you're - able to select things like: *the link that contains the text 'Next Page'*. - Because of this, we encourage you to learn about XPath even if you - already know how to construct CSS selectors. - -For working with CSS and XPath expressions, Scrapy provides the -:class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid -instantiating selectors yourself every time you need to select something from a -response. - -You can see selectors as objects that represent nodes in the document -structure. So, the first instantiated selectors are associated with the root -node, or the entire document. - -Selectors have four basic methods (click on the method to see the complete API -documentation): - -* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of - which represents the nodes selected by the xpath expression given as - argument. - -* :meth:`~scrapy.selector.Selector.css`: returns a list of selectors, each of - which represents the nodes selected by the CSS expression given as argument. - -* :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the - selected data. - -* :meth:`~scrapy.selector.Selector.re`: returns a list of unicode strings - extracted by applying the regular expression given as argument. - - -Trying Selectors in the Shell -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To illustrate the use of Selectors we're going to use the built-in :ref:`Scrapy -shell <topics-shell>`, which also requires `IPython <http://ipython.org/>`_ (an extended Python console) -installed on your system. - -To start a shell, you must go to the project's top level directory and run:: - - scrapy shell "http://quotes.toscrape.com" - -.. note:: - - Remember to always enclose urls in quotes when running Scrapy shell from - command-line, otherwise urls containing arguments (ie. ``&`` character) - will not work. - -This is what the shell looks like:: +You will see something like:: [ ... Scrapy log here ... ] - - 2016-09-01 18:14:39 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com> (referer: None) + 2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None) [s] Available Scrapy objects: - [s] crawler <scrapy.crawler.Crawler object at 0x109001c90> + [s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90> [s] item {} - [s] request <GET http://quotes.toscrape.com> - [s] response <200 http://quotes.toscrape.com> - [s] settings <scrapy.settings.Settings object at 0x109001610> - [s] spider <DefaultSpider 'default' at 0x1092808d0> + [s] request <GET http://quotes.toscrape.com/page/1/> + [s] response <200 http://quotes.toscrape.com/page/1/> + [s] settings <scrapy.settings.Settings object at 0x7fa91d888c10> + [s] spider <DefaultSpider 'default' at 0x7fa91c8af990> [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - - >>> + >>> -After the shell loads, you will have the response fetched in a local -``response`` variable, so if you type ``response.body`` you will see the body -of the response, or you can type ``response.headers`` to see its headers. +Using the shell, you can try selecting elements using `CSS`_ with the response +object:: -More importantly ``response`` has a ``selector`` attribute which is an instance of -:class:`~scrapy.selector.Selector` class, instantiated with this particular ``response``. -You can run queries on ``response`` by calling ``response.selector.xpath()`` or -``response.selector.css()``. There are also some convenience shortcuts like ``response.xpath()`` -or ``response.css()`` which map directly to ``response.selector.xpath()`` and -``response.selector.css()``. + >>> response.css('title') + [<Selector xpath=u'descendant-or-self::title' data=u'<title>Quotes to Scrape'>] +The result of running ``response.css('title')`` is a list-like object called +:class:`~scrapy.selector.SelectorList`, which represents a list of +:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +and allow you to run further queries to fine-grain the selection or extract the +data. -So let's try it:: +To extract the text from the title above, you can do:: - In [1]: response.xpath('//title') - Out[1]: [Quotes to Scrape'>] - - In [2]: response.xpath('//title').extract() - Out[2]: [u'Quotes to Scrape'] - - In [3]: response.xpath('//title/text()') - Out[3]: [] + >>> response.css('title::text').extract() + [u'Quotes to Scrape'] - In [4]: response.xpath('//title/text()').extract() - Out[4]: [u'Quotes to Scrape'] - - In [11]: response.xpath('//title/text()').re('(\w+)') - Out[11]: [u'Quotes', u'to', u'Scrape'] +There are two things to note here: one is that we've added ``::text`` to the +CSS query, to mean that we want to select the text from inside the title element. -Extracting the data -^^^^^^^^^^^^^^^^^^^ +The other is that the result of calling ``.extract()`` is a list, because we're +dealing with an instance :class:`~scrapy.selector.SelectorList`. When you know +you just want the first result, as in this case, you can do:: -Now, let's try to extract some real information from those pages. + >>> response.css('title::text').extract_first() + u'Quotes to Scrape' -You could type ``response.body`` in the console, and inspect the source code to -figure out the XPaths you need to use. However, inspecting the raw HTML code -there could become a very tedious task. To make it easier, you can -use Firefox Developer Tools or some Firefox extensions like Firebug. For more +As an alternative, you could've written:: + + >>> response.css('title::text')[0].extract() + u'Quotes to Scrape' + +However, using ``.extract_first()`` avoids an ``IndexError`` and returns +``None`` when it doesn't find any element matching the selection. + +There's a lesson here: for most scraping code, you want it to be resilient to +errors due to things not being found on a page, so that even if some parts fail +to be scraped, you can at least get **some** data. + +Besides the :meth:`~scrapy.selector.Selector.extract` and +:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use +the :meth:`~scrapy.selector.Selector.re` method to extract using a regular +expression:: + + >>> response.css('title::text').re('Quotes.*') + [u'Quotes to Scrape'] + >>> response.css('title::text').re('Q\w+') + [u'Quotes'] + >>> response.css('title::text').re('(\w+) to (\w+)') + [u'Quotes', u'Scrape'] + +In order to find the proper CSS selectors to use, you might find useful opening +the response page from the shell in your web browser using ``view(response)``. +You can use your browser developer tools or extensions like Firebug. For more information see :ref:`topics-firebug` and :ref:`topics-firefox`. -After inspecting the page source, you'll find that every quote in the website -is inside a separate ``
      `` element, such as:: -
      - “We accept the love we think we deserve.” - by Stephen Chbosky -
      - Tags: - - inspirational - love -
      -
      +XPath: a brief intro +^^^^^^^^^^^^^^^^^^^^ + +Besides CSS, Scrapy selectors also support using `XPath`_ expressions:: + + >>> response.xpath('//title') + [Quotes to Scrape'>] + >>> response.xpath('//title/text()').extract_first() + u'Quotes to Scrape' + +XPath expressions are very powerful, and are the foundation of Scrapy +Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You +can see that if you read closely the text representation of the selector +objects in the shell. + +While perhaps not as popular as CSS selectors, XPath expressions offer more +power because besides navigating the structure, it can also look at the +content. Using XPath, you're able to select things like: **select the link +that contains the text "Next Page"**. This makes XPath very fitting to the task +of scraping, and we encourage you to learn XPath even if you already know how to +construct CSS selectors, it will make scraping much easier. + +We won't cover much of XPath here. To learn more about XPath, we recommend `this tutorial to learn +XPath through examples `_, and `this +tutorial to learn "how to think in XPath" +`_. -So we can select each ``
      `` element belonging to the site's -list with this code:: +Extraction wrap-up +^^^^^^^^^^^^^^^^^^ - response.xpath('//div[@class="quote"]') +Now that you know a bit about selection and extraction, let's complete our +spider by writing the code to extract the quotes from the webpage. -From the quote elements, we can select the texts with:: - - response.xpath('//div[@class="quote"]/span[@class="text"]/text()').extract() - -The authors:: - - response.xpath('//div[@class="quote"]/span/small/text()').extract() - -As we've said before, each ``.xpath()`` call returns a list of selectors, so we can -concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use -that property here, so:: - - for quote in response.xpath('//div[@class="quote"]'): - text = quote.xpath('span[@class="text"]/text()').extract_first() - author = quote.xpath('span/small/text()').extract_first() - print({'text': text, 'author': author}) - -In the above snippet we've decided to use the method ``.extract_first()`` -instead of ``.extract()``, to extract the content from the first element from a -selector list returned by ``.xpath()``. - -.. note:: - - For a more detailed description of using nested selectors, see - :ref:`topics-selectors-nesting-selectors` and - :ref:`topics-selectors-relative-xpaths` in the :ref:`topics-selectors` - documentation - -Knowing to use selectors, extracting data from a page is just a matter of -yield the Python dictionaries from the callback method instead of printing -them. - -Let's add the necessary code to our spider:: - - import scrapy - - - class QuotesSpider(scrapy.Spider): - name = "quotes" - start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', - ] - - def parse(self, response): - for quote in response.xpath('//div[@class="quote"]'): - yield { - 'text': quote.xpath('span[@class="text"]/text()').extract_first(), - 'author': quote.xpath('span/small/text()').extract_first(), - } - -Run:: - - scrapy crawl quotes - -Now crawling quotes.toscrape.com will show dictionary objects:: - - 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> - {'author': 'Oscar Wilde', - 'text': '“We are all in the gutter, but some of us are looking at the stars.”'} - 2016-09-02 16:35:20 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/2/> - {'author': 'Mark Twain', - 'text': '“The man who does not read has no advantage over the man who cannot read.”'} +TODO: show how to extract quotes and integrate spider code here. Following links From fee07835f2f9504acc7f0952088ca2ea201027c3 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 19 Sep 2016 19:19:31 -0300 Subject: [PATCH 38/53] Completing the data extraction section --- docs/intro/tutorial.rst | 122 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 112 insertions(+), 10 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f6aa6476c..473183be3 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -106,8 +106,8 @@ and defines some attributes and methods: the scraped data as dicts and also finding new URLs to follow and creating new requests (:class:`~scrapy.http.Request`) from them. -How to run your spider ----------------------- +How to run our spider +--------------------- To put our spider to work, go to the project's top level directory and run:: @@ -148,12 +148,14 @@ objects and calls the ``parse`` callback method passing the response as argument. -Simplifying your spider ------------------------ -Instead of defining the :meth:`~scrapy.spiders.Spider.start_requests` method -generating :class:`scrapy.Request ` -objects from URLs, you can just put those URLs in the -:attr:`~scrapy.spiders.Spider.start_urls` attribute:: +A shortcut to the start_requests method +--------------------------------------- +Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method +that generates :class:`scrapy.Request ` objects from URLs, +you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +with a list of URLs. This list will then be used by the default implementation +of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +for your spider:: import scrapy @@ -174,7 +176,8 @@ objects from URLs, you can just put those URLs in the The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitely told Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` -is Scrapy's default callback method. +is Scrapy's default callback method that is called for any request that have +been generated with no callback explicitely assigned to handle it. Extracting data @@ -293,7 +296,104 @@ Extraction wrap-up Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the webpage. -TODO: show how to extract quotes and integrate spider code here. +Each quote in http://quotes.toscrape.com is represented by HTML code that looks +like this:: + +
      + “The world as we have created it is a process of our + thinking. It cannot be changed without changing our thinking.” + + by Albert Einstein + (about) + +
      + Tags: + change + deep-thoughts + thinking + world +
      +
      + +Let's open up scrapy shell and play a bit to find out how to extract the data +we want:: + + $ scrapy shell http://quotes.toscrape.com + +We get a list of selectors to the quotes using:: + + >>> response.css("div.quote") + +Each of the selectors returned by the query above allows us to run further +queries over the quotes itselves. Let's assign the first selector to a +variable, so that we can run our CSS selectors directly on a particular quote:: + + >>> quote = response.css("div.quote")[0] + +Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +using the ``quote`` object we just created:: + + >>> title = quote.css("span.text ::text").extract_first() + >>> title + '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' + >>> author = quote.css("small.author ::text").extract_first() + >>> author + 'Albert Einstein' + +Given that the tags is a list of strings, we can use the ``.extract()`` method +to get all of them:: + + >>> tags = quote.css("div.tags a.tag ::text").extract() + >>> tags + ['change', 'deep-thoughts', 'thinking', 'world'] + +Now, we can iterate over all the quotes in the page and use the CSS selectors +we defined to extract data:: + + >>> for quote in response.css("div.quote"): + ... text = quote.css("span.text ::text").extract_first() + ... author = quote.css("small.author ::text").extract_first() + ... tags = quote.css("div.tags a.tag ::text").extract() + ... print("{} - {} - {}".format(text, author, tags)) + + +Extracting data in our spider +------------------------------ + +Until now, the spider we built doesn't extract any data in particular. I just +saves the whole HTML page to a local file. Now, let's integrate the extraction +logic above in our spider. + +A Scrapy spider typically generates many dictionaries containing the data +extracted from the page. To do that, we use the ``yield`` Python keyword, as +you can see below:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css("div.tags a.tag ::text").extract(), + } + +If you run this spider, it will output the extracted data with the log:: + + 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} + 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} + +:ref:`Later in the tutorial `, we will see how to save this data to a file. Following links @@ -450,6 +550,8 @@ If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as ``http://quotes.toscrape.com/tag/humor``. +.. _storing-data: + Storing the scraped data ======================== From a135dbaf19f45b9cf59b4e89954f93c74bdf7771 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Sep 2016 12:47:33 +0200 Subject: [PATCH 39/53] Log warning when request cannot be serialized (instead of error) Fixes GH-2035 --- scrapy/core/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index dcd6fb989..a54b4daf0 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -89,8 +89,8 @@ class Scheduler(object): msg = ("Unable to serialize request: %(request)s - reason:" " %(reason)s - no more unserializable requests will be" " logged (stats being collected)") - logger.error(msg, {'request': request, 'reason': e}, - exc_info=True, extra={'spider': self.spider}) + logger.warning(msg, {'request': request, 'reason': e}, + exc_info=True, extra={'spider': self.spider}) self.logunser = False self.stats.inc_value('scheduler/unserializable', spider=self.spider) From f4f93c5c266648317ff2c2e474b7d5fd08918c07 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 20 Sep 2016 09:19:48 -0300 Subject: [PATCH 40/53] fix tox docs build, adjust title --- docs/intro/tutorial.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 473183be3..6a5e99d7a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -265,7 +265,7 @@ information see :ref:`topics-firebug` and :ref:`topics-firefox`. XPath: a brief intro ^^^^^^^^^^^^^^^^^^^^ -Besides CSS, Scrapy selectors also support using `XPath`_ expressions:: +Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: >>> response.xpath('//title') [Quotes to Scrape'>] @@ -289,6 +289,8 @@ XPath through examples `_, and `this tutorial to learn "how to think in XPath" `_. +.. _XPath: https://www.w3.org/TR/xpath +.. _CSS: https://www.w3.org/TR/selectors Extraction wrap-up ^^^^^^^^^^^^^^^^^^ @@ -453,7 +455,7 @@ using a :ref:`trick to pass additional data to the callbacks `. Another example: scraping authors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: @@ -509,8 +511,9 @@ much because of a programming mistake. This can be configured by the setting spider that implements a small rules engine that you can use to write your crawlers on top of it. -Customizing behavior via spider arguments -========================================= +Adding a spider argument +======================== + You can provide command line arguments to your spiders by using the ``-a`` option when running them:: From e59d79bf37a8ecbef6679145815b000008bb6624 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Sep 2016 17:17:22 +0200 Subject: [PATCH 41/53] Add note on "to" and "cc" as lists for sending emails Fixes GH-2244 --- docs/topics/email.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 62ebc4c08..18d2f8084 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -35,6 +35,12 @@ And here is how to use it to send an e-mail (without attachments):: mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"]) +.. note:: + As shown in the example above, ``to`` and ``cc`` need to be lists + of email addresses, not single addresses, and even for one recipient, + i.e. ``to="someone@example.com"`` will not work. + + MailSender class reference ========================== From 125b691102320864c635608618636253074eae1a Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 20 Sep 2016 12:47:03 -0300 Subject: [PATCH 42/53] more reviewing and editing, minor restructure, syntax fixes --- docs/intro/tutorial.rst | 141 ++++++++++++++++++++++++---------------- 1 file changed, 84 insertions(+), 57 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6a5e99d7a..b4a5e3b48 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -14,7 +14,9 @@ This tutorial will walk you through these tasks: 1. Creating a new Scrapy project 2. Writing a :ref:`spider ` to crawl a site and extract data -3. Exporting the scraped data using command line +3. Exporting the scraped data using the command line +4. Change spider to recursively follow links +5. Using spider arguments Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of @@ -43,7 +45,7 @@ This will create a ``tutorial`` directory with the following contents:: tutorial/ # project's Python module, you'll import your code from here __init__.py - items.py # project items file + items.py # project items definition file pipelines.py # project pipelines file @@ -109,7 +111,8 @@ and defines some attributes and methods: How to run our spider --------------------- -To put our spider to work, go to the project's top level directory and run:: +To put our spider to work, go to the project's top level directory (``cd +tutorial``) and run:: scrapy crawl quotes @@ -141,6 +144,7 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`scrapy.http.Response` @@ -173,11 +177,11 @@ for your spider:: with open(filename, 'wb') as f: f.write(response.body) -The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle -each of the requests for those URLs, even though we haven't explicitely told -Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` -is Scrapy's default callback method that is called for any request that have -been generated with no callback explicitely assigned to handle it. +The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +of the requests for those URLs, even though we haven't explicitely told Scrapy +to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +default callback method, which is called for requests without an explicitely +assigned callback. Extracting data @@ -224,10 +228,14 @@ To extract the text from the title above, you can do:: There are two things to note here: one is that we've added ``::text`` to the CSS query, to mean that we want to select the text from inside the title element. +If we don't specify ``::text``, we'd get the HTML tags:: -The other is that the result of calling ``.extract()`` is a list, because we're -dealing with an instance :class:`~scrapy.selector.SelectorList`. When you know -you just want the first result, as in this case, you can do:: + >>> response.css('title').extract() + [u'Quotes to Scrape'] + +The other thing is that the result of calling ``.extract()`` is a list, because +we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When +you know you just want the first result, as in this case, you can do:: >>> response.css('title::text').extract_first() u'Quotes to Scrape' @@ -284,22 +292,24 @@ that contains the text "Next Page"**. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. -We won't cover much of XPath here. To learn more about XPath, we recommend `this tutorial to learn -XPath through examples `_, and `this -tutorial to learn "how to think in XPath" -`_. +We won't cover much of XPath here. To learn more about XPath, we recommend +`this tutorial to learn XPath through examples +`_, and `this tutorial to learn "how +to think in XPath" `_. .. _XPath: https://www.w3.org/TR/xpath .. _CSS: https://www.w3.org/TR/selectors -Extraction wrap-up -^^^^^^^^^^^^^^^^^^ +Extracting quotes and authors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the webpage. -Each quote in http://quotes.toscrape.com is represented by HTML code that looks -like this:: +Each quote in http://quotes.toscrape.com is represented by HTML elements that look +like this: + +.. code-block:: html
      “The world as we have created it is a process of our @@ -322,12 +332,12 @@ we want:: $ scrapy shell http://quotes.toscrape.com -We get a list of selectors to the quotes using:: +We get a list of selectors for the quote HTML elements with:: >>> response.css("div.quote") Each of the selectors returned by the query above allows us to run further -queries over the quotes itselves. Let's assign the first selector to a +queries over their sub-elements. Let's assign the first selector to a variable, so that we can run our CSS selectors directly on a particular quote:: >>> quote = response.css("div.quote")[0] @@ -342,33 +352,33 @@ using the ``quote`` object we just created:: >>> author 'Albert Einstein' -Given that the tags is a list of strings, we can use the ``.extract()`` method +Given that the tags are a list of strings, we can use the ``.extract()`` method to get all of them:: >>> tags = quote.css("div.tags a.tag ::text").extract() >>> tags ['change', 'deep-thoughts', 'thinking', 'world'] -Now, we can iterate over all the quotes in the page and use the CSS selectors -we defined to extract data:: +Having figured out how to extract each bit, we can now iterate over all the +quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): ... text = quote.css("span.text ::text").extract_first() ... author = quote.css("small.author ::text").extract_first() ... tags = quote.css("div.tags a.tag ::text").extract() - ... print("{} - {} - {}".format(text, author, tags)) + ... print(dict(text=text, author=author, tags=tags)) Extracting data in our spider ------------------------------ -Until now, the spider we built doesn't extract any data in particular. I just -saves the whole HTML page to a local file. Now, let's integrate the extraction -logic above in our spider. +Let's get back to our spider. Until now, it doesn't extract any data in +particular, just saves the whole HTML page to a local file. Let's integrate the +extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data -extracted from the page. To do that, we use the ``yield`` Python keyword, as -you can see below:: +extracted from the page. To do that, we use the ``yield`` Python keyword +in the callback, as you can see below:: import scrapy @@ -395,7 +405,38 @@ If you run this spider, it will output the extracted data with the log:: 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} -:ref:`Later in the tutorial `, we will see how to save this data to a file. + +.. _storing-data: + +Storing the scraped data +======================== + +The simplest way to store the scraped data is by using :ref:`Feed exports +`, with the following command:: + + scrapy crawl quotes -o items.json + +That will generate an ``items.json`` file containing all scraped items, +serialized in `JSON`_. + +You could've also used other formats, like `JSON Lines`_:: + + scrapy crawl quotes -o items.jl + +The `JSON Lines`_ format is useful because it's stream-like, you can easily +append new records to it. As each record is a separate line, you can also +process big files without having to fit everything in memory, there are tools +like `JQ`_ to help doing that at the command-line. + +In small projects (like the one in this tutorial), that should be enough. +However, if you want to perform more complex things with the scraped items, you +can write an :ref:`Item Pipeline `. As with Items, a +placeholder file for Item Pipelines has been set up for you when the project is +created, in ``tutorial/pipelines.py``. Though you don't need to implement any item +pipelines if you just want to store the scraped items. + +.. _JSON Lines: http://jsonlines.org +.. _JQ: https://stedolan.github.io/jq Following links @@ -511,17 +552,20 @@ much because of a programming mistake. This can be configured by the setting spider that implements a small rules engine that you can use to write your crawlers on top of it. -Adding a spider argument -======================== +Using spider arguments +====================== You can provide command line arguments to your spiders by using the ``-a`` option when running them:: scrapy crawl quotes -o items.json -a tag=humor +These arguments are passed to the Spider's ``__init__`` method and become +spider attributes by default. + In this example, the value provided for the ``tag`` argument will be available -via a spider attribute. Using this, you could make your spider get only quotes -tagged with a specific tag, building the URL based on the argument:: +via ``self.tag``. You can use this to make your spider fetch only quotes +with a specific tag, building the URL based on the argument:: import scrapy @@ -553,25 +597,7 @@ If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as ``http://quotes.toscrape.com/tag/humor``. -.. _storing-data: - -Storing the scraped data -======================== - -The simplest way to store the scraped data is by using :ref:`Feed exports -`, with the following command:: - - scrapy crawl quotes -o items.json - -That will generate an ``items.json`` file containing all scraped items, -serialized in `JSON`_. - -In small projects (like the one in this tutorial), that should be enough. -However, if you want to perform more complex things with the scraped items, you -can write an :ref:`Item Pipeline `. As with Items, a -placeholder file for Item Pipelines has been set up for you when the project is -created, in ``tutorial/pipelines.py``. Though you don't need to implement any item -pipelines if you just want to store the scraped items. +You can :ref:`learn more about handling spider arguments here `. Next steps ========== @@ -580,9 +606,10 @@ This tutorial covered only the basics of Scrapy, but there's a lot of other features not mentioned here. Check the :ref:`topics-whatelse` section in :ref:`intro-overview` chapter for a quick overview of the most important ones. -Then, we recommend you continue by playing with an example project (see -:ref:`intro-examples`), and then continue with the section -:ref:`section-basics`. +You can continue from the section :ref:`section-basics` to know more about the +command-line tool, spiders and other things the tutorial haven't covered like +modeling the scraped data. If you prefer to play with an example project, check +the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON .. _dirbot: https://github.com/scrapy/dirbot From 40293551b22c0354b3bb20e8b7756e09aadf2dc3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Sep 2016 18:14:43 +0200 Subject: [PATCH 43/53] Remove mention of odd-numbered versions for development releases Fixes GH-1317 --- docs/versioning.rst | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/versioning.rst b/docs/versioning.rst index 8e7908762..0421ba544 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -7,22 +7,33 @@ Versioning and API Stability Versioning ========== -Scrapy uses the `odd-numbered versions for development releases`_. - There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very large changes. * *B* is the release number. This will include many changes including features - and things that possibly break backwards compatibility. Even Bs will be - stable branches, and odd Bs will be development. + and things that possibly break backwards compatibility, although we strive to + keep theses cases at a minimum. * *C* is the bugfix release number. +Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, +and may require special attention before upgrading. + +Development releases do not follow 3-numbers version and are generally +released as ``dev`` suffixed versions, e.g. ``1.3dev``. + +.. note:: + With Scrapy 0.* series, Scrapy used `odd-numbered versions for development releases`_. + This is not the case anymore from Scrapy 1.0 onwards. + + Starting with Scrapy 1.0, all releases should be considered production-ready. + For example: -* *0.14.1* is the first bugfix release of the *0.14* series (safe to use in +* *1.1.1* is the first bugfix release of the *1.1* series (safe to use in production) + API Stability ============= From bc41fdf20e76ec06677a3d488323796ff2e126f7 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 20 Sep 2016 15:04:08 -0300 Subject: [PATCH 44/53] address review comments, add debug log to initial spider --- docs/intro/tutorial.rst | 94 ++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index b4a5e3b48..162fa242e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -15,7 +15,7 @@ This tutorial will walk you through these tasks: 1. Creating a new Scrapy project 2. Writing a :ref:`spider ` to crawl a site and extract data 3. Exporting the scraped data using the command line -4. Change spider to recursively follow links +4. Changing spider to recursively follow links 5. Using spider arguments Scrapy is written in Python_. If you're new to the language you might want to @@ -60,9 +60,9 @@ Our first Spider Spiders are classes that you define and that Scrapy uses to scrape information from a website (or group of websites). They must subclass -:class:`scrapy.Spider` and define the initial requests to make, how to follow -links in the pages, and how to parse the downloaded page content to extract -data. +:class:`scrapy.Spider` and define the initial requests to make, optionally how +to follow links in the pages, and how to parse the downloaded page content to +extract data. This is the code for our first Spider. Save it in a file named ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: @@ -86,6 +86,7 @@ This is the code for our first Spider. Save it in a file named filename = 'quotes-%s.html' % page with open(filename, 'wb') as f: f.write(response.body) + self.log('Saved file %s' % filename) As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -120,19 +121,17 @@ This command runs the spider with name ``quotes`` that we've just added, that will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: - - 2016-09-01 16:51:27 [scrapy] INFO: Scrapy started (bot: tutorial) - 2016-09-01 16:51:27 [scrapy] INFO: Overridden settings: {...} - 2016-09-01 16:51:27 [scrapy] INFO: Enabled extensions: ... - 2016-09-01 16:51:27 [scrapy] INFO: Enabled downloader middlewares: ... - 2016-09-01 16:51:27 [scrapy] INFO: Enabled spider middlewares: ... - 2016-09-01 16:51:27 [scrapy] INFO: Enabled item pipelines: ... - 2016-09-01 16:51:27 [scrapy] INFO: Spider opened - 2016-09-01 16:51:27 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) - 2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (404) (referer: None) - 2016-09-01 16:51:28 [scrapy] DEBUG: Crawled (200) (referer: None) - 2016-09-01 16:51:29 [scrapy] DEBUG: Crawled (200) (referer: None) - 2016-09-01 16:51:29 [scrapy] INFO: Closing spider (finished) + ... (omitted for brevity) + 2016-09-20 14:48:00 [scrapy] INFO: Spider opened + 2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023 + 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) (referer: None) + 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html + 2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html + 2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished) + ... Now, check the files in the current directory. You should notice that two new files have been created: *quotes-1.html* and *quotes-2.html*, with the content @@ -178,9 +177,9 @@ for your spider:: f.write(response.body) The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each -of the requests for those URLs, even though we haven't explicitely told Scrapy +of the requests for those URLs, even though we haven't explicitly told Scrapy to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's -default callback method, which is called for requests without an explicitely +default callback method, which is called for requests without an explicitly assigned callback. @@ -190,7 +189,7 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the shell :ref:`Scrapy shell `. Run:: - scrapy crawl http://quotes.toscrape.com/page/1/ + scrapy shell http://quotes.toscrape.com/page/1/ You will see something like:: @@ -304,7 +303,7 @@ Extracting quotes and authors ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Now that you know a bit about selection and extraction, let's complete our -spider by writing the code to extract the quotes from the webpage. +spider by writing the code to extract the quotes from the web page. Each quote in http://quotes.toscrape.com is represented by HTML elements that look like this: @@ -345,17 +344,17 @@ variable, so that we can run our CSS selectors directly on a particular quote:: Now, let's extract ``title``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text ::text").extract_first() + >>> title = quote.css("span.text::text").extract_first() >>> title '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' - >>> author = quote.css("small.author ::text").extract_first() + >>> author = quote.css("small.author::text").extract_first() >>> author 'Albert Einstein' Given that the tags are a list of strings, we can use the ``.extract()`` method to get all of them:: - >>> tags = quote.css("div.tags a.tag ::text").extract() + >>> tags = quote.css("div.tags a.tag::text").extract() >>> tags ['change', 'deep-thoughts', 'thinking', 'world'] @@ -363,10 +362,14 @@ Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): - ... text = quote.css("span.text ::text").extract_first() - ... author = quote.css("small.author ::text").extract_first() - ... tags = quote.css("div.tags a.tag ::text").extract() + ... text = quote.css("span.text::text").extract_first() + ... author = quote.css("small.author::text").extract_first() + ... tags = quote.css("div.tags a.tag::text").extract() ... print(dict(text=text, author=author, tags=tags)) + {'text': u'\u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.\u201d', 'tags': [u'change', u'deep-thoughts', u'thinking', u'world'], 'author': u'Albert Einstein'} + {'text': u'\u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.\u201d', 'tags': [u'abilities', u'choices'], 'author': u'J.K. Rowling'} + ... a few more of these, omitted for brevity + >>> Extracting data in our spider @@ -395,7 +398,7 @@ in the callback, as you can see below:: yield { 'text': quote.css('span.text::text').extract_first(), 'author': quote.css('span small::text').extract_first(), - 'tags': quote.css("div.tags a.tag ::text").extract(), + 'tags': quote.css("div.tags a.tag::text").extract(), } If you run this spider, it will output the extracted data with the log:: @@ -430,9 +433,9 @@ like `JQ`_ to help doing that at the command-line. In small projects (like the one in this tutorial), that should be enough. However, if you want to perform more complex things with the scraped items, you -can write an :ref:`Item Pipeline `. As with Items, a -placeholder file for Item Pipelines has been set up for you when the project is -created, in ``tutorial/pipelines.py``. Though you don't need to implement any item +can write an :ref:`Item Pipeline `. A placeholder file +for Item Pipelines has been set up for you when the project is created, in +``tutorial/pipelines.py``. Though you don't need to implement any item pipelines if you just want to store the scraped items. .. _JSON Lines: http://jsonlines.org @@ -448,7 +451,31 @@ from http://quotes.toscrape.com, you want quotes from all the pages in the websi Now that you know how to extract data from pages, let's see how to follow links from them. -Here is a modification of our spider that recursively follows the link to the next +First thing is to extract the link to the page we want to follow. Examining +our page, we can see there is a link to the next page with the following +markup: + +.. code-block:: html + + + +We can try extracting it in the shell:: + + >>> response.css('li.next a').extract_first() + u'Next ' + +This gets the anchor element, but we want the attribute ``href``. For that, +Scrapy supports a CSS extension that let's you select the attribute contents, +like this:: + + >>> response.css('li.next a::attr("href")').extract_first() + u'/page/2/' + +Let's see now our spider modified to recursively follows the link to the next page, extracting data from it:: import scrapy @@ -465,6 +492,7 @@ page, extracting data from it:: yield { 'text': quote.css('span.text::text').extract_first(), 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css("div.tags a.tag::text").extract(), } next_page = response.css('li.next a::attr("href")').extract_first() @@ -534,7 +562,7 @@ this time for scraping author information:: This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also -the pagination links too with the ``parse`` callback as we saw before. +the pagination links with the ``parse`` callback as we saw before. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. From a876ea5bd2911d5f7d06dfbb4ddbbdde8c51bc27 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 20 Sep 2016 15:10:49 -0300 Subject: [PATCH 45/53] minor grammar fix --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 162fa242e..f4b2d0693 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -475,7 +475,7 @@ like this:: >>> response.css('li.next a::attr("href")').extract_first() u'/page/2/' -Let's see now our spider modified to recursively follows the link to the next +Let's see now our spider modified to recursively follow the link to the next page, extracting data from it:: import scrapy From c126c593619ebbab6367f22557a44df93486942f Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 20 Sep 2016 18:19:25 -0300 Subject: [PATCH 46/53] address more review comments --- docs/intro/tutorial.rst | 119 +++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 50 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f4b2d0693..65746c389 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -59,7 +59,7 @@ Our first Spider ================ Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or group of websites). They must subclass +from a website (or a group of websites). They must subclass :class:`scrapy.Spider` and define the initial requests to make, optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. @@ -96,7 +96,7 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must return a list +* :meth:`~scrapy.spiders.Spider.start_requests`: must generate or return a list of requests where the Spider will begin to crawl from. Subsequent requests will be generated successively from these initial requests. @@ -112,8 +112,7 @@ and defines some attributes and methods: How to run our spider --------------------- -To put our spider to work, go to the project's top level directory (``cd -tutorial``) and run:: +To put our spider to work, go to the project's top level directory and run:: scrapy crawl quotes @@ -145,10 +144,10 @@ What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving -a response for each one, it instantiates :class:`scrapy.http.Response` -objects and calls the ``parse`` callback method passing the response as -argument. +returned by the ``start_requests`` method of the Spider. Upon receiving a +response for each one, it instantiates :class:`scrapy.http.Response` objects +and calls the callback method associated with the request (in this case, the +``parse`` method) passing the response as argument. A shortcut to the start_requests method @@ -166,8 +165,8 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -189,13 +188,20 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the shell :ref:`Scrapy shell `. Run:: - scrapy shell http://quotes.toscrape.com/page/1/ + scrapy shell 'http://quotes.toscrape.com/page/1/' + +.. note:: + + Remember to always enclose urls in quotes when running Scrapy shell from + command-line, otherwise urls containing arguments (ie. ``&`` character) + will not work. You will see something like:: [ ... Scrapy log here ... ] 2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} [s] request @@ -212,7 +218,7 @@ Using the shell, you can try selecting elements using `CSS`_ with the response object:: >>> response.css('title') - [Quotes to Scrape'>] + [] The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of @@ -223,26 +229,27 @@ data. To extract the text from the title above, you can do:: >>> response.css('title::text').extract() - [u'Quotes to Scrape'] + ['Quotes to Scrape'] There are two things to note here: one is that we've added ``::text`` to the -CSS query, to mean that we want to select the text from inside the title element. -If we don't specify ``::text``, we'd get the HTML tags:: +CSS query, to mean we want to select only the text elements directly inside +```` element. If we don't specify ``::text``, we'd get the full title +element, including its tags:: >>> response.css('title').extract() - [u'<title>Quotes to Scrape'] + ['Quotes to Scrape'] The other thing is that the result of calling ``.extract()`` is a list, because we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When you know you just want the first result, as in this case, you can do:: >>> response.css('title::text').extract_first() - u'Quotes to Scrape' + 'Quotes to Scrape' As an alternative, you could've written:: >>> response.css('title::text')[0].extract() - u'Quotes to Scrape' + 'Quotes to Scrape' However, using ``.extract_first()`` avoids an ``IndexError`` and returns ``None`` when it doesn't find any element matching the selection. @@ -253,21 +260,27 @@ to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.Selector.extract` and :meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use -the :meth:`~scrapy.selector.Selector.re` method to extract using a regular -expression:: +the :meth:`~scrapy.selector.Selector.re` method to extract using `regular +expressions`:: - >>> response.css('title::text').re('Quotes.*') - [u'Quotes to Scrape'] - >>> response.css('title::text').re('Q\w+') - [u'Quotes'] - >>> response.css('title::text').re('(\w+) to (\w+)') - [u'Quotes', u'Scrape'] + >>> response.css('title::text').re(r'Quotes.*') + ['Quotes to Scrape'] + >>> response.css('title::text').re(r'Q\w+') + ['Quotes'] + >>> response.css('title::text').re(r'(\w+) to (\w+)') + ['Quotes', 'Scrape'] In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools or extensions like Firebug. For more +You can use your browser developer tools or extensions like Firebug. For more information see :ref:`topics-firebug` and :ref:`topics-firefox`. +`Selector Gadget`_ is also a nice tool to quickly find CSS selector for +visually selected elements. + +.. _regular expressions: https://docs.python.org/3/library/re.html +.. _Selector Gadget: http://selectorgadget.com/ + XPath: a brief intro ^^^^^^^^^^^^^^^^^^^^ @@ -275,9 +288,9 @@ XPath: a brief intro Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: >>> response.xpath('//title') - [Quotes to Scrape'>] + [] >>> response.xpath('//title/text()').extract_first() - u'Quotes to Scrape' + 'Quotes to Scrape' XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You @@ -291,8 +304,9 @@ that contains the text "Next Page"**. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. -We won't cover much of XPath here. To learn more about XPath, we recommend -`this tutorial to learn XPath through examples +We won't cover much of XPath here, but you can read more about `using XPath +with Scrapy Selectors here `_. To learn more about XPath, we +recommend `this tutorial to learn XPath through examples `_, and `this tutorial to learn "how to think in XPath" `_. @@ -366,8 +380,8 @@ quotes elements and put them together into a Python dictionary:: ... author = quote.css("small.author::text").extract_first() ... tags = quote.css("div.tags a.tag::text").extract() ... print(dict(text=text, author=author, tags=tags)) - {'text': u'\u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.\u201d', 'tags': [u'change', u'deep-thoughts', u'thinking', u'world'], 'author': u'Albert Einstein'} - {'text': u'\u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.\u201d', 'tags': [u'abilities', u'choices'], 'author': u'J.K. Rowling'} + {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} + {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} ... a few more of these, omitted for brevity >>> @@ -417,19 +431,24 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o items.json + scrapy crawl quotes -o quotes.json -That will generate an ``items.json`` file containing all scraped items, +That will generate an ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -You could've also used other formats, like `JSON Lines`_:: +For historic reasons, Scrapy appends to a given file instead of overwriting +its contents. If you run this command twice without removing the file +before the second time, you'll end up with a broken JSON file. - scrapy crawl quotes -o items.jl +You can also used other formats, like `JSON Lines`_:: + + scrapy crawl quotes -o quotes.jl The `JSON Lines`_ format is useful because it's stream-like, you can easily -append new records to it. As each record is a separate line, you can also -process big files without having to fit everything in memory, there are tools -like `JQ`_ to help doing that at the command-line. +append new records to it. It doesn't have the same problem of JSON when you run +twice. Also, as each record is a separate line, you can process big files +without having to fit everything in memory, there are tools like `JQ`_ to help +doing that at the command-line. In small projects (like the one in this tutorial), that should be enough. However, if you want to perform more complex things with the scraped items, you @@ -466,14 +485,14 @@ markup: We can try extracting it in the shell:: >>> response.css('li.next a').extract_first() - u'Next ' + 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, Scrapy supports a CSS extension that let's you select the attribute contents, like this:: - >>> response.css('li.next a::attr("href")').extract_first() - u'/page/2/' + >>> response.css('li.next a::attr(href)').extract_first() + '/page/2/' Let's see now our spider modified to recursively follow the link to the next page, extracting data from it:: @@ -495,7 +514,7 @@ page, extracting data from it:: 'tags': quote.css("div.tags a.tag::text").extract(), } - next_page = response.css('li.next a::attr("href")').extract_first() + next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -540,12 +559,12 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author a::attr("href")').extract(): + for href in response.css('.author a::attr(href)').extract(): yield scrapy.Request(response.urljoin(href), callback=self.parse_author) # follow pagination links - next_page = response.css('li.next a::attr("href")').extract_first() + next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -586,7 +605,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o items.json -a tag=humor + scrapy crawl quotes -o quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. @@ -606,7 +625,7 @@ with a specific tag, building the URL based on the argument:: tag = getattr(self, 'tag', None) if tag is not None: url = url + 'tag/' + tag - yield scrapy.Request(url) + yield scrapy.Request(url, self.parse) def parse(self, response): for quote in response.css('div.quote'): @@ -615,10 +634,10 @@ with a specific tag, building the URL based on the argument:: 'author': quote.css('span small a::text').extract_first(), } - next_page = response.css('li.next a::attr("href")').extract_first() + next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + yield scrapy.Request(next_page, self.parse) If you pass the ``tag=humor`` argument to this spider, you'll notice that it From 38266cc949f594c7f596728876151a65b481c966 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Wed, 21 Sep 2016 11:02:24 -0300 Subject: [PATCH 47/53] recommend Dive into Python and Python tutorial instead of LPTHW for non-beginners --- docs/intro/tutorial.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 65746c389..ec68bf922 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -20,14 +20,17 @@ This tutorial will walk you through these tasks: Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of -Scrapy. If you're already familiar with other languages, and want to learn -Python quickly, we recommend `Learn Python The Hard Way`_. If you're new to programming -and want to start with Python, take a look at `this list of Python resources -for non-programmers`_. +Scrapy. If you're already familiar with other languages, and want to learn +Python quickly, we recommend reading through `Dive Into Python 3`_. +Alternatively, you can follow the `Python Tutorial`_. If you're new to +programming and want to start with Python, take a look at `this list of Python +resources for non-programmers`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers -.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ +.. _Dive Into Python 3: http://www.diveintopython3.net +.. _Python Tutorial: https://docs.python.org/3/tutorial + Creating a project ================== From 32017a76f8560d8cba2746d541adb63e03e68f62 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Wed, 21 Sep 2016 11:06:36 -0300 Subject: [PATCH 48/53] recommend learn python the hard way for beginners --- docs/intro/tutorial.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ec68bf922..31228017b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -20,16 +20,21 @@ This tutorial will walk you through these tasks: Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of -Scrapy. If you're already familiar with other languages, and want to learn -Python quickly, we recommend reading through `Dive Into Python 3`_. -Alternatively, you can follow the `Python Tutorial`_. If you're new to -programming and want to start with Python, take a look at `this list of Python -resources for non-programmers`_. +Scrapy. + +If you're already familiar with other languages, and want to learn Python +quickly, we recommend reading through `Dive Into Python 3`_. Alternatively, +you can follow the `Python Tutorial`_. + +If you're new to programming and want to start with Python, you may find useful +the online book `Learn Python The Hard Way`_. You can also take a look at `this +list of Python resources for non-programmers`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _Dive Into Python 3: http://www.diveintopython3.net .. _Python Tutorial: https://docs.python.org/3/tutorial +.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ Creating a project From d636e5baa8a077e2869bfe3b76525efec42392ec Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Wed, 21 Sep 2016 18:54:12 -0300 Subject: [PATCH 49/53] better description for start_requests expected return value --- docs/intro/tutorial.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 31228017b..e85219e06 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -104,9 +104,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must generate or return a list - of requests where the Spider will begin to crawl from. - Subsequent requests will be generated successively from these initial requests. +* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of + Requests (you can return a list of requests or write a generator function) + which the Spider will begin to crawl from. Subsequent requests will be + generated successively from these initial requests. * :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter From f4a22089168c31a2b6c2f03c0053073eb80e33b3 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 22 Sep 2016 11:04:45 -0300 Subject: [PATCH 50/53] addressing review comments and other minor editing --- docs/intro/tutorial.rst | 63 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index e85219e06..4f2736709 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -111,8 +111,8 @@ and defines some attributes and methods: * :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter - is an instance of :class:`~scrapy.http.Response` that holds the page content and - has further helpful methods to handle it. + is an instance of :class:`~scrapy.http.TextResponse` that holds + the page content and has further helpful methods to handle it. The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting the scraped data as dicts and also finding new URLs to @@ -154,7 +154,7 @@ What just happened under the hood? Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`scrapy.http.Response` objects +response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the ``parse`` method) passing the response as argument. @@ -281,11 +281,11 @@ expressions`:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools or extensions like Firebug. For more -information see :ref:`topics-firebug` and :ref:`topics-firefox`. +You can use your browser developer tools or extensions like Firebug (see +sections about :ref:`topics-firebug` and :ref:`topics-firefox`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for -visually selected elements. +visually selected elements, which works in many browsers. .. _regular expressions: https://docs.python.org/3/library/re.html .. _Selector Gadget: http://selectorgadget.com/ @@ -308,13 +308,13 @@ objects in the shell. While perhaps not as popular as CSS selectors, XPath expressions offer more power because besides navigating the structure, it can also look at the -content. Using XPath, you're able to select things like: **select the link -that contains the text "Next Page"**. This makes XPath very fitting to the task +content. Using XPath, you're able to select things like: *select the link +that contains the text "Next Page"*. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. -We won't cover much of XPath here, but you can read more about `using XPath -with Scrapy Selectors here `_. To learn more about XPath, we +We won't cover much of XPath here, but you can read more about :ref:`using XPath +with Scrapy Selectors here `. To learn more about XPath, we recommend `this tutorial to learn XPath through examples `_, and `this tutorial to learn "how to think in XPath" `_. @@ -352,7 +352,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell http://quotes.toscrape.com + $ scrapy shell 'http://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with:: @@ -394,7 +394,6 @@ quotes elements and put them together into a Python dictionary:: ... a few more of these, omitted for brevity >>> - Extracting data in our spider ------------------------------ @@ -421,7 +420,7 @@ in the callback, as you can see below:: yield { 'text': quote.css('span.text::text').extract_first(), 'author': quote.css('span small::text').extract_first(), - 'tags': quote.css("div.tags a.tag::text").extract(), + 'tags': quote.css('div.tags a.tag::text').extract(), } If you run this spider, it will output the extracted data with the log:: @@ -520,7 +519,7 @@ page, extracting data from it:: yield { 'text': quote.css('span.text::text').extract_first(), 'author': quote.css('span small::text').extract_first(), - 'tags': quote.css("div.tags a.tag::text").extract(), + 'tags': quote.css('div.tags a.tag::text').extract(), } next_page = response.css('li.next a::attr(href)').extract_first() @@ -530,10 +529,11 @@ page, extracting data from it:: Now, after extracting the data, the ``parse()`` method looks for the link to -the next page, builds a full absolute URL using the ``response.urljoin`` method -(since the links can be relative) and yields a new request to the next page, -registering itself as callback to handle the data extraction for the next page -and to keep the crawling going through all the pages. +the next page, builds a full absolute URL using the +:meth:`~scrapy.http.Response.urljoin` method (since the links can be +relative) and yields a new request to the next page, registering itself as +callback to handle the data extraction for the next page and to keep the +crawling going through all the pages. What you see here is Scrapy's mechanism of following links: when you yield a Request in a callback method, Scrapy will schedule that request to be sent @@ -547,12 +547,8 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. -Another common pattern is to build an item with data from more than one page, -using a :ref:`trick to pass additional data to the callbacks -`. - -Another example: scraping authors ---------------------------------- +More examples and patterns +-------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: @@ -602,11 +598,18 @@ requests to URLs already visited, avoiding the problem of hitting servers too much because of a programming mistake. This can be configured by the setting :setting:`DUPEFILTER_CLASS`. -.. note:: - As another example spider that leverages the mechanism of following links, - check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic - spider that implements a small rules engine that you can use to write your - crawlers on top of it. +Hopefully by now you have a good understanding of how to use the mechanism +of following links and callbacks with Scrapy. + +As yet another example spider that leverages the mechanism of following links, +check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic +spider that implements a small rules engine that you can use to write your +crawlers on top of it. + +Also, a common pattern is to build an item with data from more than one page, +using a :ref:`trick to pass additional data to the callbacks +`. + Using spider arguments ====================== @@ -663,7 +666,7 @@ features not mentioned here. Check the :ref:`topics-whatelse` section in :ref:`intro-overview` chapter for a quick overview of the most important ones. You can continue from the section :ref:`section-basics` to know more about the -command-line tool, spiders and other things the tutorial haven't covered like +command-line tool, spiders, selectors and other things the tutorial hasn't covered like modeling the scraped data. If you prefer to play with an example project, check the :ref:`intro-examples` section. From b2bfd1e5c536997a2c260e56e8e9fa1303691384 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 23 Sep 2016 10:36:03 +0200 Subject: [PATCH 51/53] [docs] document that process_item can return Deferred --- docs/topics/item-pipeline.rst | 55 +++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index b9b4c2058..a31fe7423 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -27,9 +27,10 @@ Each item pipeline component is a Python class that must implement the following .. method:: process_item(self, item, spider) - This method is called for every item pipeline component and must either return - a dict with data, :class:`~scrapy.item.Item` (or any descendant class) object - or raise a :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer + This method is called for every item pipeline component. :meth:`process_item` + must either: return a dict with data, return an :class:`~scrapy.item.Item` + (or any descendant class) object, return a `Twisted Deferred`_ or raise + :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. :param item: the item scraped @@ -66,6 +67,8 @@ Additionally, they may also implement the following methods: :type crawler: :class:`~scrapy.crawler.Crawler` object +.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html + Item pipeline example ===================== @@ -163,6 +166,52 @@ method and how to clean up the resources properly. .. _MongoDB: https://www.mongodb.org/ .. _pymongo: https://api.mongodb.org/python/current/ + +Take screenshot of item +----------------------- + +This example demonstrates how to return Deferred_ from :meth:`process_item` method. +It uses Splash_ to render screenshot of item url. Pipeline +makes request to locally running instance of Splash_. After request is downloaded +and Deferred callback fires, it saves item to a file and adds filename to an item. + +:: + + import scrapy + + + class ScreenshotPipeline(object): + """Pipeline that uses Splash to render screenshot of + every Scrapy item.""" + + SPLASH_URL = "http://localhost:8050/render.png?url={}" + + def process_item(self, item, spider): + item_url = item["url"] + screenshot_url = self.SPLASH_URL.format(item_url) + request = scrapy.Request(screenshot_url) + dfd = spider.crawler.engine.download(request, spider) + dfd.addBoth(self.return_item, item) + return dfd + + def return_item(self, response, item): + if response.status != 200: + # Error happened, return item. + return item + + # Save screenshot to file. + filename = "item_file_name.png" + with open(filename, "wb") as f: + f.write(response.body) + + # Store filename in item. + item["screenshot_filename"] = filename + return item + + +.. _Splash: http://splash.readthedocs.io/en/stable/ +.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html + Duplicates filter ----------------- From a0f87d2f45865158a6ea687a1603e4b38106ff3f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 22 Sep 2016 17:21:26 +0200 Subject: [PATCH 52/53] Update release notes for upcoming 1.1.3 bugfix release --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5395db8e3..0992ebf89 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +1.1.3 (YYYY-MM-DD) +------------------ + +Bug fixes +~~~~~~~~~ + +- Class attributes for subclasses of ``ImagesPipeline`` and ``FilesPipeline`` + work at they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`) + +Documentation +~~~~~~~~~~~~~ + +- :ref:`Overview ` and :ref:`tutorial ` + rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`). + + 1.1.2 (2016-08-18) ------------------ From 80c1e5dc252f09b3ab864405c87586d7db42dfac Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 22 Sep 2016 21:33:05 +0200 Subject: [PATCH 53/53] Set release date, fix typo and add tutorial improvement issue number --- docs/news.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 0992ebf89..0e9efe6f6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,20 +3,21 @@ Release notes ============= -1.1.3 (YYYY-MM-DD) +1.1.3 (2016-09-22) ------------------ Bug fixes ~~~~~~~~~ - Class attributes for subclasses of ``ImagesPipeline`` and ``FilesPipeline`` - work at they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`) + work as they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`) Documentation ~~~~~~~~~~~~~ - :ref:`Overview ` and :ref:`tutorial ` - rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`). + rewritten to use http://toscrape.com websites + (:issue:`2236`, :issue:`2249`, :issue:`2252`). 1.1.2 (2016-08-18)