diff --git a/.travis.yml b/.travis.yml index c58ab39a5..506f3779b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ branches: - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ env: - TOXENV=py27 - - TOXENV=precise + - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 - TOXENV=docs diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..a6f6cbda8 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,16 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + +{% endblock %} diff --git a/docs/index.rst b/docs/index.rst index 4cb3eb741..b4272e47f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,13 +13,15 @@ Having trouble? We'd like to help! * Try the :doc:`FAQ ` -- it's got answers to some common questions. * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`. +* Ask or search questions in `StackOverflow using the scrapy tag`_, * Search for information in the `archives of the scrapy-users mailing list`_, or `post a question`_. -* Ask a question in the `#scrapy IRC channel`_. +* Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. .. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _post a question: https://groups.google.com/forum/#!forum/scrapy-users +.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6c1291c1f..7195017ff 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -19,74 +19,69 @@ Walk-through of an example spider In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. -So, here's the code for a spider that follows the links to the top -voted questions on StackOverflow and scrapes some data from each page:: +Here's the code for a spider that scrapes famous quotes from website +http://quotes.toscrape.com, following the pagination:: import scrapy - class StackOverflowSpider(scrapy.Spider): - name = 'stackoverflow' - start_urls = ['http://stackoverflow.com/questions?sort=votes'] + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/tag/humor/', + ] def parse(self, response): - for href in response.css('.question-summary h3 a::attr(href)'): - full_url = response.urljoin(href.extract()) - yield scrapy.Request(full_url, callback=self.parse_question) + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } - def parse_question(self, response): - yield { - 'title': response.css('h1 a::text').extract_first(), - 'votes': response.css('.question .vote-count-post::text').extract_first(), - 'body': response.css('.question .post-text').extract_first(), - 'tags': response.css('.question .post-tag::text').extract(), - 'link': response.url, - } + next_page = response.css('li.next a::attr("href")').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Put this in a file, name it to something like ``stackoverflow_spider.py`` +Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json + scrapy runspider quotes_spider.py -o quotes.json -When this finishes you will have in the ``top-stackoverflow-questions.json`` file -a list of the most upvoted questions in StackOverflow in JSON format, containing the -title, link, number of upvotes, a list of the tags and the question content in HTML, -looking like this (reformatted for easier reading):: +When this finishes you will have in the ``quotes.json`` file a list of the +quotes in JSON format, containing text and author, looking like this (reformatted +here for better readability):: [{ - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array", - "tags": ["java", "c++", "performance", "optimization"], - "title": "Why is processing a sorted array faster than an unsorted array?", - "votes": "9924" + "author": "Jane Austen", + "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" }, { - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule", - "tags": ["git", "git-submodules"], - "title": "How do I remove a Git submodule?", - "votes": "1764" + "author": "Groucho Marx", + "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" + }, + { + "author": "Steve Martin", + "text": "\u201cA day without sunshine is like, you know, night.\u201d" }, ...] - What just happened? ------------------- -When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a +When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked for a Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for StackOverflow top questions page) +attribute (in this case, only the URL for quotes in *humor* category) and called the default callback method ``parse``, passing the response object as -an argument. In the ``parse`` callback we extract the links to the -question pages using a CSS Selector with a custom extension that allows to get -the value for an attribute. Then we yield a few more requests to be sent, -registering the method ``parse_question`` as the callback to be called for each -of them as they finish. +an argument. In the ``parse`` callback, we loop through the quote elements +using a CSS Selector, yield a Python dict with the extracted quote text and author, +look for a link to the next page and schedule another request using the same +``parse`` method as callback. Here you notice one of the main advantages about Scrapy: requests are :ref:`scheduled and processed asynchronously `. This @@ -103,10 +98,6 @@ each request, limiting amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension ` that tries to figure out these automatically. -Finally, the ``parse_question`` callback scrapes the question data for each -page yielding a dict, which Scrapy then collects and writes to a JSON file as -requested in the command line. - .. note:: This is using :ref:`feed exports ` to generate the @@ -145,12 +136,13 @@ scraping easy and efficient, such as: :ref:`pipelines `). * Wide range of built-in extensions and middlewares for handling: - * cookies and session handling - * HTTP features like compression, authentication, caching - * user-agent spoofing - * robots.txt - * crawl depth restriction - * and more + + - cookies and session handling + - HTTP features like compression, authentication, caching + - user-agent spoofing + - robots.txt + - crawl depth restriction + - and more * A :ref:`Telnet console ` for hooking into a Python console running inside your Scrapy process, to introspect and debug your @@ -165,8 +157,8 @@ What's next? ============ The next steps for you are to :ref:`install Scrapy `, -:ref:`follow through the tutorial ` to learn how to organize -your code in Scrapy projects and `join the community`_. Thanks for your +:ref:`follow through the tutorial ` to learn how to create +a full-blown Scrapy project and `join the community`_. Thanks for your interest! .. _join the community: http://scrapy.org/community/ diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6ecd637c3..4f2736709 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,29 +7,36 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to use `Open directory project (dmoz) `_ as -our example domain to scrape. +We are going to scrape `quotes.toscrape.com `_, a website +that lists quotes from famous authors. This tutorial will walk you through these tasks: 1. Creating a new Scrapy project -2. Defining the Items you will extract -3. Writing a :ref:`spider ` to crawl a site and extract - :ref:`Items ` -4. Writing an :ref:`Item Pipeline ` to store the - extracted Items +2. Writing a :ref:`spider ` to crawl a site and extract data +3. Exporting the scraped data using the command line +4. Changing spider to recursively follow links +5. Using spider arguments Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of -Scrapy. If you're already familiar with other languages, and want to learn -Python quickly, we recommend `Learn Python The Hard Way`_. If you're new to programming -and want to start with Python, take a look at `this list of Python resources -for non-programmers`_. +Scrapy. + +If you're already familiar with other languages, and want to learn Python +quickly, we recommend reading through `Dive Into Python 3`_. Alternatively, +you can follow the `Python Tutorial`_. + +If you're new to programming and want to start with Python, you may find useful +the online book `Learn Python The Hard Way`_. You can also take a look at `this +list of Python resources for non-programmers`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers +.. _Dive Into Python 3: http://www.diveintopython3.net +.. _Python Tutorial: https://docs.python.org/3/tutorial .. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ + Creating a project ================== @@ -46,7 +53,7 @@ This will create a ``tutorial`` directory with the following contents:: tutorial/ # project's Python module, you'll import your code from here __init__.py - items.py # project items file + items.py # project items definition file pipelines.py # project pipelines file @@ -54,208 +61,143 @@ This will create a ``tutorial`` directory with the following contents:: spiders/ # a directory where you'll later put your spiders __init__.py - ... -Defining our Item -================= - -`Items` are containers that will be loaded with the scraped data; they work -like simple Python dicts. While you can use plain Python dicts with Scrapy, -`Items` provide additional protection against populating undeclared fields, -preventing typos. They can also be used with :ref:`Item Loaders -`, a mechanism with helpers to conveniently populate `Items`. - -They are declared by creating a :class:`scrapy.Item ` class and defining -its attributes as :class:`scrapy.Field ` objects, much like in an ORM -(don't worry if you're not familiar with ORMs, you will see that this is an -easy task). - -We begin by modeling the item that we will use to hold the site's data obtained -from dmoz.org. As we want to capture the name, url and description of the -sites, we define fields for each of these three attributes. To do that, we edit -``items.py``, found in the ``tutorial`` directory. Our Item class looks like this:: - - import scrapy - - class DmozItem(scrapy.Item): - title = scrapy.Field() - link = scrapy.Field() - desc = scrapy.Field() - -This may seem complicated at first, but defining an item class allows you to use other handy -components and helpers within Scrapy. - Our first Spider ================ -Spiders are classes that you define and Scrapy uses to scrape information from a -domain (or group of domains). +Spiders are classes that you define and that Scrapy uses to scrape information +from a website (or a group of websites). They must subclass +:class:`scrapy.Spider` and define the initial requests to make, optionally how +to follow links in the pages, and how to parse the downloaded page content to +extract data. -They define an initial list of URLs to download, how to follow links, and how -to parse the contents of pages to extract :ref:`items `. - -To create a Spider, you must subclass :class:`scrapy.Spider -` and define some attributes: - -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be - unique, that is, you can't set the same name for different Spiders. - -* :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the - Spider will begin to crawl from. The first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data - contained in the start URLs. - -* :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will - be called with the downloaded :class:`~scrapy.http.Response` object of each - start URL. The response is passed to the method as the first and only - argument. - - This method is responsible for parsing the response data and extracting - scraped data (as scraped items) and more URLs to follow. - - The :meth:`~scrapy.spiders.Spider.parse` method is in charge of processing - the response and returning scraped data (as :class:`~scrapy.item.Item` - objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects). - -This is the code for our first Spider; save it in a file named -``dmoz_spider.py`` under the ``tutorial/spiders`` directory:: +This is the code for our first Spider. Save it in a file named +``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: import scrapy - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] + + class QuotesSpider(scrapy.Spider): + name = "quotes" + + def start_requests(self): + urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): - filename = response.url.split("/")[-2] + '.html' + page = response.url.split("/")[-2] + filename = 'quotes-%s.html' % page with open(filename, 'wb') as f: f.write(response.body) + self.log('Saved file %s' % filename) -Crawling --------- + +As you can see, our Spider subclasses :class:`scrapy.Spider ` +and defines some attributes and methods: + +* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be + unique within a project, that is, you can't set the same name for different + Spiders. + +* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of + Requests (you can return a list of requests or write a generator function) + which the Spider will begin to crawl from. Subsequent requests will be + generated successively from these initial requests. + +* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle + the response downloaded for each of the requests made. The response parameter + is an instance of :class:`~scrapy.http.TextResponse` that holds + the page content and has further helpful methods to handle it. + + The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + the scraped data as dicts and also finding new URLs to + follow and creating new requests (:class:`~scrapy.http.Request`) from them. + +How to run our spider +--------------------- To put our spider to work, go to the project's top level directory and run:: - scrapy crawl dmoz + scrapy crawl quotes -This command runs the spider with name ``dmoz`` that we've just added, that -will send some requests for the ``dmoz.org`` domain. You will get an output +This command runs the spider with name ``quotes`` that we've just added, that +will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: - 2014-01-23 18:13:07-0400 [scrapy] INFO: Scrapy started (bot: tutorial) - 2014-01-23 18:13:07-0400 [scrapy] INFO: Optional features available: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Overridden settings: {} - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled extensions: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled downloader middlewares: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled spider middlewares: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled item pipelines: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Spider opened - 2014-01-23 18:13:08-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 18:13:09-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 18:13:09-0400 [scrapy] INFO: Closing spider (finished) + ... (omitted for brevity) + 2016-09-20 14:48:00 [scrapy] INFO: Spider opened + 2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023 + 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) (referer: None) + 2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html + 2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) (referer: None) + 2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html + 2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished) + ... +Now, check the files in the current directory. You should notice that two new +files have been created: *quotes-1.html* and *quotes-2.html*, with the content +for the respective URLs, as our ``parse`` method instructs. -.. note:: - At the end you can see a log line for each URL defined in ``start_urls``. - Because these URLs are the starting ones, they have no referrers, which is - shown at the end of the log line, where it says ``(referer: None)``. +.. note:: If you are wondering why we haven't parsed the HTML yet, hold + on, we will cover that soon. -Now, check the files in the current directory. You should notice two new files -have been created: *Books.html* and *Resources.html*, with the content for the respective -URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy creates :class:`scrapy.Request ` objects -for each URL in the ``start_urls`` attribute of the Spider, and assigns -them the ``parse`` method of the spider as their callback function. - -These Requests are scheduled, then executed, and :class:`scrapy.http.Response` -objects are returned and then fed back to the spider, through the -:meth:`~scrapy.spiders.Spider.parse` method. - -Extracting Items ----------------- - -Introduction to Selectors -^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to extract data from web pages. Scrapy uses a mechanism -based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors -`. For more information about selectors and other extraction -mechanisms see the :ref:`Selectors documentation `. - -.. _XPath: https://www.w3.org/TR/xpath -.. _CSS: https://www.w3.org/TR/selectors - -Here are some examples of XPath expressions and their meanings: - -* ``/html/head/title``: selects the ```` element, inside the ``<head>`` - element of an HTML document - -* ``/html/head/title/text()``: selects the text inside the aforementioned - ``<title>`` element. - -* ``//td``: selects all the ``<td>`` elements - -* ``//div[@class="mine"]``: selects all ``div`` elements which contain an - attribute ``class="mine"`` - -These are just a couple of simple examples of what you can do with XPath, but -XPath expressions are indeed much more powerful. To learn more about XPath, we -recommend `this tutorial to learn XPath through examples -<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how -to think in XPath" <http://plasmasturm.org/log/xpath101/>`_. - -.. note:: **CSS vs XPath:** you can go a long way extracting data from web pages - using only CSS selectors. However, XPath offers more power because besides - navigating the structure, it can also look at the content: you're - able to select things like: *the link that contains the text 'Next Page'*. - Because of this, we encourage you to learn about XPath even if you - already know how to construct CSS selectors. - -For working with CSS and XPath expressions, Scrapy provides -:class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid -instantiating selectors yourself every time you need to select something from a -response. - -You can see selectors as objects that represent nodes in the document -structure. So, the first instantiated selectors are associated with the root -node, or the entire document. - -Selectors have four basic methods (click on the method to see the complete API -documentation): - -* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of - which represents the nodes selected by the xpath expression given as - argument. - -* :meth:`~scrapy.selector.Selector.css`: returns a list of selectors, each of - which represents the nodes selected by the CSS expression given as argument. - -* :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the - selected data. - -* :meth:`~scrapy.selector.Selector.re`: returns a list of unicode strings - extracted by applying the regular expression given as argument. +Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects +returned by the ``start_requests`` method of the Spider. Upon receiving a +response for each one, it instantiates :class:`~scrapy.http.Response` objects +and calls the callback method associated with the request (in this case, the +``parse`` method) passing the response as argument. -Trying Selectors in the Shell -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +A shortcut to the start_requests method +--------------------------------------- +Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method +that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs, +you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +with a list of URLs. This list will then be used by the default implementation +of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +for your spider:: -To illustrate the use of Selectors we're going to use the built-in :ref:`Scrapy -shell <topics-shell>`, which also requires `IPython <http://ipython.org/>`_ (an extended Python console) -installed on your system. + import scrapy -To start a shell, you must go to the project's top level directory and run:: - scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/" + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + + def parse(self, response): + page = response.url.split("/")[-2] + filename = 'quotes-%s.html' % page + with open(filename, 'wb') as f: + f.write(response.body) + +The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +of the requests for those URLs, even though we haven't explicitly told Scrapy +to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +default callback method, which is called for requests without an explicitly +assigned callback. + + +Extracting data +--------------- + +The best way to learn how to extract data with Scrapy is trying selectors +using the shell :ref:`Scrapy shell <topics-shell>`. Run:: + + scrapy shell 'http://quotes.toscrape.com/page/1/' .. note:: @@ -263,217 +205,335 @@ To start a shell, you must go to the project's top level directory and run:: command-line, otherwise urls containing arguments (ie. ``&`` character) will not work. -This is what the shell looks like:: +You will see something like:: [ ... Scrapy log here ... ] - - 2014-01-23 17:11:42-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None) + 2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None) [s] Available Scrapy objects: - [s] crawler <scrapy.crawler.Crawler object at 0x3636b50> + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90> [s] item {} - [s] request <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] settings <scrapy.settings.Settings object at 0x3fadc50> - [s] spider <Spider 'default' at 0x3cebf50> + [s] request <GET http://quotes.toscrape.com/page/1/> + [s] response <200 http://quotes.toscrape.com/page/1/> + [s] settings <scrapy.settings.Settings object at 0x7fa91d888c10> + [s] spider <DefaultSpider 'default' at 0x7fa91c8af990> [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser + >>> - In [1]: +Using the shell, you can try selecting elements using `CSS`_ with the response +object:: -After the shell loads, you will have the response fetched in a local -``response`` variable, so if you type ``response.body`` you will see the body -of the response, or you can type ``response.headers`` to see its headers. + >>> response.css('title') + [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape'>] -More importantly ``response`` has a ``selector`` attribute which is an instance of -:class:`~scrapy.selector.Selector` class, instantiated with this particular ``response``. -You can run queries on ``response`` by calling ``response.selector.xpath()`` or -``response.selector.css()``. There are also some convenience shortcuts like ``response.xpath()`` -or ``response.css()`` which map directly to ``response.selector.xpath()`` and -``response.selector.css()``. +The result of running ``response.css('title')`` is a list-like object called +:class:`~scrapy.selector.SelectorList`, which represents a list of +:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +and allow you to run further queries to fine-grain the selection or extract the +data. + +To extract the text from the title above, you can do:: + + >>> response.css('title::text').extract() + ['Quotes to Scrape'] + +There are two things to note here: one is that we've added ``::text`` to the +CSS query, to mean we want to select only the text elements directly inside +```` element. If we don't specify ``::text``, we'd get the full title +element, including its tags:: + + >>> response.css('title').extract() + ['<title>Quotes to Scrape'] + +The other thing is that the result of calling ``.extract()`` is a list, because +we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When +you know you just want the first result, as in this case, you can do:: + + >>> response.css('title::text').extract_first() + 'Quotes to Scrape' + +As an alternative, you could've written:: + + >>> response.css('title::text')[0].extract() + 'Quotes to Scrape' + +However, using ``.extract_first()`` avoids an ``IndexError`` and returns +``None`` when it doesn't find any element matching the selection. + +There's a lesson here: for most scraping code, you want it to be resilient to +errors due to things not being found on a page, so that even if some parts fail +to be scraped, you can at least get **some** data. + +Besides the :meth:`~scrapy.selector.Selector.extract` and +:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use +the :meth:`~scrapy.selector.Selector.re` method to extract using `regular +expressions`:: + + >>> response.css('title::text').re(r'Quotes.*') + ['Quotes to Scrape'] + >>> response.css('title::text').re(r'Q\w+') + ['Quotes'] + >>> response.css('title::text').re(r'(\w+) to (\w+)') + ['Quotes', 'Scrape'] + +In order to find the proper CSS selectors to use, you might find useful opening +the response page from the shell in your web browser using ``view(response)``. +You can use your browser developer tools or extensions like Firebug (see +sections about :ref:`topics-firebug` and :ref:`topics-firefox`). + +`Selector Gadget`_ is also a nice tool to quickly find CSS selector for +visually selected elements, which works in many browsers. + +.. _regular expressions: https://docs.python.org/3/library/re.html +.. _Selector Gadget: http://selectorgadget.com/ -So let's try it:: +XPath: a brief intro +^^^^^^^^^^^^^^^^^^^^ - In [1]: response.xpath('//title') - Out[1]: [Open Directory - Computers: Progr'>] - - In [2]: response.xpath('//title').extract() - Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - - In [3]: response.xpath('//title/text()') - Out[3]: [] - - In [4]: response.xpath('//title/text()').extract() - Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - - In [5]: response.xpath('//title/text()').re('(\w+):') - Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] +Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: -Extracting the data -^^^^^^^^^^^^^^^^^^^ + >>> response.xpath('//title') + [] + >>> response.xpath('//title/text()').extract_first() + 'Quotes to Scrape' -Now, let's try to extract some real information from those pages. +XPath expressions are very powerful, and are the foundation of Scrapy +Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You +can see that if you read closely the text representation of the selector +objects in the shell. -You could type ``response.body`` in the console, and inspect the source code to -figure out the XPaths you need to use. However, inspecting the raw HTML code -there could become a very tedious task. To make it easier, you can -use Firefox Developer Tools or some Firefox extensions like Firebug. For more -information see :ref:`topics-firebug` and :ref:`topics-firefox`. +While perhaps not as popular as CSS selectors, XPath expressions offer more +power because besides navigating the structure, it can also look at the +content. Using XPath, you're able to select things like: *select the link +that contains the text "Next Page"*. This makes XPath very fitting to the task +of scraping, and we encourage you to learn XPath even if you already know how to +construct CSS selectors, it will make scraping much easier. -After inspecting the page source, you'll find that the web site's information -is inside a ``
    `` element, in fact the *second* ``
      `` element. +We won't cover much of XPath here, but you can read more about :ref:`using XPath +with Scrapy Selectors here `. To learn more about XPath, we +recommend `this tutorial to learn XPath through examples +`_, and `this tutorial to learn "how +to think in XPath" `_. -So we can select each ``
    • `` element belonging to the site's list with this -code:: +.. _XPath: https://www.w3.org/TR/xpath +.. _CSS: https://www.w3.org/TR/selectors - response.xpath('//ul/li') +Extracting quotes and authors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -And from them, the site's descriptions:: +Now that you know a bit about selection and extraction, let's complete our +spider by writing the code to extract the quotes from the web page. - response.xpath('//ul/li/text()').extract() +Each quote in http://quotes.toscrape.com is represented by HTML elements that look +like this: -The site's titles:: +.. code-block:: html - response.xpath('//ul/li/a/text()').extract() +
      + “The world as we have created it is a process of our + thinking. It cannot be changed without changing our thinking.” + + by Albert Einstein + (about) + +
      + Tags: + change + deep-thoughts + thinking + world +
      +
      -And the site's links:: +Let's open up scrapy shell and play a bit to find out how to extract the data +we want:: - response.xpath('//ul/li/a/@href').extract() + $ scrapy shell 'http://quotes.toscrape.com' -As we've said before, each ``.xpath()`` call returns a list of selectors, so we can -concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use -that property here, so:: +We get a list of selectors for the quote HTML elements with:: - for sel in response.xpath('//ul/li'): - title = sel.xpath('a/text()').extract() - link = sel.xpath('a/@href').extract() - desc = sel.xpath('text()').extract() - print title, link, desc + >>> response.css("div.quote") -.. note:: +Each of the selectors returned by the query above allows us to run further +queries over their sub-elements. Let's assign the first selector to a +variable, so that we can run our CSS selectors directly on a particular quote:: - For a more detailed description of using nested selectors, see - :ref:`topics-selectors-nesting-selectors` and - :ref:`topics-selectors-relative-xpaths` in the :ref:`topics-selectors` - documentation + >>> quote = response.css("div.quote")[0] -Let's add this code to our spider:: +Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +using the ``quote`` object we just created:: - import scrapy - - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] - - def parse(self, response): - for sel in response.xpath('//ul/li'): - title = sel.xpath('a/text()').extract() - link = sel.xpath('a/@href').extract() - desc = sel.xpath('text()').extract() - print title, link, desc + >>> title = quote.css("span.text::text").extract_first() + >>> title + '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' + >>> author = quote.css("small.author::text").extract_first() + >>> author + 'Albert Einstein' -Now try crawling dmoz.org again and you'll see sites being printed -in your output. Run:: +Given that the tags are a list of strings, we can use the ``.extract()`` method +to get all of them:: - scrapy crawl dmoz + >>> tags = quote.css("div.tags a.tag::text").extract() + >>> tags + ['change', 'deep-thoughts', 'thinking', 'world'] -Using our item --------------- +Having figured out how to extract each bit, we can now iterate over all the +quotes elements and put them together into a Python dictionary:: -:class:`~scrapy.item.Item` objects are custom Python dicts; you can access the -values of their fields (attributes of the class we defined earlier) using the -standard dict syntax like:: + >>> for quote in response.css("div.quote"): + ... text = quote.css("span.text::text").extract_first() + ... author = quote.css("small.author::text").extract_first() + ... tags = quote.css("div.tags a.tag::text").extract() + ... print(dict(text=text, author=author, tags=tags)) + {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} + {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} + ... a few more of these, omitted for brevity + >>> - >>> item = DmozItem() - >>> item['title'] = 'Example title' - >>> item['title'] - 'Example title' +Extracting data in our spider +------------------------------ -So, in order to return the data we've scraped so far, the final code for our -Spider would be like this:: +Let's get back to our spider. Until now, it doesn't extract any data in +particular, just saves the whole HTML page to a local file. Let's integrate the +extraction logic above into our spider. + +A Scrapy spider typically generates many dictionaries containing the data +extracted from the page. To do that, we use the ``yield`` Python keyword +in the callback, as you can see below:: import scrapy - from tutorial.items import DmozItem - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', ] def parse(self, response): - for sel in response.xpath('//ul/li'): - item = DmozItem() - item['title'] = sel.xpath('a/text()').extract() - item['link'] = sel.xpath('a/@href').extract() - item['desc'] = sel.xpath('text()').extract() - yield item + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } -.. note:: You can find a fully-functional variant of this spider in the dirbot_ - project available at https://github.com/scrapy/dirbot +If you run this spider, it will output the extracted data with the log:: -Now crawling dmoz.org yields ``DmozItem`` objects:: + 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} + 2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} - [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n], - 'link': [u'http://gnosis.cx/TPiP/'], - 'title': [u'Text Processing in Python']} - [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], - 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], - 'title': [u'XML Processing with Python']} + +.. _storing-data: + +Storing the scraped data +======================== + +The simplest way to store the scraped data is by using :ref:`Feed exports +`, with the following command:: + + scrapy crawl quotes -o quotes.json + +That will generate an ``quotes.json`` file containing all scraped items, +serialized in `JSON`_. + +For historic reasons, Scrapy appends to a given file instead of overwriting +its contents. If you run this command twice without removing the file +before the second time, you'll end up with a broken JSON file. + +You can also used other formats, like `JSON Lines`_:: + + scrapy crawl quotes -o quotes.jl + +The `JSON Lines`_ format is useful because it's stream-like, you can easily +append new records to it. It doesn't have the same problem of JSON when you run +twice. Also, as each record is a separate line, you can process big files +without having to fit everything in memory, there are tools like `JQ`_ to help +doing that at the command-line. + +In small projects (like the one in this tutorial), that should be enough. +However, if you want to perform more complex things with the scraped items, you +can write an :ref:`Item Pipeline `. A placeholder file +for Item Pipelines has been set up for you when the project is created, in +``tutorial/pipelines.py``. Though you don't need to implement any item +pipelines if you just want to store the scraped items. + +.. _JSON Lines: http://jsonlines.org +.. _JQ: https://stedolan.github.io/jq Following links =============== -Let's say, instead of just scraping the stuff in *Books* and *Resources* pages, -you want everything that is under the `Python directory -`_. +Let's say, instead of just scraping the stuff from the first two pages +from http://quotes.toscrape.com, you want quotes from all the pages in the website. -Now that you know how to extract data from a page, why not extract the links -for the pages you are interested, follow them and then extract the data you -want for all of them? +Now that you know how to extract data from pages, let's see how to follow links +from them. -Here is a modification to our spider that does just that:: +First thing is to extract the link to the page we want to follow. Examining +our page, we can see there is a link to the next page with the following +markup: + +.. code-block:: html + + + +We can try extracting it in the shell:: + + >>> response.css('li.next a').extract_first() + 'Next ' + +This gets the anchor element, but we want the attribute ``href``. For that, +Scrapy supports a CSS extension that let's you select the attribute contents, +like this:: + + >>> response.css('li.next a::attr(href)').extract_first() + '/page/2/' + +Let's see now our spider modified to recursively follow the link to the next +page, extracting data from it:: import scrapy - from tutorial.items import DmozItem - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/", + 'http://quotes.toscrape.com/page/1/', ] def parse(self, response): - for href in response.css("ul.directory.dir-col > li > a::attr('href')"): - url = response.urljoin(href.extract()) - yield scrapy.Request(url, callback=self.parse_dir_contents) + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } - def parse_dir_contents(self, response): - for sel in response.xpath('//ul/li'): - item = DmozItem() - item['title'] = sel.xpath('a/text()').extract() - item['link'] = sel.xpath('a/@href').extract() - item['desc'] = sel.xpath('text()').extract() - yield item + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Now the `parse()` method only extracts the interesting links from the page, -builds a full absolute URL using the `response.urljoin` method (since the links can -be relative) and yields new requests to be sent later, registering as callback -the method `parse_dir_contents()` that will ultimately scrape the data we want. + +Now, after extracting the data, the ``parse()`` method looks for the link to +the next page, builds a full absolute URL using the +:meth:`~scrapy.http.Response.urljoin` method (since the links can be +relative) and yields a new request to the next page, registering itself as +callback to handle the data extraction for the next page and to keep the +crawling going through all the pages. What you see here is Scrapy's mechanism of following links: when you yield a Request in a callback method, Scrapy will schedule that request to be sent @@ -483,55 +543,120 @@ Using this, you can build complex crawlers that follow links according to rules you define, and extract different kinds of data depending on the page it's visiting. -A common pattern is a callback method that extracts some items, looks for a link -to follow to the next page and then yields a `Request` with the same callback -for it:: - - def parse_articles_follow_next_page(self, response): - for article in response.xpath("//article"): - item = ArticleItem() - - ... extract article data here - - yield item - - next_page = response.css("ul.navigation > li.next-page > a::attr('href')") - if next_page: - url = response.urljoin(next_page[0].extract()) - yield scrapy.Request(url, self.parse_articles_follow_next_page) - -This creates a sort of loop, following all the links to the next page until it -doesn't find one -- handy for crawling blogs, forums and other sites with +In our example, it creates a sort of loop, following all the links to the next page +until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. -Another common pattern is to build an item with data from more than one page, +More examples and patterns +-------------------------- + +Here is another spider that illustrates callbacks and following links, +this time for scraping author information:: + + + import scrapy + + + class AuthorSpider(scrapy.Spider): + name = 'author' + + start_urls = ['http://quotes.toscrape.com/'] + + def parse(self, response): + # follow links to author pages + for href in response.css('.author a::attr(href)').extract(): + yield scrapy.Request(response.urljoin(href), + callback=self.parse_author) + + # follow pagination links + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) + + def parse_author(self, response): + def extract_with_css(query): + return response.css(query).extract_first().strip() + + yield { + 'name': extract_with_css('h3.author-title::text'), + 'birthdate': extract_with_css('.author-born-date::text'), + 'bio': extract_with_css('.author-description::text'), + } + +This spider will start from the main page, it will follow all the links to the +authors pages calling the ``parse_author`` callback for each of them, and also +the pagination links with the ``parse`` callback as we saw before. + +The ``parse_author`` callback defines a helper function to extract and cleanup the +data from a CSS query and yields the Python dict with the author data. + +Another interesting thing this spider demonstrates is that, even if there are +many quotes from the same author, we don't need to worry about visiting the +same author page multiple times. By default, Scrapy filters out duplicated +requests to URLs already visited, avoiding the problem of hitting servers too +much because of a programming mistake. This can be configured by the setting +:setting:`DUPEFILTER_CLASS`. + +Hopefully by now you have a good understanding of how to use the mechanism +of following links and callbacks with Scrapy. + +As yet another example spider that leverages the mechanism of following links, +check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic +spider that implements a small rules engine that you can use to write your +crawlers on top of it. + +Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks `. -.. note:: - As an example spider that leverages this mechanism, check out the - :class:`~scrapy.spiders.CrawlSpider` class for a generic spider - that implements a small rules engine that you can use to write your - crawlers on top of it. +Using spider arguments +====================== -Storing the scraped data -======================== +You can provide command line arguments to your spiders by using the ``-a`` +option when running them:: -The simplest way to store the scraped data is by using :ref:`Feed exports -`, with the following command:: + scrapy crawl quotes -o quotes-humor.json -a tag=humor - scrapy crawl dmoz -o items.json +These arguments are passed to the Spider's ``__init__`` method and become +spider attributes by default. -That will generate an ``items.json`` file containing all scraped items, -serialized in `JSON`_. +In this example, the value provided for the ``tag`` argument will be available +via ``self.tag``. You can use this to make your spider fetch only quotes +with a specific tag, building the URL based on the argument:: -In small projects (like the one in this tutorial), that should be enough. -However, if you want to perform more complex things with the scraped items, you -can write an :ref:`Item Pipeline `. As with Items, a -placeholder file for Item Pipelines has been set up for you when the project is -created, in ``tutorial/pipelines.py``. Though you don't need to implement any item -pipelines if you just want to store the scraped items. + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + + def start_requests(self): + url = 'http://quotes.toscrape.com/' + tag = getattr(self, 'tag', None) + if tag is not None: + url = url + 'tag/' + tag + yield scrapy.Request(url, self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small a::text').extract_first(), + } + + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, self.parse) + + +If you pass the ``tag=humor`` argument to this spider, you'll notice that it +will only visit URLs from the ``humor`` tag, such as +``http://quotes.toscrape.com/tag/humor``. + +You can :ref:`learn more about handling spider arguments here `. Next steps ========== @@ -540,9 +665,10 @@ This tutorial covered only the basics of Scrapy, but there's a lot of other features not mentioned here. Check the :ref:`topics-whatelse` section in :ref:`intro-overview` chapter for a quick overview of the most important ones. -Then, we recommend you continue by playing with an example project (see -:ref:`intro-examples`), and then continue with the section -:ref:`section-basics`. +You can continue from the section :ref:`section-basics` to know more about the +command-line tool, spiders, selectors and other things the tutorial hasn't covered like +modeling the scraped data. If you prefer to play with an example project, check +the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON .. _dirbot: https://github.com/scrapy/dirbot diff --git a/docs/news.rst b/docs/news.rst index 985eeb796..0ee7642cf 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -46,8 +46,6 @@ Refactoring Documentation ~~~~~~~~~~~~~ -- :ref:`Overview ` and :ref:`tutorial ` - rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`). - Grammar fixes: :issue:`2128`, :issue:`1566`. - Download stats badge removed from README (:issue:`2160`). - New scrapy :ref:`architecture diagram ` (:issue:`2165`). @@ -56,6 +54,23 @@ Documentation - Add StackOverflow as a support channel (:issue:`2257`). +1.1.3 (2016-09-22) +------------------ + +Bug fixes +~~~~~~~~~ + +- Class attributes for subclasses of ``ImagesPipeline`` and ``FilesPipeline`` + work as they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`) + +Documentation +~~~~~~~~~~~~~ + +- :ref:`Overview ` and :ref:`tutorial ` + rewritten to use http://toscrape.com websites + (:issue:`2236`, :issue:`2249`, :issue:`2252`). + + 1.1.2 (2016-08-18) ------------------ diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ba0e2c61c..39e54ee99 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -16,20 +16,71 @@ components and an outline of the data flow that takes place inside the system below with links for more detailed information about them. The data flow is also described below. +.. _data-flow: + +Data flow +========= + .. image:: _images/scrapy_architecture_02.png :width: 700 :height: 470 :alt: Scrapy architecture +The data flow in Scrapy is controlled by the execution engine, and goes like +this: + +1. The :ref:`Engine ` gets the first URLs to crawl from the + :ref:`Spider `. + +2. The :ref:`Engine ` schedules the URLs in the + :ref:`Scheduler ` as Requests and asks for the + next URLs to crawl. + +3. The :ref:`Scheduler ` returns the next URLs to crawl + to the :ref:`Engine `. + +4. The :ref:`Engine ` sends the URLs to the + :ref:`Downloader `, passing through the + :ref:`Downloader Middleware ` + (request direction). + +5. Once the page finishes downloading the + :ref:`Downloader ` generates a Response (with + that page) and sends it to the Engine, passing through the + :ref:`Downloader Middleware ` + (response direction). + +6. The :ref:`Engine ` receives the Response from the + :ref:`Downloader ` and sends it to the + :ref:`Spider ` for processing, passing + through the :ref:`Spider Middleware ` + (input direction). + +7. The :ref:`Spider ` processes the Response and returns + scraped items and new Requests (to follow) to the + :ref:`Engine `, passing through the + :ref:`Spider Middleware ` (output direction). + +8. The :ref:`Engine ` sends processed items to + :ref:`Item Pipelines ` and processed Requests to + the :ref:`Scheduler `. + +9. The process repeats (from step 1) until there are no more requests from the + :ref:`Scheduler `. + Components ========== +.. _component-engine: + Scrapy Engine ------------- The engine is responsible for controlling the data flow between all components -of the system, and triggering events when certain actions occur. See the Data -Flow section below for more details. +of the system, and triggering events when certain actions occur. See the +:ref:`Data Flow ` section above for more details. + +.. _component-scheduler: Scheduler --------- @@ -37,12 +88,16 @@ Scheduler The Scheduler receives requests from the engine and enqueues them for feeding them later (also to the engine) when the engine requests them. +.. _component-downloader: + Downloader ---------- The Downloader is responsible for fetching web pages and feeding them to the engine which, in turn, feeds them to the spiders. +.. _component-spiders: + Spiders ------- @@ -50,6 +105,8 @@ Spiders are custom classes written by Scrapy users to parse responses and extract items (aka scraped items) from them or additional URLs (requests) to follow. For more information see :ref:`topics-spiders`. +.. _component-pipelines: + Item Pipeline ------------- @@ -58,6 +115,8 @@ extracted (or scraped) by the spiders. Typical tasks include cleansing, validation and persistence (like storing the item in a database). For more information see :ref:`topics-item-pipeline`. +.. _component-downloader-middleware: + Downloader middlewares ---------------------- @@ -76,6 +135,8 @@ Use a Downloader middleware if you need to do one of the following: For more information see :ref:`topics-downloader-middleware`. +.. _component-spider-middleware: + Spider middlewares ------------------ @@ -93,39 +154,6 @@ Use a Spider middleware if you need to For more information see :ref:`topics-spider-middleware`. -Data flow -========= - -The data flow in Scrapy is controlled by the execution engine, and goes like -this: - -1. The Engine gets the first URLs to crawl from the Spider. - -2. The Engine schedules the URLs in the Scheduler as Requests and asks for the - next URLs to crawl. - -3. The Scheduler returns the next URLs to crawl to the Engine. - -4. The Engine sends the URLs to the Downloader, passing through the - Downloader Middleware (request direction). - -5. Once the page finishes downloading the Downloader generates a Response (with - that page) and sends it to the Engine, passing through the Downloader - Middleware (response direction). - -6. The Engine receives the Response from the Downloader and sends it to the - Spider for processing, passing through the Spider Middleware (input direction). - -7. The Spider processes the Response and returns scraped items and new Requests - (to follow) to the Engine, passing through the Spider Middleware - (output direction). - -8. The Engine sends processed items to Item Pipelines and processed Requests to - the Scheduler. - -9. The process repeats (from step 1) until there are no more requests from the - Scheduler. - Event-driven networking ======================= diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 62ebc4c08..18d2f8084 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -35,6 +35,12 @@ And here is how to use it to send an e-mail (without attachments):: mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"]) +.. note:: + As shown in the example above, ``to`` and ``cc`` need to be lists + of email addresses, not single addresses, and even for one recipient, + i.e. ``to="someone@example.com"`` will not work. + + MailSender class reference ========================== diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 9f8d16d84..cc02369d4 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -60,7 +60,7 @@ remain disabled. Those components include: * Downloader middlewares * Spider middlewares -The exception must be raised in the component constructor. +The exception must be raised in the component's ``__init__`` method. NotSupported ------------ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index b9b4c2058..a31fe7423 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -27,9 +27,10 @@ Each item pipeline component is a Python class that must implement the following .. method:: process_item(self, item, spider) - This method is called for every item pipeline component and must either return - a dict with data, :class:`~scrapy.item.Item` (or any descendant class) object - or raise a :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer + This method is called for every item pipeline component. :meth:`process_item` + must either: return a dict with data, return an :class:`~scrapy.item.Item` + (or any descendant class) object, return a `Twisted Deferred`_ or raise + :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. :param item: the item scraped @@ -66,6 +67,8 @@ Additionally, they may also implement the following methods: :type crawler: :class:`~scrapy.crawler.Crawler` object +.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html + Item pipeline example ===================== @@ -163,6 +166,52 @@ method and how to clean up the resources properly. .. _MongoDB: https://www.mongodb.org/ .. _pymongo: https://api.mongodb.org/python/current/ + +Take screenshot of item +----------------------- + +This example demonstrates how to return Deferred_ from :meth:`process_item` method. +It uses Splash_ to render screenshot of item url. Pipeline +makes request to locally running instance of Splash_. After request is downloaded +and Deferred callback fires, it saves item to a file and adds filename to an item. + +:: + + import scrapy + + + class ScreenshotPipeline(object): + """Pipeline that uses Splash to render screenshot of + every Scrapy item.""" + + SPLASH_URL = "http://localhost:8050/render.png?url={}" + + def process_item(self, item, spider): + item_url = item["url"] + screenshot_url = self.SPLASH_URL.format(item_url) + request = scrapy.Request(screenshot_url) + dfd = spider.crawler.engine.download(request, spider) + dfd.addBoth(self.return_item, item) + return dfd + + def return_item(self, response, item): + if response.status != 200: + # Error happened, return item. + return item + + # Save screenshot to file. + filename = "item_file_name.png" + with open(filename, "wb") as f: + f.write(response.body) + + # Store filename in item. + item["screenshot_filename"] = filename + return item + + +.. _Splash: http://splash.readthedocs.io/en/stable/ +.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html + Duplicates filter ----------------- diff --git a/docs/versioning.rst b/docs/versioning.rst index 8e7908762..0421ba544 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -7,22 +7,33 @@ Versioning and API Stability Versioning ========== -Scrapy uses the `odd-numbered versions for development releases`_. - There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very large changes. * *B* is the release number. This will include many changes including features - and things that possibly break backwards compatibility. Even Bs will be - stable branches, and odd Bs will be development. + and things that possibly break backwards compatibility, although we strive to + keep theses cases at a minimum. * *C* is the bugfix release number. +Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, +and may require special attention before upgrading. + +Development releases do not follow 3-numbers version and are generally +released as ``dev`` suffixed versions, e.g. ``1.3dev``. + +.. note:: + With Scrapy 0.* series, Scrapy used `odd-numbered versions for development releases`_. + This is not the case anymore from Scrapy 1.0 onwards. + + Starting with Scrapy 1.0, all releases should be considered production-ready. + For example: -* *0.14.1* is the first bugfix release of the *0.14* series (safe to use in +* *1.1.1* is the first bugfix release of the *1.1* series (safe to use in production) + API Stability ============= diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6a8978415..5264982b6 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -121,8 +121,8 @@ class Command(ScrapyCommand): def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): for rule in spider.rules: - if rule.link_extractor.matches(response.url) and rule.callback: - return rule.callback + if rule.link_extractor.matches(response.url): + return rule.callback or "parse" else: logger.error('No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', @@ -166,6 +166,11 @@ class Command(ScrapyCommand): if not cb: if opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) + + if not cb: + logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', + {'url': response.url, 'spider': spider.name}) + return else: cb = 'parse' diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index dcd6fb989..a54b4daf0 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -89,8 +89,8 @@ class Scheduler(object): msg = ("Unable to serialize request: %(request)s - reason:" " %(reason)s - no more unserializable requests will be" " logged (stats being collected)") - logger.error(msg, {'request': request, 'reason': e}, - exc_info=True, extra={'spider': self.spider}) + logger.warning(msg, {'request': request, 'reason': e}, + exc_info=True, extra={'spider': self.spider}) self.logunser = False self.stats.inc_value('scheduler/unserializable', spider=self.spider) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index b01bab76d..98c87aa9c 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -43,7 +43,7 @@ class HttpProxyMiddleware(object): return creds, proxy_url def process_request(self, request, spider): - # ignore if proxy is already seted + # ignore if proxy is already set if 'proxy' in request.meta: return diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c3fc66de5..85d328528 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -170,6 +170,7 @@ class FeedExporter(object): if not self._exporter_supported(self.format): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') + self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,14 +189,18 @@ class FeedExporter(object): file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, encoding=self.export_encoding) - exporter.start_exporting() + if self.store_empty: + exporter.start_exporting() + self._exporting = True self.slot = SpiderSlot(file, exporter, storage, uri) def close_spider(self, spider): slot = self.slot if not slot.itemcount and not self.store_empty: return - slot.exporter.finish_exporting() + if self._exporting: + slot.exporter.finish_exporting() + self._exporting = False logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" log_args = {'format': self.format, 'itemcount': slot.itemcount, @@ -210,6 +215,9 @@ class FeedExporter(object): def item_scraped(self, item, spider): slot = self.slot + if not self._exporting: + slot.exporter.start_exporting() + self._exporting = True slot.exporter.export_item(item) slot.itemcount += 1 return item diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8cdc548f6..843b4d3ec 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -233,7 +233,8 @@ class FilesPipeline(MediaPipeline): cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, - base_class_name=cls_name) + base_class_name=cls_name, + settings=settings) self.expires = settings.getint( resolve('FILES_EXPIRES'), self.EXPIRES ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index af5825c0b..5796bfb80 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -55,7 +55,8 @@ class ImagesPipeline(FilesPipeline): settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, - base_class_name="ImagesPipeline") + base_class_name="ImagesPipeline", + settings=settings) self.expires = settings.getint( resolve("IMAGES_EXPIRES"), self.EXPIRES ) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 82b4b462e..57f70499e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -28,7 +28,8 @@ class MediaPipeline(object): self.download_func = download_func - def _key_for_pipe(self, key, base_class_name=None): + def _key_for_pipe(self, key, base_class_name=None, + settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -38,9 +39,11 @@ class MediaPipeline(object): 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - if class_name == base_class_name or not base_class_name: + formatted_key = "{}_{}".format(class_name.upper(), key) + if class_name == base_class_name or not base_class_name \ + or (settings and not settings.get(formatted_key)): return key - return "{}_{}".format(class_name.upper(), key) + return formatted_key @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 075f757af..4a2d5bf52 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -25,6 +25,7 @@ class ResponseTypes(object): 'application/xml': 'scrapy.http.XmlResponse', 'application/json': 'scrapy.http.TextResponse', 'application/x-json': 'scrapy.http.TextResponse', + 'application/json-amazonui-streaming': 'scrapy.http.TextResponse', 'application/javascript': 'scrapy.http.TextResponse', 'application/x-javascript': 'scrapy.http.TextResponse', 'text/xml': 'scrapy.http.XmlResponse', diff --git a/scrapy/shell.py b/scrapy/shell.py index 099e1af0a..183ee1f70 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -115,6 +115,9 @@ class Shell(object): self.populate_vars(response, request, spider) def populate_vars(self, response=None, request=None, spider=None): + import scrapy + + self.vars['scrapy'] = scrapy self.vars['crawler'] = self.crawler self.vars['item'] = self.item_class() self.vars['settings'] = self.crawler.settings @@ -136,6 +139,7 @@ class Shell(object): def get_help(self): b = [] b.append("Available Scrapy objects:") + b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index a712df30e..567fd51bc 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -13,7 +13,9 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - shell = InteractiveShellEmbed( + # Always use .instace() to ensure _instance propagation to all parents + # this is needed for completion works well for new imports + shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config) shell() return wrapper diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cc2f0b164..51f303216 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -145,6 +145,10 @@ class StreamLogger(object): for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) + def flush(self): + for h in self.logger.handlers: + h.flush() + class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py new file mode 100644 index 000000000..b6d6db9ee --- /dev/null +++ b/tests/test_command_parse.py @@ -0,0 +1,156 @@ +from os.path import join, abspath +from twisted.trial import unittest +from twisted.internet import defer +from scrapy.utils.testsite import SiteTest +from scrapy.utils.testproc import ProcessTest +from scrapy.utils.python import to_native_str +from tests.test_commands import CommandTest + + +class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + command = 'parse' + + def setUp(self): + super(ParseCommandTest, self).setUp() + self.spider_name = 'parse_spider' + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(""" +import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class MySpider(scrapy.Spider): + name = '{0}' + + def parse(self, response): + if getattr(self, 'test_arg', None): + self.logger.debug('It Works!') + return [scrapy.Item(), dict(foo='bar')] + + +class MyGoodCrawlSpider(CrawlSpider): + name = 'goodcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + Rule(LinkExtractor(allow=r'/text'), follow=True), + ) + + def parse_item(self, response): + return [scrapy.Item(), dict(foo='bar')] + + def parse(self, response): + return [scrapy.Item(), dict(nomatch='default')] + + +class MyBadCrawlSpider(CrawlSpider): + '''Spider which doesn't define a parse_item callback while using it in a rule.''' + name = 'badcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + ) + + def parse(self, response): + return [scrapy.Item(), dict(foo='bar')] +""".format(self.spider_name)) + + fname = abspath(join(self.proj_mod_path, 'pipelines.py')) + with open(fname, 'w') as f: + f.write(""" +import logging + +class MyPipeline(object): + component_name = 'my_pipeline' + + def process_item(self, item, spider): + logging.info('It Works!') + return item +""") + + fname = abspath(join(self.proj_mod_path, 'settings.py')) + with open(fname, 'a') as f: + f.write(""" +ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} +""" % self.project_name) + + @defer.inlineCallbacks + def test_spider_arguments(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-a', 'test_arg=1', + '-c', 'parse', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_pipelines(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--pipelines', + '-c', 'parse', + self.url('/html')]) + self.assertIn("INFO: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_parse_items_no_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_wrong_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_callback_set(self): + """If a rule matches the URL, use it's defined callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_default_callback(self): + """If a rule match but it has no callback set, use the 'parse' callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ) + self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_spider_with_no_rules_attribute(self): + """Using -r with a spider with no rule should not produce items.""" + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_missing_callback(self): + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_no_matching_rule(self): + """The requested URL has no matching rule, so no items should be scraped""" + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index c532fc0d8..7bb7439d6 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -56,6 +56,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, _ = yield self.execute(['-c', code.format(url)]) self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks + def test_scrapy_import(self): + url = self.url('/text') + code = "fetch(scrapy.Request('{0}'))" + errcode, out, _ = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks def test_local_file(self): filepath = join(tests_datadir, 'test_site/index.html') diff --git a/tests/test_commands.py b/tests/test_commands.py index d25045cb9..b507c46bc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -246,71 +246,6 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' - - def setUp(self): - super(ParseCommandTest, self).setUp() - self.spider_name = 'parse_spider' - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(""" -import scrapy - -class MySpider(scrapy.Spider): - name = '{0}' - - def parse(self, response): - if getattr(self, 'test_arg', None): - self.logger.debug('It Works!') - return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) - - fname = abspath(join(self.proj_mod_path, 'pipelines.py')) - with open(fname, 'w') as f: - f.write(""" -import logging - -class MyPipeline(object): - component_name = 'my_pipeline' - - def process_item(self, item, spider): - logging.info('It Works!') - return item -""") - - fname = abspath(join(self.proj_mod_path, 'settings.py')) - with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) - - @defer.inlineCallbacks - def test_spider_arguments(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-a', 'test_arg=1', - '-c', 'parse', - self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_pipelines(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--pipelines', - '-c', 'parse', - self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_parse_items(self): - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] - ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) - - - class BenchCommandTest(CommandTest): def test_run(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 353b21927..e93d2bafb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + class TestSpider(scrapy.Spider): + name = 'testspider' + start_urls = ['http://localhost:8998/'] + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, ordered=False) + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + formats = ('json', + 'jsonlines', + 'xml', + 'csv',) + + for fmt in formats: + settings = {'FEED_FORMAT': fmt} + data = yield self.exported_no_data(settings) + self.assertEqual(data, b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[\n\n]'), + ('jsonlines', b''), + ('xml', b'\n'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + data = yield self.exported_no_data(settings) + self.assertEqual(data, expctd) + @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase): def test_export_encoding(self): items = [dict({'foo': u'Test\xd6'})] header = ['foo'] - + formats = { 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - + for format in formats: settings = {'FEED_FORMAT': format} data = yield self.exported_data(items, settings) self.assertEqual(formats[format], data) - + formats = { 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - + for format in formats: settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} data = yield self.exported_data(items, settings) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bda2a2199..157c21a89 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -255,16 +255,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If file settings are defined but they are not defined for subclass class attributes - should be preserved. + If file settings are defined but they are not defined for subclass + settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) + setting_value = settings.get(settings_attr) self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + self.assertEqual(value, setting_value) def test_no_custom_settings_for_subclasses(self): """ @@ -321,6 +322,24 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(pipeline.files_urls_field, "that") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of FilesPipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(FilesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), + expected_value) + + class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8286582de..6c1976b63 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -309,17 +309,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If image settings are defined but they are not defined for subclass class attributes - should be preserved. + If image settings are defined but they are not defined for subclass default + values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: - # Instance attribute (lowercase) must be equal to class attribute (uppercase). + # Instance attribute (lowercase) must be equal to + # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + setings_value = settings.get(settings_attr) + self.assertEqual(value, setings_value) def test_no_custom_settings_for_subclasses(self): """ @@ -370,11 +372,26 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir})) self.assertEqual(pipeline.images_result_field, "something_else") self.assertEqual(pipeline.images_urls_field, "something") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of ImagePipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(ImagesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr in self.img_cls_attribute_names: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), + expected_value) def _create_image(format, *a, **kw): buf = TemporaryFile() diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 118136ac4..f89042b3d 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -43,6 +43,7 @@ class ResponseTypesTest(unittest.TestCase): ('application/xml; charset=UTF-8', XmlResponse), ('application/octet-stream', Response), ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse), + ('application/json-amazonui-streaming;charset=UTF-8', TextResponse), ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) diff --git a/tox.ini b/tox.ini index f6de64b27..812302b4c 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,20 @@ deps = zope.interface==3.6.1 -rtests/requirements.txt +[testenv:jessie] +# https://packages.debian.org/en/jessie/python/ +# https://packages.debian.org/en/jessie/zope/ +basepython = python2.7 +deps = + pyOpenSSL==0.14 + lxml==3.4.0 + Twisted==14.0.2 + boto==2.34.0 + Pillow==2.6.1 + cssselect==0.9.1 + zope.interface==4.1.1 + -rtests/requirements.txt + [testenv:trunk] basepython = python2.7 commands =