diff --git a/.travis.yml b/.travis.yml index c58ab39a5..506f3779b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ branches: - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ env: - TOXENV=py27 - - TOXENV=precise + - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 - TOXENV=docs diff --git a/docs/index.rst b/docs/index.rst index 4cb3eb741..b4272e47f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,13 +13,15 @@ Having trouble? We'd like to help! * Try the :doc:`FAQ ` -- it's got answers to some common questions. * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`. +* Ask or search questions in `StackOverflow using the scrapy tag`_, * Search for information in the `archives of the scrapy-users mailing list`_, or `post a question`_. -* Ask a question in the `#scrapy IRC channel`_. +* Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. .. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _post a question: https://groups.google.com/forum/#!forum/scrapy-users +.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6c1291c1f..7195017ff 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -19,74 +19,69 @@ Walk-through of an example spider In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. -So, here's the code for a spider that follows the links to the top -voted questions on StackOverflow and scrapes some data from each page:: +Here's the code for a spider that scrapes famous quotes from website +http://quotes.toscrape.com, following the pagination:: import scrapy - class StackOverflowSpider(scrapy.Spider): - name = 'stackoverflow' - start_urls = ['http://stackoverflow.com/questions?sort=votes'] + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/tag/humor/', + ] def parse(self, response): - for href in response.css('.question-summary h3 a::attr(href)'): - full_url = response.urljoin(href.extract()) - yield scrapy.Request(full_url, callback=self.parse_question) + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } - def parse_question(self, response): - yield { - 'title': response.css('h1 a::text').extract_first(), - 'votes': response.css('.question .vote-count-post::text').extract_first(), - 'body': response.css('.question .post-text').extract_first(), - 'tags': response.css('.question .post-tag::text').extract(), - 'link': response.url, - } + next_page = response.css('li.next a::attr("href")').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Put this in a file, name it to something like ``stackoverflow_spider.py`` +Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json + scrapy runspider quotes_spider.py -o quotes.json -When this finishes you will have in the ``top-stackoverflow-questions.json`` file -a list of the most upvoted questions in StackOverflow in JSON format, containing the -title, link, number of upvotes, a list of the tags and the question content in HTML, -looking like this (reformatted for easier reading):: +When this finishes you will have in the ``quotes.json`` file a list of the +quotes in JSON format, containing text and author, looking like this (reformatted +here for better readability):: [{ - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array", - "tags": ["java", "c++", "performance", "optimization"], - "title": "Why is processing a sorted array faster than an unsorted array?", - "votes": "9924" + "author": "Jane Austen", + "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" }, { - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule", - "tags": ["git", "git-submodules"], - "title": "How do I remove a Git submodule?", - "votes": "1764" + "author": "Groucho Marx", + "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" + }, + { + "author": "Steve Martin", + "text": "\u201cA day without sunshine is like, you know, night.\u201d" }, ...] - What just happened? ------------------- -When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a +When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked for a Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for StackOverflow top questions page) +attribute (in this case, only the URL for quotes in *humor* category) and called the default callback method ``parse``, passing the response object as -an argument. In the ``parse`` callback we extract the links to the -question pages using a CSS Selector with a custom extension that allows to get -the value for an attribute. Then we yield a few more requests to be sent, -registering the method ``parse_question`` as the callback to be called for each -of them as they finish. +an argument. In the ``parse`` callback, we loop through the quote elements +using a CSS Selector, yield a Python dict with the extracted quote text and author, +look for a link to the next page and schedule another request using the same +``parse`` method as callback. Here you notice one of the main advantages about Scrapy: requests are :ref:`scheduled and processed asynchronously `. This @@ -103,10 +98,6 @@ each request, limiting amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension ` that tries to figure out these automatically. -Finally, the ``parse_question`` callback scrapes the question data for each -page yielding a dict, which Scrapy then collects and writes to a JSON file as -requested in the command line. - .. note:: This is using :ref:`feed exports ` to generate the @@ -145,12 +136,13 @@ scraping easy and efficient, such as: :ref:`pipelines `). * Wide range of built-in extensions and middlewares for handling: - * cookies and session handling - * HTTP features like compression, authentication, caching - * user-agent spoofing - * robots.txt - * crawl depth restriction - * and more + + - cookies and session handling + - HTTP features like compression, authentication, caching + - user-agent spoofing + - robots.txt + - crawl depth restriction + - and more * A :ref:`Telnet console ` for hooking into a Python console running inside your Scrapy process, to introspect and debug your @@ -165,8 +157,8 @@ What's next? ============ The next steps for you are to :ref:`install Scrapy `, -:ref:`follow through the tutorial ` to learn how to organize -your code in Scrapy projects and `join the community`_. Thanks for your +:ref:`follow through the tutorial ` to learn how to create +a full-blown Scrapy project and `join the community`_. Thanks for your interest! .. _join the community: http://scrapy.org/community/ diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ba0e2c61c..39e54ee99 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -16,20 +16,71 @@ components and an outline of the data flow that takes place inside the system below with links for more detailed information about them. The data flow is also described below. +.. _data-flow: + +Data flow +========= + .. image:: _images/scrapy_architecture_02.png :width: 700 :height: 470 :alt: Scrapy architecture +The data flow in Scrapy is controlled by the execution engine, and goes like +this: + +1. The :ref:`Engine ` gets the first URLs to crawl from the + :ref:`Spider `. + +2. The :ref:`Engine ` schedules the URLs in the + :ref:`Scheduler ` as Requests and asks for the + next URLs to crawl. + +3. The :ref:`Scheduler ` returns the next URLs to crawl + to the :ref:`Engine `. + +4. The :ref:`Engine ` sends the URLs to the + :ref:`Downloader `, passing through the + :ref:`Downloader Middleware ` + (request direction). + +5. Once the page finishes downloading the + :ref:`Downloader ` generates a Response (with + that page) and sends it to the Engine, passing through the + :ref:`Downloader Middleware ` + (response direction). + +6. The :ref:`Engine ` receives the Response from the + :ref:`Downloader ` and sends it to the + :ref:`Spider ` for processing, passing + through the :ref:`Spider Middleware ` + (input direction). + +7. The :ref:`Spider ` processes the Response and returns + scraped items and new Requests (to follow) to the + :ref:`Engine `, passing through the + :ref:`Spider Middleware ` (output direction). + +8. The :ref:`Engine ` sends processed items to + :ref:`Item Pipelines ` and processed Requests to + the :ref:`Scheduler `. + +9. The process repeats (from step 1) until there are no more requests from the + :ref:`Scheduler `. + Components ========== +.. _component-engine: + Scrapy Engine ------------- The engine is responsible for controlling the data flow between all components -of the system, and triggering events when certain actions occur. See the Data -Flow section below for more details. +of the system, and triggering events when certain actions occur. See the +:ref:`Data Flow ` section above for more details. + +.. _component-scheduler: Scheduler --------- @@ -37,12 +88,16 @@ Scheduler The Scheduler receives requests from the engine and enqueues them for feeding them later (also to the engine) when the engine requests them. +.. _component-downloader: + Downloader ---------- The Downloader is responsible for fetching web pages and feeding them to the engine which, in turn, feeds them to the spiders. +.. _component-spiders: + Spiders ------- @@ -50,6 +105,8 @@ Spiders are custom classes written by Scrapy users to parse responses and extract items (aka scraped items) from them or additional URLs (requests) to follow. For more information see :ref:`topics-spiders`. +.. _component-pipelines: + Item Pipeline ------------- @@ -58,6 +115,8 @@ extracted (or scraped) by the spiders. Typical tasks include cleansing, validation and persistence (like storing the item in a database). For more information see :ref:`topics-item-pipeline`. +.. _component-downloader-middleware: + Downloader middlewares ---------------------- @@ -76,6 +135,8 @@ Use a Downloader middleware if you need to do one of the following: For more information see :ref:`topics-downloader-middleware`. +.. _component-spider-middleware: + Spider middlewares ------------------ @@ -93,39 +154,6 @@ Use a Spider middleware if you need to For more information see :ref:`topics-spider-middleware`. -Data flow -========= - -The data flow in Scrapy is controlled by the execution engine, and goes like -this: - -1. The Engine gets the first URLs to crawl from the Spider. - -2. The Engine schedules the URLs in the Scheduler as Requests and asks for the - next URLs to crawl. - -3. The Scheduler returns the next URLs to crawl to the Engine. - -4. The Engine sends the URLs to the Downloader, passing through the - Downloader Middleware (request direction). - -5. Once the page finishes downloading the Downloader generates a Response (with - that page) and sends it to the Engine, passing through the Downloader - Middleware (response direction). - -6. The Engine receives the Response from the Downloader and sends it to the - Spider for processing, passing through the Spider Middleware (input direction). - -7. The Spider processes the Response and returns scraped items and new Requests - (to follow) to the Engine, passing through the Spider Middleware - (output direction). - -8. The Engine sends processed items to Item Pipelines and processed Requests to - the Scheduler. - -9. The process repeats (from step 1) until there are no more requests from the - Scheduler. - Event-driven networking ======================= diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6a8978415..5264982b6 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -121,8 +121,8 @@ class Command(ScrapyCommand): def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): for rule in spider.rules: - if rule.link_extractor.matches(response.url) and rule.callback: - return rule.callback + if rule.link_extractor.matches(response.url): + return rule.callback or "parse" else: logger.error('No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', @@ -166,6 +166,11 @@ class Command(ScrapyCommand): if not cb: if opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) + + if not cb: + logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', + {'url': response.url, 'spider': spider.name}) + return else: cb = 'parse' diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index dcd6fb989..a54b4daf0 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -89,8 +89,8 @@ class Scheduler(object): msg = ("Unable to serialize request: %(request)s - reason:" " %(reason)s - no more unserializable requests will be" " logged (stats being collected)") - logger.error(msg, {'request': request, 'reason': e}, - exc_info=True, extra={'spider': self.spider}) + logger.warning(msg, {'request': request, 'reason': e}, + exc_info=True, extra={'spider': self.spider}) self.logunser = False self.stats.inc_value('scheduler/unserializable', spider=self.spider) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c3fc66de5..85d328528 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -170,6 +170,7 @@ class FeedExporter(object): if not self._exporter_supported(self.format): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') + self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,14 +189,18 @@ class FeedExporter(object): file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, encoding=self.export_encoding) - exporter.start_exporting() + if self.store_empty: + exporter.start_exporting() + self._exporting = True self.slot = SpiderSlot(file, exporter, storage, uri) def close_spider(self, spider): slot = self.slot if not slot.itemcount and not self.store_empty: return - slot.exporter.finish_exporting() + if self._exporting: + slot.exporter.finish_exporting() + self._exporting = False logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" log_args = {'format': self.format, 'itemcount': slot.itemcount, @@ -210,6 +215,9 @@ class FeedExporter(object): def item_scraped(self, item, spider): slot = self.slot + if not self._exporting: + slot.exporter.start_exporting() + self._exporting = True slot.exporter.export_item(item) slot.itemcount += 1 return item diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8cdc548f6..843b4d3ec 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -233,7 +233,8 @@ class FilesPipeline(MediaPipeline): cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, - base_class_name=cls_name) + base_class_name=cls_name, + settings=settings) self.expires = settings.getint( resolve('FILES_EXPIRES'), self.EXPIRES ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index af5825c0b..5796bfb80 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -55,7 +55,8 @@ class ImagesPipeline(FilesPipeline): settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, - base_class_name="ImagesPipeline") + base_class_name="ImagesPipeline", + settings=settings) self.expires = settings.getint( resolve("IMAGES_EXPIRES"), self.EXPIRES ) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 82b4b462e..57f70499e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -28,7 +28,8 @@ class MediaPipeline(object): self.download_func = download_func - def _key_for_pipe(self, key, base_class_name=None): + def _key_for_pipe(self, key, base_class_name=None, + settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -38,9 +39,11 @@ class MediaPipeline(object): 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - if class_name == base_class_name or not base_class_name: + formatted_key = "{}_{}".format(class_name.upper(), key) + if class_name == base_class_name or not base_class_name \ + or (settings and not settings.get(formatted_key)): return key - return "{}_{}".format(class_name.upper(), key) + return formatted_key @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/shell.py b/scrapy/shell.py index 099e1af0a..183ee1f70 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -115,6 +115,9 @@ class Shell(object): self.populate_vars(response, request, spider) def populate_vars(self, response=None, request=None, spider=None): + import scrapy + + self.vars['scrapy'] = scrapy self.vars['crawler'] = self.crawler self.vars['item'] = self.item_class() self.vars['settings'] = self.crawler.settings @@ -136,6 +139,7 @@ class Shell(object): def get_help(self): b = [] b.append("Available Scrapy objects:") + b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cc2f0b164..51f303216 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -145,6 +145,10 @@ class StreamLogger(object): for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) + def flush(self): + for h in self.logger.handlers: + h.flush() + class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py new file mode 100644 index 000000000..b6d6db9ee --- /dev/null +++ b/tests/test_command_parse.py @@ -0,0 +1,156 @@ +from os.path import join, abspath +from twisted.trial import unittest +from twisted.internet import defer +from scrapy.utils.testsite import SiteTest +from scrapy.utils.testproc import ProcessTest +from scrapy.utils.python import to_native_str +from tests.test_commands import CommandTest + + +class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + command = 'parse' + + def setUp(self): + super(ParseCommandTest, self).setUp() + self.spider_name = 'parse_spider' + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(""" +import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class MySpider(scrapy.Spider): + name = '{0}' + + def parse(self, response): + if getattr(self, 'test_arg', None): + self.logger.debug('It Works!') + return [scrapy.Item(), dict(foo='bar')] + + +class MyGoodCrawlSpider(CrawlSpider): + name = 'goodcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + Rule(LinkExtractor(allow=r'/text'), follow=True), + ) + + def parse_item(self, response): + return [scrapy.Item(), dict(foo='bar')] + + def parse(self, response): + return [scrapy.Item(), dict(nomatch='default')] + + +class MyBadCrawlSpider(CrawlSpider): + '''Spider which doesn't define a parse_item callback while using it in a rule.''' + name = 'badcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + ) + + def parse(self, response): + return [scrapy.Item(), dict(foo='bar')] +""".format(self.spider_name)) + + fname = abspath(join(self.proj_mod_path, 'pipelines.py')) + with open(fname, 'w') as f: + f.write(""" +import logging + +class MyPipeline(object): + component_name = 'my_pipeline' + + def process_item(self, item, spider): + logging.info('It Works!') + return item +""") + + fname = abspath(join(self.proj_mod_path, 'settings.py')) + with open(fname, 'a') as f: + f.write(""" +ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} +""" % self.project_name) + + @defer.inlineCallbacks + def test_spider_arguments(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-a', 'test_arg=1', + '-c', 'parse', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_pipelines(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--pipelines', + '-c', 'parse', + self.url('/html')]) + self.assertIn("INFO: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_parse_items_no_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_wrong_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_callback_set(self): + """If a rule matches the URL, use it's defined callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_default_callback(self): + """If a rule match but it has no callback set, use the 'parse' callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ) + self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_spider_with_no_rules_attribute(self): + """Using -r with a spider with no rule should not produce items.""" + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_missing_callback(self): + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_no_matching_rule(self): + """The requested URL has no matching rule, so no items should be scraped""" + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index c532fc0d8..7bb7439d6 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -56,6 +56,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, _ = yield self.execute(['-c', code.format(url)]) self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks + def test_scrapy_import(self): + url = self.url('/text') + code = "fetch(scrapy.Request('{0}'))" + errcode, out, _ = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks def test_local_file(self): filepath = join(tests_datadir, 'test_site/index.html') diff --git a/tests/test_commands.py b/tests/test_commands.py index d25045cb9..b507c46bc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -246,71 +246,6 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' - - def setUp(self): - super(ParseCommandTest, self).setUp() - self.spider_name = 'parse_spider' - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(""" -import scrapy - -class MySpider(scrapy.Spider): - name = '{0}' - - def parse(self, response): - if getattr(self, 'test_arg', None): - self.logger.debug('It Works!') - return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) - - fname = abspath(join(self.proj_mod_path, 'pipelines.py')) - with open(fname, 'w') as f: - f.write(""" -import logging - -class MyPipeline(object): - component_name = 'my_pipeline' - - def process_item(self, item, spider): - logging.info('It Works!') - return item -""") - - fname = abspath(join(self.proj_mod_path, 'settings.py')) - with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) - - @defer.inlineCallbacks - def test_spider_arguments(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-a', 'test_arg=1', - '-c', 'parse', - self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_pipelines(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--pipelines', - '-c', 'parse', - self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_parse_items(self): - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] - ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) - - - class BenchCommandTest(CommandTest): def test_run(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 353b21927..e93d2bafb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + class TestSpider(scrapy.Spider): + name = 'testspider' + start_urls = ['http://localhost:8998/'] + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, ordered=False) + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + formats = ('json', + 'jsonlines', + 'xml', + 'csv',) + + for fmt in formats: + settings = {'FEED_FORMAT': fmt} + data = yield self.exported_no_data(settings) + self.assertEqual(data, b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[\n\n]'), + ('jsonlines', b''), + ('xml', b'\n'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + data = yield self.exported_no_data(settings) + self.assertEqual(data, expctd) + @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase): def test_export_encoding(self): items = [dict({'foo': u'Test\xd6'})] header = ['foo'] - + formats = { 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - + for format in formats: settings = {'FEED_FORMAT': format} data = yield self.exported_data(items, settings) self.assertEqual(formats[format], data) - + formats = { 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - + for format in formats: settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} data = yield self.exported_data(items, settings) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bda2a2199..157c21a89 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -255,16 +255,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If file settings are defined but they are not defined for subclass class attributes - should be preserved. + If file settings are defined but they are not defined for subclass + settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) + setting_value = settings.get(settings_attr) self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + self.assertEqual(value, setting_value) def test_no_custom_settings_for_subclasses(self): """ @@ -321,6 +322,24 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(pipeline.files_urls_field, "that") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of FilesPipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(FilesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), + expected_value) + + class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8286582de..6c1976b63 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -309,17 +309,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If image settings are defined but they are not defined for subclass class attributes - should be preserved. + If image settings are defined but they are not defined for subclass default + values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: - # Instance attribute (lowercase) must be equal to class attribute (uppercase). + # Instance attribute (lowercase) must be equal to + # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + setings_value = settings.get(settings_attr) + self.assertEqual(value, setings_value) def test_no_custom_settings_for_subclasses(self): """ @@ -370,11 +372,26 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir})) self.assertEqual(pipeline.images_result_field, "something_else") self.assertEqual(pipeline.images_urls_field, "something") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of ImagePipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(ImagesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr in self.img_cls_attribute_names: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), + expected_value) def _create_image(format, *a, **kw): buf = TemporaryFile() diff --git a/tox.ini b/tox.ini index f6de64b27..812302b4c 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,20 @@ deps = zope.interface==3.6.1 -rtests/requirements.txt +[testenv:jessie] +# https://packages.debian.org/en/jessie/python/ +# https://packages.debian.org/en/jessie/zope/ +basepython = python2.7 +deps = + pyOpenSSL==0.14 + lxml==3.4.0 + Twisted==14.0.2 + boto==2.34.0 + Pillow==2.6.1 + cssselect==0.9.1 + zope.interface==4.1.1 + -rtests/requirements.txt + [testenv:trunk] basepython = python2.7 commands =