From 968141cd42b108e4343e3e6775ea223ad8d9b4b0 Mon Sep 17 00:00:00 2001 From: alexanderlukanin13 Date: Sat, 11 Jan 2014 14:30:27 +0600 Subject: [PATCH] test_command_deploy, test_contrib_linkextractors --- requirements.txt | 1 + scrapy/tests/test_command_deploy.py | 60 +++++++++++++++++++ scrapy/tests/test_commands.py | 27 +++++++++ scrapy/tests/test_contrib_linkextractors.py | 18 ++++++ .../test_downloadermiddleware_robotstxt.py | 27 +++++++++ 5 files changed, 133 insertions(+) create mode 100644 scrapy/tests/test_command_deploy.py create mode 100644 scrapy/tests/test_downloadermiddleware_robotstxt.py diff --git a/requirements.txt b/requirements.txt index e070a183e..0df9a558c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pyOpenSSL cssselect>=0.9 w3lib>=1.2 queuelib +six>=1.5.2 diff --git a/scrapy/tests/test_command_deploy.py b/scrapy/tests/test_command_deploy.py new file mode 100644 index 000000000..e6f008099 --- /dev/null +++ b/scrapy/tests/test_command_deploy.py @@ -0,0 +1,60 @@ +import re +from six.moves import cStringIO +from mock import patch +from twisted.trial import unittest + +import scrapy.commands.deploy as deploy +from scrapy.exceptions import UsageError +from scrapy.tests.test_commands import CommandMockTest + + +@patch('urllib2.install_opener') +class DeployMockTest(CommandMockTest, unittest.TestCase): + + Command = deploy.Command + + def test_wrong_target(self, install_opener_mock): + self.assertRaisesRegexp(UsageError, r'^Unknown target: wrong$', self.run_command, ['wrong']) + + @patch('scrapy.commands.deploy._get_targets', autospec=True) + def test_list_targets(self, _get_targets_mock, install_opener_mock): + _get_targets_mock.return_value = { + 'target1': {'url': 'url1'}, + 'target2': {'url': 'url2'}, + } + stream = cStringIO() + with patch('sys.stdout') as stdout_mock: + stdout_mock.write = stream.write + self.run_command(['--list-targets']) + # We need to sort output strings and replace arbitrary number of spaces with 1 space to make comparison exact + sorted_output = '\n'.join(x for x in sorted(re.sub(r'\s{2,}', ' ', stream.getvalue()).splitlines()) if x.strip()) + self.assertEqual(sorted_output, 'target1 url1\ntarget2 url2') + + @patch('urllib2.urlopen', autospec=True) + @patch('scrapy.commands.deploy._get_targets', autospec=True) + def test_list_projects(self, _get_targets_mock, urlopen_mock, install_opener_mock): + urlopen_mock.return_value = cStringIO('{"projects": ["project1", "project2"]}') + _get_targets_mock.return_value = { + 'target1': {'url': 'http://localhost/target1'}, + 'target2': {'url': 'http://localhost/target2'}, + } + stream = cStringIO() + with patch('sys.stdout') as stdout_mock: + stdout_mock.write = stream.write + self.run_command(['--list-projects', 'target1']) + self.assertEqual(stream.getvalue().strip(), 'project1\nproject2') + + @patch('scrapy.commands.deploy._build_egg', autospec=True) + @patch('shutil.copyfile', autospec=True) + @patch('shutil.rmtree', autospec=True) + def test_build_egg(self, rmtree_mock, copyfile_mock, _build_egg_mock, install_opener_mock): + _build_egg_mock.return_value = ('egg', '/egg_temp_dir') + stream = cStringIO() + with patch('sys.stderr') as stdout_mock: + stdout_mock.write = stream.write + self.run_command(['--build-egg', '/target/egg']) + self.assertEqual(stream.getvalue().strip(), 'Writing egg to /target/egg') + self.assertEqual(copyfile_mock.call_count, 1) + self.assertEqual(copyfile_mock.call_args[0], ('egg', '/target/egg')) + self.assertEqual(rmtree_mock.call_count, 1) + self.assertEqual(rmtree_mock.call_args[0], ('/egg_temp_dir',)) diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index f585ba9fc..93f57cfb3 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -1,3 +1,4 @@ +import optparse import os import sys import subprocess @@ -7,6 +8,7 @@ from shutil import rmtree from tempfile import mkdtemp from twisted.trial import unittest +from scrapy.settings import CrawlerSettings from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv @@ -235,3 +237,28 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') log = p.stderr.read() self.assert_('INFO: Crawled' in log, log) + + +class CommandMockTest(object): + """This class is used to test commands and improve code coverage + without invoking actual command subprocess. + + Subclass must: + 1. Define Command = CommandClassToTest in class scope. + 2. Call super.setUp() and super.tearDown() if methods are overridden. + 3. Call run_command(argv) in test methods. + """ + + def setUp(self): + self.cmd = self.__class__.Command() + self.cmd.settings = CrawlerSettings(object()) + self.parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve') + self.cmd.add_options(self.parser) + + def tearDown(self): + pass + + def run_command(self, argv): + opts, args = self.parser.parse_args(argv) + self.cmd.process_options(args, opts) + self.cmd.run(args, opts) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 1bd129a03..f94d22ce0 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -2,6 +2,7 @@ import re import unittest from scrapy.http import HtmlResponse from scrapy.link import Link +from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor from scrapy.tests import get_testdata @@ -294,5 +295,22 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) +class HtmlParserLinkExtractorTestCase(unittest.TestCase): + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = HtmlParserLinkExtractor() + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://www.google.com/something', text=''), + ]) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_downloadermiddleware_robotstxt.py b/scrapy/tests/test_downloadermiddleware_robotstxt.py new file mode 100644 index 000000000..fbd4b2339 --- /dev/null +++ b/scrapy/tests/test_downloadermiddleware_robotstxt.py @@ -0,0 +1,27 @@ +import mock +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.trial import unittest +from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.settings import CrawlerSettings + + +class RobotsTxtMiddlewareTest(unittest.TestCase): + + def test(self): + crawler = mock.MagicMock() + crawler.settings = CrawlerSettings() + crawler.settings.overrides['USER_AGENT'] = 'CustomAgent' + crawler.settings.overrides['ROBOTSTXT_OBEY'] = True + crawler.engine.download = mock.MagicMock() + deferred = Deferred() + crawler.engine.download.return_value = deferred + ROBOTS = '' + response = Response('http://site.local/robots.txt', body=ROBOTS) + reactor.callLater(0, deferred.callback, response) + middleware = RobotsTxtMiddleware(crawler) + spider = None # Not actually used + self.assertIsNone(middleware.process_request(Request('http://site.local/dummy'), spider)) + self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/forbidden'), spider) \ No newline at end of file