test_command_deploy, test_contrib_linkextractors

requirements.txt

@@ -4,3 +4,4 @@ pyOpenSSL
 cssselect>=0.9
 w3lib>=1.2
 queuelib
+six>=1.5.2

scrapy/tests/test_command_deploy.py

@@ -0,0 +1,60 @@
+import re
+from six.moves import cStringIO
+from mock import patch
+from twisted.trial import unittest
+
+import scrapy.commands.deploy as deploy
+from scrapy.exceptions import UsageError
+from scrapy.tests.test_commands import CommandMockTest
+
+
+@patch('urllib2.install_opener')
+class DeployMockTest(CommandMockTest, unittest.TestCase):
+
+    Command = deploy.Command
+
+    def test_wrong_target(self, install_opener_mock):
+        self.assertRaisesRegexp(UsageError, r'^Unknown target: wrong$', self.run_command, ['wrong'])
+
+    @patch('scrapy.commands.deploy._get_targets', autospec=True)
+    def test_list_targets(self, _get_targets_mock, install_opener_mock):
+        _get_targets_mock.return_value = {
+            'target1': {'url': 'url1'},
+            'target2': {'url': 'url2'},
+        }
+        stream = cStringIO()
+        with patch('sys.stdout') as stdout_mock:
+            stdout_mock.write = stream.write
+            self.run_command(['--list-targets'])
+        # We need to sort output strings and replace arbitrary number of spaces with 1 space to make comparison exact
+        sorted_output = '\n'.join(x for x in sorted(re.sub(r'\s{2,}', ' ', stream.getvalue()).splitlines()) if x.strip())
+        self.assertEqual(sorted_output, 'target1 url1\ntarget2 url2')
+
+    @patch('urllib2.urlopen', autospec=True)
+    @patch('scrapy.commands.deploy._get_targets', autospec=True)
+    def test_list_projects(self, _get_targets_mock, urlopen_mock, install_opener_mock):
+        urlopen_mock.return_value = cStringIO('{"projects": ["project1", "project2"]}')
+        _get_targets_mock.return_value = {
+            'target1': {'url': 'http://localhost/target1'},
+            'target2': {'url': 'http://localhost/target2'},
+        }
+        stream = cStringIO()
+        with patch('sys.stdout') as stdout_mock:
+            stdout_mock.write = stream.write
+            self.run_command(['--list-projects', 'target1'])
+        self.assertEqual(stream.getvalue().strip(), 'project1\nproject2')
+
+    @patch('scrapy.commands.deploy._build_egg', autospec=True)
+    @patch('shutil.copyfile', autospec=True)
+    @patch('shutil.rmtree', autospec=True)
+    def test_build_egg(self, rmtree_mock, copyfile_mock, _build_egg_mock, install_opener_mock):
+        _build_egg_mock.return_value = ('egg', '/egg_temp_dir')
+        stream = cStringIO()
+        with patch('sys.stderr') as stdout_mock:
+            stdout_mock.write = stream.write
+            self.run_command(['--build-egg', '/target/egg'])
+        self.assertEqual(stream.getvalue().strip(), 'Writing egg to /target/egg')
+        self.assertEqual(copyfile_mock.call_count, 1)
+        self.assertEqual(copyfile_mock.call_args[0], ('egg', '/target/egg'))
+        self.assertEqual(rmtree_mock.call_count, 1)
+        self.assertEqual(rmtree_mock.call_args[0], ('/egg_temp_dir',))

scrapy/tests/test_commands.py

@@ -1,3 +1,4 @@
+import optparse
 import os
 import sys
 import subprocess
@@ -7,6 +8,7 @@ from shutil import rmtree
 from tempfile import mkdtemp
 
 from twisted.trial import unittest
+from scrapy.settings import CrawlerSettings
 
 from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
@@ -235,3 +237,28 @@ class BenchCommandTest(CommandTest):
                 '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         log = p.stderr.read()
         self.assert_('INFO: Crawled' in log, log)
+
+
+class CommandMockTest(object):
+    """This class is used to test commands and improve code coverage
+    without invoking actual command subprocess.
+
+    Subclass must:
+    1. Define Command = CommandClassToTest in class scope.
+    2. Call super.setUp() and super.tearDown() if methods are overridden.
+    3. Call run_command(argv) in test methods.
+    """
+
+    def setUp(self):
+        self.cmd = self.__class__.Command()
+        self.cmd.settings = CrawlerSettings(object())
+        self.parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve')
+        self.cmd.add_options(self.parser)
+
+    def tearDown(self):
+        pass
+
+    def run_command(self, argv):
+        opts, args = self.parser.parse_args(argv)
+        self.cmd.process_options(args, opts)
+        self.cmd.run(args, opts)

scrapy/tests/test_contrib_linkextractors.py

@@ -2,6 +2,7 @@ import re
 import unittest
 from scrapy.http import HtmlResponse
 from scrapy.link import Link
+from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
 from scrapy.tests import get_testdata
 
@@ -294,5 +295,22 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
 
 
+class HtmlParserLinkExtractorTestCase(unittest.TestCase):
+
+    def setUp(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        self.response = HtmlResponse(url='http://example.com/index', body=body)
+
+    def test_extraction(self):
+        # Default arguments
+        lx = HtmlParserLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(self.response)], [
+            Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+            Link(url='http://www.google.com/something', text=''),
+        ])
+
+
 if __name__ == "__main__":
     unittest.main()

scrapy/tests/test_downloadermiddleware_robotstxt.py

@@ -0,0 +1,27 @@
+import mock
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.trial import unittest
+from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware
+from scrapy.exceptions import IgnoreRequest
+from scrapy.http import Request, Response
+from scrapy.settings import CrawlerSettings
+
+
+class RobotsTxtMiddlewareTest(unittest.TestCase):
+
+    def test(self):
+        crawler = mock.MagicMock()
+        crawler.settings = CrawlerSettings()
+        crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
+        crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
+        crawler.engine.download = mock.MagicMock()
+        deferred = Deferred()
+        crawler.engine.download.return_value = deferred
+        ROBOTS = ''
+        response = Response('http://site.local/robots.txt', body=ROBOTS)
+        reactor.callLater(0, deferred.callback, response)
+        middleware = RobotsTxtMiddleware(crawler)
+        spider = None # Not actually used
+        self.assertIsNone(middleware.process_request(Request('http://site.local/dummy'), spider))
+        self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/forbidden'), spider)