From 968141cd42b108e4343e3e6775ea223ad8d9b4b0 Mon Sep 17 00:00:00 2001 From: alexanderlukanin13 Date: Sat, 11 Jan 2014 14:30:27 +0600 Subject: [PATCH 1/5] test_command_deploy, test_contrib_linkextractors --- requirements.txt | 1 + scrapy/tests/test_command_deploy.py | 60 +++++++++++++++++++ scrapy/tests/test_commands.py | 27 +++++++++ scrapy/tests/test_contrib_linkextractors.py | 18 ++++++ .../test_downloadermiddleware_robotstxt.py | 27 +++++++++ 5 files changed, 133 insertions(+) create mode 100644 scrapy/tests/test_command_deploy.py create mode 100644 scrapy/tests/test_downloadermiddleware_robotstxt.py diff --git a/requirements.txt b/requirements.txt index e070a183e..0df9a558c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pyOpenSSL cssselect>=0.9 w3lib>=1.2 queuelib +six>=1.5.2 diff --git a/scrapy/tests/test_command_deploy.py b/scrapy/tests/test_command_deploy.py new file mode 100644 index 000000000..e6f008099 --- /dev/null +++ b/scrapy/tests/test_command_deploy.py @@ -0,0 +1,60 @@ +import re +from six.moves import cStringIO +from mock import patch +from twisted.trial import unittest + +import scrapy.commands.deploy as deploy +from scrapy.exceptions import UsageError +from scrapy.tests.test_commands import CommandMockTest + + +@patch('urllib2.install_opener') +class DeployMockTest(CommandMockTest, unittest.TestCase): + + Command = deploy.Command + + def test_wrong_target(self, install_opener_mock): + self.assertRaisesRegexp(UsageError, r'^Unknown target: wrong$', self.run_command, ['wrong']) + + @patch('scrapy.commands.deploy._get_targets', autospec=True) + def test_list_targets(self, _get_targets_mock, install_opener_mock): + _get_targets_mock.return_value = { + 'target1': {'url': 'url1'}, + 'target2': {'url': 'url2'}, + } + stream = cStringIO() + with patch('sys.stdout') as stdout_mock: + stdout_mock.write = stream.write + self.run_command(['--list-targets']) + # We need to sort output strings and replace arbitrary number of spaces with 1 space to make comparison exact + sorted_output = '\n'.join(x for x in sorted(re.sub(r'\s{2,}', ' ', stream.getvalue()).splitlines()) if x.strip()) + self.assertEqual(sorted_output, 'target1 url1\ntarget2 url2') + + @patch('urllib2.urlopen', autospec=True) + @patch('scrapy.commands.deploy._get_targets', autospec=True) + def test_list_projects(self, _get_targets_mock, urlopen_mock, install_opener_mock): + urlopen_mock.return_value = cStringIO('{"projects": ["project1", "project2"]}') + _get_targets_mock.return_value = { + 'target1': {'url': 'http://localhost/target1'}, + 'target2': {'url': 'http://localhost/target2'}, + } + stream = cStringIO() + with patch('sys.stdout') as stdout_mock: + stdout_mock.write = stream.write + self.run_command(['--list-projects', 'target1']) + self.assertEqual(stream.getvalue().strip(), 'project1\nproject2') + + @patch('scrapy.commands.deploy._build_egg', autospec=True) + @patch('shutil.copyfile', autospec=True) + @patch('shutil.rmtree', autospec=True) + def test_build_egg(self, rmtree_mock, copyfile_mock, _build_egg_mock, install_opener_mock): + _build_egg_mock.return_value = ('egg', '/egg_temp_dir') + stream = cStringIO() + with patch('sys.stderr') as stdout_mock: + stdout_mock.write = stream.write + self.run_command(['--build-egg', '/target/egg']) + self.assertEqual(stream.getvalue().strip(), 'Writing egg to /target/egg') + self.assertEqual(copyfile_mock.call_count, 1) + self.assertEqual(copyfile_mock.call_args[0], ('egg', '/target/egg')) + self.assertEqual(rmtree_mock.call_count, 1) + self.assertEqual(rmtree_mock.call_args[0], ('/egg_temp_dir',)) diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index f585ba9fc..93f57cfb3 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -1,3 +1,4 @@ +import optparse import os import sys import subprocess @@ -7,6 +8,7 @@ from shutil import rmtree from tempfile import mkdtemp from twisted.trial import unittest +from scrapy.settings import CrawlerSettings from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv @@ -235,3 +237,28 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') log = p.stderr.read() self.assert_('INFO: Crawled' in log, log) + + +class CommandMockTest(object): + """This class is used to test commands and improve code coverage + without invoking actual command subprocess. + + Subclass must: + 1. Define Command = CommandClassToTest in class scope. + 2. Call super.setUp() and super.tearDown() if methods are overridden. + 3. Call run_command(argv) in test methods. + """ + + def setUp(self): + self.cmd = self.__class__.Command() + self.cmd.settings = CrawlerSettings(object()) + self.parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve') + self.cmd.add_options(self.parser) + + def tearDown(self): + pass + + def run_command(self, argv): + opts, args = self.parser.parse_args(argv) + self.cmd.process_options(args, opts) + self.cmd.run(args, opts) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 1bd129a03..f94d22ce0 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -2,6 +2,7 @@ import re import unittest from scrapy.http import HtmlResponse from scrapy.link import Link +from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor from scrapy.tests import get_testdata @@ -294,5 +295,22 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) +class HtmlParserLinkExtractorTestCase(unittest.TestCase): + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = HtmlParserLinkExtractor() + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://www.google.com/something', text=''), + ]) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_downloadermiddleware_robotstxt.py b/scrapy/tests/test_downloadermiddleware_robotstxt.py new file mode 100644 index 000000000..fbd4b2339 --- /dev/null +++ b/scrapy/tests/test_downloadermiddleware_robotstxt.py @@ -0,0 +1,27 @@ +import mock +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.trial import unittest +from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.settings import CrawlerSettings + + +class RobotsTxtMiddlewareTest(unittest.TestCase): + + def test(self): + crawler = mock.MagicMock() + crawler.settings = CrawlerSettings() + crawler.settings.overrides['USER_AGENT'] = 'CustomAgent' + crawler.settings.overrides['ROBOTSTXT_OBEY'] = True + crawler.engine.download = mock.MagicMock() + deferred = Deferred() + crawler.engine.download.return_value = deferred + ROBOTS = '' + response = Response('http://site.local/robots.txt', body=ROBOTS) + reactor.callLater(0, deferred.callback, response) + middleware = RobotsTxtMiddleware(crawler) + spider = None # Not actually used + self.assertIsNone(middleware.process_request(Request('http://site.local/dummy'), spider)) + self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/forbidden'), spider) \ No newline at end of file From a54e31cebc267339ba805aa06157fba8cb1a6dbb Mon Sep 17 00:00:00 2001 From: alexanderlukanin13 Date: Sat, 11 Jan 2014 15:12:54 +0600 Subject: [PATCH 2/5] RegexLinkExtractorTestCase --- scrapy/tests/test_contrib_linkextractors.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index f94d22ce0..4135b086c 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -1,5 +1,6 @@ import re import unittest +from scrapy.contrib.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse from scrapy.link import Link from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor @@ -312,5 +313,23 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): ]) +class RegexLinkExtractorTestCase(unittest.TestCase): + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = RegexLinkExtractor() + # Note that RegexLinkExtractor returns links in arbitrary order, + # so we need to sort them for comparison + self.assertEqual(sorted(lx.extract_links(self.response), key=lambda x: x.url), [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://www.google.com/something', text=u''), + ]) + + if __name__ == "__main__": unittest.main() From 4862a7d7fea87e0861ff5429673b8ab131f6a1e9 Mon Sep 17 00:00:00 2001 From: alexanderlukanin13 Date: Sat, 11 Jan 2014 17:29:44 +0600 Subject: [PATCH 3/5] RobotsTxtMiddlewareTest --- .../test_downloadermiddleware_robotstxt.py | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/scrapy/tests/test_downloadermiddleware_robotstxt.py b/scrapy/tests/test_downloadermiddleware_robotstxt.py index fbd4b2339..a98522510 100644 --- a/scrapy/tests/test_downloadermiddleware_robotstxt.py +++ b/scrapy/tests/test_downloadermiddleware_robotstxt.py @@ -1,9 +1,10 @@ +import re import mock from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.settings import CrawlerSettings @@ -14,14 +15,32 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): crawler = mock.MagicMock() crawler.settings = CrawlerSettings() crawler.settings.overrides['USER_AGENT'] = 'CustomAgent' + self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler) crawler.settings.overrides['ROBOTSTXT_OBEY'] = True crawler.engine.download = mock.MagicMock() - deferred = Deferred() - crawler.engine.download.return_value = deferred - ROBOTS = '' + ROBOTS = re.sub(r'^\s+(?m)', '', ''' + User-Agent: * + Disallow: /admin/ + Disallow: /static/ + ''') response = Response('http://site.local/robots.txt', body=ROBOTS) - reactor.callLater(0, deferred.callback, response) + def return_response(request, spider): + deferred = Deferred() + reactor.callFromThread(deferred.callback, response) + return deferred + crawler.engine.download.side_effect = return_response middleware = RobotsTxtMiddleware(crawler) - spider = None # Not actually used - self.assertIsNone(middleware.process_request(Request('http://site.local/dummy'), spider)) - self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/forbidden'), spider) \ No newline at end of file + spider = None # not actually used + # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously, + # and it is actually fetched only *after* first process_request completes. + # So, first process_request will always succeed. + # We defer test() because otherwise robots.txt download mock will be called after assertRaises failure. + self.assertIsNone(middleware.process_request(Request('http://site.local'), spider)) # not affected by robots.txt + def test(r): + self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider)) + self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider) + self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider) + deferred = Deferred() + deferred.addCallback(test) + reactor.callFromThread(deferred.callback, None) + return deferred From ab6496647c1e4b379898e5942ef5085e640fd63c Mon Sep 17 00:00:00 2001 From: alexanderlukanin13 Date: Wed, 15 Jan 2014 13:05:00 +0600 Subject: [PATCH 4/5] Added "six>=1.5.2" to requirements --- debian/control | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 682002916..85ecdd135 100644 --- a/debian/control +++ b/debian/control @@ -2,14 +2,14 @@ Source: scrapy-SUFFIX Section: python Priority: optional Maintainer: Scrapinghub Team -Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml +Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) Standards-Version: 3.8.4 Homepage: http://scrapy.org/ Package: scrapy-SUFFIX Architecture: all Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl, - python-w3lib (>= 1.2), python-queuelib, python-cssselect (>= 0.9) + python-w3lib (>= 1.2), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2) Recommends: python-setuptools Conflicts: python-scrapy, scrapy, scrapy-0.11 Provides: python-scrapy, scrapy diff --git a/setup.py b/setup.py index a0f982567..6efe64074 100644 --- a/setup.py +++ b/setup.py @@ -129,6 +129,7 @@ else: 'lxml', 'pyOpenSSL', 'cssselect>=0.9', + 'six>=1.5.2', ] setup(**setup_args) From 25759bdf5c891fa4084abe8bc8c86b56dae0a38f Mon Sep 17 00:00:00 2001 From: alexanderlukanin13 Date: Wed, 15 Jan 2014 13:09:36 +0600 Subject: [PATCH 5/5] Removed deploy test (as deploy command is deprecated and moved to scrapyd) --- scrapy/tests/test_command_deploy.py | 60 ----------------------------- scrapy/tests/test_commands.py | 27 ------------- 2 files changed, 87 deletions(-) delete mode 100644 scrapy/tests/test_command_deploy.py diff --git a/scrapy/tests/test_command_deploy.py b/scrapy/tests/test_command_deploy.py deleted file mode 100644 index e6f008099..000000000 --- a/scrapy/tests/test_command_deploy.py +++ /dev/null @@ -1,60 +0,0 @@ -import re -from six.moves import cStringIO -from mock import patch -from twisted.trial import unittest - -import scrapy.commands.deploy as deploy -from scrapy.exceptions import UsageError -from scrapy.tests.test_commands import CommandMockTest - - -@patch('urllib2.install_opener') -class DeployMockTest(CommandMockTest, unittest.TestCase): - - Command = deploy.Command - - def test_wrong_target(self, install_opener_mock): - self.assertRaisesRegexp(UsageError, r'^Unknown target: wrong$', self.run_command, ['wrong']) - - @patch('scrapy.commands.deploy._get_targets', autospec=True) - def test_list_targets(self, _get_targets_mock, install_opener_mock): - _get_targets_mock.return_value = { - 'target1': {'url': 'url1'}, - 'target2': {'url': 'url2'}, - } - stream = cStringIO() - with patch('sys.stdout') as stdout_mock: - stdout_mock.write = stream.write - self.run_command(['--list-targets']) - # We need to sort output strings and replace arbitrary number of spaces with 1 space to make comparison exact - sorted_output = '\n'.join(x for x in sorted(re.sub(r'\s{2,}', ' ', stream.getvalue()).splitlines()) if x.strip()) - self.assertEqual(sorted_output, 'target1 url1\ntarget2 url2') - - @patch('urllib2.urlopen', autospec=True) - @patch('scrapy.commands.deploy._get_targets', autospec=True) - def test_list_projects(self, _get_targets_mock, urlopen_mock, install_opener_mock): - urlopen_mock.return_value = cStringIO('{"projects": ["project1", "project2"]}') - _get_targets_mock.return_value = { - 'target1': {'url': 'http://localhost/target1'}, - 'target2': {'url': 'http://localhost/target2'}, - } - stream = cStringIO() - with patch('sys.stdout') as stdout_mock: - stdout_mock.write = stream.write - self.run_command(['--list-projects', 'target1']) - self.assertEqual(stream.getvalue().strip(), 'project1\nproject2') - - @patch('scrapy.commands.deploy._build_egg', autospec=True) - @patch('shutil.copyfile', autospec=True) - @patch('shutil.rmtree', autospec=True) - def test_build_egg(self, rmtree_mock, copyfile_mock, _build_egg_mock, install_opener_mock): - _build_egg_mock.return_value = ('egg', '/egg_temp_dir') - stream = cStringIO() - with patch('sys.stderr') as stdout_mock: - stdout_mock.write = stream.write - self.run_command(['--build-egg', '/target/egg']) - self.assertEqual(stream.getvalue().strip(), 'Writing egg to /target/egg') - self.assertEqual(copyfile_mock.call_count, 1) - self.assertEqual(copyfile_mock.call_args[0], ('egg', '/target/egg')) - self.assertEqual(rmtree_mock.call_count, 1) - self.assertEqual(rmtree_mock.call_args[0], ('/egg_temp_dir',)) diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 93f57cfb3..f585ba9fc 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -1,4 +1,3 @@ -import optparse import os import sys import subprocess @@ -8,7 +7,6 @@ from shutil import rmtree from tempfile import mkdtemp from twisted.trial import unittest -from scrapy.settings import CrawlerSettings from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv @@ -237,28 +235,3 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') log = p.stderr.read() self.assert_('INFO: Crawled' in log, log) - - -class CommandMockTest(object): - """This class is used to test commands and improve code coverage - without invoking actual command subprocess. - - Subclass must: - 1. Define Command = CommandClassToTest in class scope. - 2. Call super.setUp() and super.tearDown() if methods are overridden. - 3. Call run_command(argv) in test methods. - """ - - def setUp(self): - self.cmd = self.__class__.Command() - self.cmd.settings = CrawlerSettings(object()) - self.parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve') - self.cmd.add_options(self.parser) - - def tearDown(self): - pass - - def run_command(self, argv): - opts, args = self.parser.parse_args(argv) - self.cmd.process_options(args, opts) - self.cmd.run(args, opts)