diff --git a/debian/control b/debian/control index 682002916..85ecdd135 100644 --- a/debian/control +++ b/debian/control @@ -2,14 +2,14 @@ Source: scrapy-SUFFIX Section: python Priority: optional Maintainer: Scrapinghub Team -Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml +Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) Standards-Version: 3.8.4 Homepage: http://scrapy.org/ Package: scrapy-SUFFIX Architecture: all Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl, - python-w3lib (>= 1.2), python-queuelib, python-cssselect (>= 0.9) + python-w3lib (>= 1.2), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2) Recommends: python-setuptools Conflicts: python-scrapy, scrapy, scrapy-0.11 Provides: python-scrapy, scrapy diff --git a/requirements.txt b/requirements.txt index e070a183e..0df9a558c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pyOpenSSL cssselect>=0.9 w3lib>=1.2 queuelib +six>=1.5.2 diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 1bd129a03..4135b086c 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -1,7 +1,9 @@ import re import unittest +from scrapy.contrib.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse from scrapy.link import Link +from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor from scrapy.tests import get_testdata @@ -294,5 +296,40 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) +class HtmlParserLinkExtractorTestCase(unittest.TestCase): + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = HtmlParserLinkExtractor() + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://www.google.com/something', text=''), + ]) + + +class RegexLinkExtractorTestCase(unittest.TestCase): + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = RegexLinkExtractor() + # Note that RegexLinkExtractor returns links in arbitrary order, + # so we need to sort them for comparison + self.assertEqual(sorted(lx.extract_links(self.response), key=lambda x: x.url), [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://www.google.com/something', text=u''), + ]) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_downloadermiddleware_robotstxt.py b/scrapy/tests/test_downloadermiddleware_robotstxt.py new file mode 100644 index 000000000..a98522510 --- /dev/null +++ b/scrapy/tests/test_downloadermiddleware_robotstxt.py @@ -0,0 +1,46 @@ +import re +import mock +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.trial import unittest +from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware +from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http import Request, Response +from scrapy.settings import CrawlerSettings + + +class RobotsTxtMiddlewareTest(unittest.TestCase): + + def test(self): + crawler = mock.MagicMock() + crawler.settings = CrawlerSettings() + crawler.settings.overrides['USER_AGENT'] = 'CustomAgent' + self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler) + crawler.settings.overrides['ROBOTSTXT_OBEY'] = True + crawler.engine.download = mock.MagicMock() + ROBOTS = re.sub(r'^\s+(?m)', '', ''' + User-Agent: * + Disallow: /admin/ + Disallow: /static/ + ''') + response = Response('http://site.local/robots.txt', body=ROBOTS) + def return_response(request, spider): + deferred = Deferred() + reactor.callFromThread(deferred.callback, response) + return deferred + crawler.engine.download.side_effect = return_response + middleware = RobotsTxtMiddleware(crawler) + spider = None # not actually used + # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously, + # and it is actually fetched only *after* first process_request completes. + # So, first process_request will always succeed. + # We defer test() because otherwise robots.txt download mock will be called after assertRaises failure. + self.assertIsNone(middleware.process_request(Request('http://site.local'), spider)) # not affected by robots.txt + def test(r): + self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider)) + self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider) + self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider) + deferred = Deferred() + deferred.addCallback(test) + reactor.callFromThread(deferred.callback, None) + return deferred diff --git a/setup.py b/setup.py index a0f982567..6efe64074 100644 --- a/setup.py +++ b/setup.py @@ -129,6 +129,7 @@ else: 'lxml', 'pyOpenSSL', 'cssselect>=0.9', + 'six>=1.5.2', ] setup(**setup_args)