Merge pull request #525 from alexanderlukanin13/urllib_test_coverage

Improved test coverage

debian/control

@@ -2,14 +2,14 @@ Source: scrapy-SUFFIX
 Section: python
 Priority: optional
 Maintainer: Scrapinghub Team <info@scrapinghub.com>
-Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml
+Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
 Standards-Version: 3.8.4
 Homepage: http://scrapy.org/
 
 Package: scrapy-SUFFIX
 Architecture: all
 Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
- python-w3lib (>= 1.2), python-queuelib, python-cssselect (>= 0.9)
+ python-w3lib (>= 1.2), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
 Recommends: python-setuptools
 Conflicts: python-scrapy, scrapy, scrapy-0.11
 Provides: python-scrapy, scrapy

requirements.txt

@@ -4,3 +4,4 @@ pyOpenSSL
 cssselect>=0.9
 w3lib>=1.2
 queuelib
+six>=1.5.2

scrapy/tests/test_contrib_linkextractors.py

@@ -1,7 +1,9 @@
 import re
 import unittest
+from scrapy.contrib.linkextractors.regex import RegexLinkExtractor
 from scrapy.http import HtmlResponse
 from scrapy.link import Link
+from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
 from scrapy.tests import get_testdata
 
@@ -294,5 +296,40 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
 
 
+class HtmlParserLinkExtractorTestCase(unittest.TestCase):
+
+    def setUp(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        self.response = HtmlResponse(url='http://example.com/index', body=body)
+
+    def test_extraction(self):
+        # Default arguments
+        lx = HtmlParserLinkExtractor()
+        self.assertEqual([link for link in lx.extract_links(self.response)], [
+            Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+            Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+            Link(url='http://www.google.com/something', text=''),
+        ])
+
+
+class RegexLinkExtractorTestCase(unittest.TestCase):
+
+    def setUp(self):
+        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+        self.response = HtmlResponse(url='http://example.com/index', body=body)
+
+    def test_extraction(self):
+        # Default arguments
+        lx = RegexLinkExtractor()
+        # Note that RegexLinkExtractor returns links in arbitrary order,
+        # so we need to sort them for comparison
+        self.assertEqual(sorted(lx.extract_links(self.response), key=lambda x: x.url), [
+            Link(url='http://example.com/sample2.html', text=u'sample 2'),
+            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+            Link(url='http://www.google.com/something', text=u''),
+        ])
+
+
 if __name__ == "__main__":
     unittest.main()

scrapy/tests/test_downloadermiddleware_robotstxt.py

@@ -0,0 +1,46 @@
+import re
+import mock
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.trial import unittest
+from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware
+from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.http import Request, Response
+from scrapy.settings import CrawlerSettings
+
+
+class RobotsTxtMiddlewareTest(unittest.TestCase):
+
+    def test(self):
+        crawler = mock.MagicMock()
+        crawler.settings = CrawlerSettings()
+        crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
+        self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
+        crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
+        crawler.engine.download = mock.MagicMock()
+        ROBOTS = re.sub(r'^\s+(?m)', '', '''
+        User-Agent: *
+        Disallow: /admin/
+        Disallow: /static/
+        ''')
+        response = Response('http://site.local/robots.txt', body=ROBOTS)
+        def return_response(request, spider):
+            deferred = Deferred()
+            reactor.callFromThread(deferred.callback, response)
+            return deferred
+        crawler.engine.download.side_effect = return_response
+        middleware = RobotsTxtMiddleware(crawler)
+        spider = None  # not actually used
+        # There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
+        # and it is actually fetched only *after* first process_request completes.
+        # So, first process_request will always succeed.
+        # We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
+        self.assertIsNone(middleware.process_request(Request('http://site.local'), spider))  # not affected by robots.txt
+        def test(r):
+            self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider))
+            self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider)
+            self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider)
+        deferred = Deferred()
+        deferred.addCallback(test)
+        reactor.callFromThread(deferred.callback, None)
+        return deferred

setup.py

@@ -129,6 +129,7 @@ else:
         'lxml',
         'pyOpenSSL',
         'cssselect>=0.9',
+        'six>=1.5.2',
     ]
 
 setup(**setup_args)