From 668e5fd25706171fe227f2f161aa331435560849 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 2 Sep 2015 01:43:22 -0300 Subject: [PATCH] Test for robots.txt middleware for processing a request for which the robots.txt parser is ready. --- tests/test_downloadermiddleware_robotstxt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index e0e8ade47..0c03f96a9 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -50,6 +50,12 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): self.assertIgnored(Request('http://site.local/static/'), middleware) ], fireOnOneErrback=True) + def test_robotstxt_ready_parser(self): + middleware = RobotsTxtMiddleware(self._get_successful_crawler()) + d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware) + d.addCallback(lambda _: self.assertNotIgnored(Request('http://site.local/allowed'), middleware)) + return d + def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) meta = {'dont_obey_robotstxt': True}