RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Test local files aren't processed

This commit is contained in:
Cj Malone 2023-01-26 19:51:20 +00:00 committed by GitHub
parent 3054235dc0
commit 33b85a9e2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/test_downloadermiddleware_robotstxt.py
View File

@ -214,6 +214,19 @@ Disallow: /some/randome/page.html
middleware.process_request_2(rp, Request("http://site.local/allowed"), None)
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
assert not middleware.process_request(
Request("data:text/plain,Hello World data"), None
)
assert not middleware.process_request(
Request("file:///tests/sample_data/test_site/nothinghere.html"), None
)
assert isinstance(
middleware.process_request(Request("http://site.local/allowed"), None),
Deferred,
)
def assertNotIgnored(self, request, middleware):
spider = None # not actually used
dfd = maybeDeferred(middleware.process_request, request, spider)