From 3012030b2faea43b00cb8ae233321c1f9eb8a579 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 12 Dec 2009 10:57:17 -0200 Subject: [PATCH] Fixed bug in file:// downloader handler with uris containing percent-escaped chars --- scrapy/core/downloader/handlers/file.py | 4 +++- scrapy/tests/test_downloader_handlers.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 8a02d02d6..98bfa3342 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,6 +1,8 @@ """Download handler for file:// scheme""" from __future__ import with_statement +from urllib import url2pathname + from twisted.internet import defer from scrapy.core.downloader.responsetypes import responsetypes @@ -10,7 +12,7 @@ def download_file(request, spider): return defer.maybeDeferred(_all_in_one_read_download_file, request, spider) def _all_in_one_read_download_file(request, spider): - filepath = request.url.split("file://")[1] + filepath = url2pathname(request.url.split("file://")[1]) with open(filepath) as f: body = f.read() respcls = responsetypes.from_args(filename=filepath, body=body) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 596a75962..a29f47b90 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -20,7 +20,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): self.tmpname = self.mktemp() - fd = open(self.tmpname, 'w') + fd = open(self.tmpname + '<', 'w') fd.write('0123456789') fd.close() @@ -30,7 +30,8 @@ class FileTestCase(unittest.TestCase): self.assertEquals(response.status, 200) self.assertEquals(response.body, '0123456789') - request = Request('file://%s' % self.tmpname) + request = Request('file://%s' % self.tmpname + '<') + assert request.url.upper().endswith('%3C') return download_file(request, BaseSpider()).addCallback(_test) def test_non_existent(self):