Simplified file:// download handler, adding support for reading binary files

scrapy/core/downloader/handlers/file.py

@@ -1,21 +1,12 @@
-"""Download handler for file:// scheme"""
-from __future__ import with_statement
-
-from urllib import url2pathname
-
-from twisted.internet import defer
 from scrapy.core.downloader.responsetypes import responsetypes
-
+from scrapy.utils.url import file_uri_to_path
+from scrapy.utils.decorator import defers
 
 class FileRequestHandler(object):
-    """file download"""
 
+    @defers
     def download_request(self, request, spider):
-        return defer.maybeDeferred(self._one_pass_read, request)
-
-    def _one_pass_read(self, request):
-        filepath = url2pathname(request.url.split("file://")[1])
-        with open(filepath) as f:
-            body = f.read()
+        filepath = file_uri_to_path(request.url)
+        body = open(filepath, 'rb').read()
         respcls = responsetypes.from_args(filename=filepath, body=body)
         return respcls(url=request.url, body=body)

scrapy/utils/decorator.py

@@ -1,6 +1,8 @@
 import warnings
 from functools import wraps
 
+from twisted.internet.defer import maybeDeferred
+
 
 def deprecated(use_instead=None):
     """This is a decorator which can be used to mark functions
@@ -17,3 +19,10 @@ def deprecated(use_instead=None):
             return func(*args, **kwargs)
         return new_func
     return wrapped
+
+def defers(func):
+    """Decorator to make sure a function always returns a deferred"""
+    @wraps(func)
+    def wrapped(*a, **kw):
+        return maybeDeferred(func, *a, **kw)
+    return wrapped