From fd784cd131a7e48ce7261ab54662f0affac0d4ec Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 22 Aug 2010 19:37:20 -0300 Subject: [PATCH] Fixed tests on Windows --- scrapy/contrib/feedexport.py | 4 ++-- scrapy/tests/test_contrib_feedexport.py | 5 +++-- scrapy/tests/test_contrib_ibl/test_pageparsing.py | 2 +- scrapy/utils/url.py | 2 ++ 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index 7dff100aa..53f845c22 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -17,6 +17,7 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object +from scrapy.utils.url import file_uri_to_path from scrapy.conf import settings class BlockingFeedStorage(object): @@ -40,8 +41,7 @@ class StdoutFeedStorage(object): class FileFeedStorage(BlockingFeedStorage): def __init__(self, uri): - u = urlparse(uri) - self.path = u.path + self.path = file_uri_to_path(uri) def _store_in_thread(self, file, spider): dirname = os.path.dirname(self.path) diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index 69e7f4324..a3edabb7b 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -6,6 +6,7 @@ from cStringIO import StringIO from scrapy.spider import BaseSpider from scrapy.contrib.feedexport import FileFeedStorage, FTPFeedStorage, S3FeedStorage +from scrapy.utils.url import path_to_file_uri class FeedStorageTest(unittest.TestCase): @@ -23,13 +24,13 @@ class FileFeedStorageTest(FeedStorageTest): def test_store_file_uri(self): path = os.path.abspath(self.mktemp()) - uri = "file://%s" % path + uri = path_to_file_uri(path) return self._assert_stores(FileFeedStorage(uri), path) def test_store_file_uri_makedirs(self): path = os.path.abspath(self.mktemp()) path = os.path.join(path, 'more', 'paths', 'file.txt') - uri = "file://%s" % path + uri = path_to_file_uri(path) return self._assert_stores(FileFeedStorage(uri), path) def test_store_direct_path(self): diff --git a/scrapy/tests/test_contrib_ibl/test_pageparsing.py b/scrapy/tests/test_contrib_ibl/test_pageparsing.py index 12a6d346e..6d9f072a6 100644 --- a/scrapy/tests/test_contrib_ibl/test_pageparsing.py +++ b/scrapy/tests/test_contrib_ibl/test_pageparsing.py @@ -274,7 +274,7 @@ class TestPageParsing(TestCase): """ Tests from real pages. More reliable and easy to build for more complicated structures """ - samples_file = open(os.path.join(path, "samples_pageparsing.json.gz"), "r") + samples_file = open(os.path.join(path, "samples_pageparsing.json.gz"), "rb") samples = [] for line in GzipFile(fileobj=StringIO(samples_file.read())).readlines(): samples.append(json.loads(line)) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index f2ac4719c..552f343f5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -173,5 +173,7 @@ def any_to_uri(uri_or_path): """If given a path name, return its File URI, otherwise return it unmodified """ + if os.path.splitdrive(uri_or_path)[0]: + return path_to_file_uri(uri_or_path) u = urlparse.urlparse(uri_or_path) return uri_or_path if u.scheme else path_to_file_uri(uri_or_path)