From 1b11563383a9971c9e1375f2e7987042e64ebf93 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Sat, 4 Sep 2010 01:02:52 -0300 Subject: [PATCH] monkeypatch urlparse if s3 netloc parsing fails (python issue7904). closes #223 --- scrapy/__init__.py | 2 +- scrapy/tests/test_urlparse_monkeypatches.py | 8 ++++++++ scrapy/xlib/urlparse_monkeypatches.py | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 scrapy/tests/test_urlparse_monkeypatches.py create mode 100644 scrapy/xlib/urlparse_monkeypatches.py diff --git a/scrapy/__init__.py b/scrapy/__init__.py index cc9152892..696fe10aa 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -20,7 +20,7 @@ from twisted.python.zippath import ZipPath ZipPath.setContent = lambda x, y: None # monkey patches to fix external library issues -from scrapy.xlib import twisted_250_monkeypatches +from scrapy.xlib import twisted_250_monkeypatches, urlparse_monkeypatches # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/tests/test_urlparse_monkeypatches.py b/scrapy/tests/test_urlparse_monkeypatches.py new file mode 100644 index 000000000..7dfca7162 --- /dev/null +++ b/scrapy/tests/test_urlparse_monkeypatches.py @@ -0,0 +1,8 @@ +from urlparse import urlparse +import unittest + + +class UrlparseTestCase(unittest.TestCase): + + def test_s3_netloc(self): + self.assertEqual(urlparse('s3://bucket/key').netloc, 'bucket') diff --git a/scrapy/xlib/urlparse_monkeypatches.py b/scrapy/xlib/urlparse_monkeypatches.py new file mode 100644 index 000000000..010a8bb01 --- /dev/null +++ b/scrapy/xlib/urlparse_monkeypatches.py @@ -0,0 +1,5 @@ +from urlparse import urlparse, uses_netloc + +# workaround for http://bugs.python.org/issue7904 +if urlparse('s3://bucket/key').netloc != 'bucket': + uses_netloc.append('s3')