monkeypatch urlparse if s3 netloc parsing fails (python issue7904). closes #223

scrapy/__init__.py

@@ -20,7 +20,7 @@ from twisted.python.zippath import ZipPath
 ZipPath.setContent = lambda x, y: None
 
 # monkey patches to fix external library issues
-from scrapy.xlib import twisted_250_monkeypatches
+from scrapy.xlib import twisted_250_monkeypatches, urlparse_monkeypatches
 
 # optional_features is a set containing Scrapy optional features
 optional_features = set()

scrapy/tests/test_urlparse_monkeypatches.py

@@ -0,0 +1,8 @@
+from urlparse import urlparse
+import unittest
+
+
+class UrlparseTestCase(unittest.TestCase):
+
+    def test_s3_netloc(self):
+        self.assertEqual(urlparse('s3://bucket/key').netloc, 'bucket')

scrapy/xlib/urlparse_monkeypatches.py

@@ -0,0 +1,5 @@
+from urlparse import urlparse, uses_netloc
+
+# workaround for http://bugs.python.org/issue7904
+if urlparse('s3://bucket/key').netloc != 'bucket':
+    uses_netloc.append('s3')