remove xlib hack that appends scrapy/xlib to sys.path

scrapy/__init__.py

@@ -11,9 +11,6 @@ if sys.version_info < (2,5):
     print "Scrapy %s requires Python 2.5 or above" % __version__
     sys.exit(1)
 
-# add external python libraries bundled into scrapy
-sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "xlib"))
-
 # monkey patches to fix external library issues
 from scrapy.patches import monkeypatches
 monkeypatches.apply_patches()

scrapy/command/cmdline.py

@@ -143,7 +143,7 @@ def execute_with_args(argv):
         p.runctx('ret = cmd.run(args, opts)', globals(), loc)
         p.dump_stats(opts.profile)
         try:
-            import lsprofcalltree
+            from scrapy.xlib import lsprofcalltree
             fn = opts.profile + ".cachegrind"
             k = lsprofcalltree.KCacheGrind(p)
             with open(fn, 'w') as f:

scrapy/contrib/closedomain.py

@@ -6,7 +6,7 @@ time has expired.
 import datetime
 import pprint
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy import log

scrapy/contrib/cluster/master/manager.py

@@ -3,7 +3,7 @@ from __future__ import with_statement
 import datetime
 import cPickle as pickle
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from twisted.spread import pb
 from twisted.internet import reactor
 

scrapy/contrib/cluster/master/web.py

@@ -1,6 +1,6 @@
 import datetime
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.spider import spiders 
 from scrapy.management.web import banner, webconsole_discover_module

scrapy/contrib/delayedclosedomain.py

@@ -5,7 +5,7 @@ configurable amount of idle time is reached
 
 from datetime import datetime
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from collections import defaultdict
 
 from scrapy.core import signals

scrapy/contrib/downloadermiddleware/cookies.py

@@ -1,7 +1,7 @@
 import operator
 from itertools import groupby
 from collections import defaultdict
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.http import Response

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -5,7 +5,7 @@ import os
 import hashlib
 import datetime
 import cPickle as pickle
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy import log

scrapy/contrib/downloadermiddleware/robotstxt.py

@@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 import urlparse
 import robotparser
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.core.engine import scrapyengine

scrapy/contrib/itemsampler.py

@@ -25,7 +25,7 @@ from __future__ import with_statement
 
 import cPickle as pickle
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core.engine import scrapyengine
 from scrapy.core.exceptions import NotConfigured

scrapy/contrib/memdebug.py

@@ -9,7 +9,7 @@ import gc
 import socket
 
 import libxml2
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.core.exceptions import NotConfigured

scrapy/contrib/memusage.py

@@ -9,7 +9,7 @@ import os
 import pprint
 import socket
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy import log

scrapy/contrib/pipeline/media.py

@@ -1,4 +1,4 @@
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from twisted.internet import defer
 
 from scrapy.utils.defer import mustbe_deferred, defer_result

scrapy/contrib/response/soup.py

@@ -4,7 +4,7 @@ ResponseSoup extension
 See documentation in docs/ref/extensions.rst
 """ 
 
-from BeautifulSoup import BeautifulSoup
+from scrapy.xlib.BeautifulSoup import BeautifulSoup
 
 from scrapy.http import Response
 

scrapy/contrib/spider/profiler.py

@@ -11,7 +11,7 @@ when needed.
 
 import datetime
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.extension import extensions
 from scrapy.core import signals

scrapy/contrib/spider/reloader.py

@@ -4,7 +4,7 @@ Reload spider modules once they are finished scraping
 This is to release any resources held on to by scraping spiders.
 """
 import sys
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core import signals
 from scrapy import log
 

scrapy/contrib/spidermiddleware/requestlimit.py

@@ -8,7 +8,7 @@ setting "REQUESTS_QUEUE_SIZE". If not specified (or 0), no limit will be
 applied. 
 """
 from itertools import imap
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.core.engine import scrapyengine

scrapy/contrib/statsmailer.py

@@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
 
 import pprint
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.stats import stats
 from scrapy.mail import MailSender

scrapy/contrib/web/service.py

@@ -10,7 +10,7 @@ from scrapy.item import ScrapedItem
 from scrapy.core.exceptions import NotConfigured
 from scrapy.conf import settings
 from scrapy.utils.misc import memoize, arg_to_iter
-from lrucache import LRUCache
+from scrapy.xlib.lrucache import LRUCache
 
 from .site import WebSite, WebResource
 from .http import HttpResponse

scrapy/contrib/webconsole/enginestatus.py

@@ -1,7 +1,7 @@
 """
 Scheduler information module for Scrapy webconsole
 """
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core.engine import scrapyengine
 from scrapy.management.web import banner
 

scrapy/contrib/webconsole/livestats.py

@@ -2,7 +2,7 @@
 Live statistics extension
 """
 from datetime import datetime
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core import signals
 from scrapy.core.engine import scrapyengine
 from scrapy.management.web import banner, webconsole_discover_module

scrapy/contrib/webconsole/scheduler.py

@@ -4,7 +4,7 @@ Scheduler queue web console module
 See documentation in docs/ref/extensions.rst
 """
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core.engine import scrapyengine
 from scrapy.management.web import banner, webconsole_discover_module
 

scrapy/contrib/webconsole/spiderctl.py

@@ -2,7 +2,7 @@
 Extensions for allowing spider control from web console
 """
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.core.manager import scrapymanager

scrapy/contrib/webconsole/spiderstats.py

@@ -4,7 +4,7 @@ Extension for persistent statistics by domain
 import re
 import pprint
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.spider import spiders
 from scrapy.core.exceptions import NotConfigured

scrapy/contrib/webconsole/stats.py

@@ -1,6 +1,6 @@
 import pprint
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.stats import stats
 from scrapy.management.web import banner, webconsole_discover_module

scrapy/contrib_exp/history/middleware.py

@@ -1,7 +1,7 @@
 import hashlib
 from datetime import datetime
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.utils.misc import load_object
 from scrapy.core import signals

scrapy/contrib_exp/pipeline/shoveitem.py

@@ -8,7 +8,7 @@ http://pypi.python.org/pypi/shove
 from string import Template
 
 from shove import Shove
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import log
 from scrapy.core import signals

scrapy/core/downloader/dnscache.py

@@ -5,7 +5,7 @@ crawler, reducing the dns lookups.
 """
 import socket
 from scrapy.core import signals
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 class DNSCache(object):
     """

scrapy/core/engine.py

@@ -9,7 +9,7 @@ from datetime import datetime
 from twisted.internet import reactor, task
 from twisted.internet.error import CannotListenError
 from twisted.python.failure import Failure
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import log
 from scrapy.conf import settings

scrapy/core/signals.py

@@ -5,7 +5,7 @@ These signals are documented in docs/ref/signals.rst. Please don't add new
 signals here without documenting them there.
 """
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import log
 

scrapy/http/request/form.py

@@ -8,7 +8,7 @@ See documentation in docs/ref/request-response.rst
 import urllib
 from cStringIO import StringIO
 
-from ClientForm import ParseFile
+from scrapy.xlib.ClientForm import ParseFile
 
 from scrapy.http.request import Request
 from scrapy.utils.python import unicode_to_str

scrapy/http/response/text.py

@@ -7,7 +7,7 @@ See documentation in docs/ref/request-response.rst
 
 import re
 
-from BeautifulSoup import UnicodeDammit
+from scrapy.xlib.BeautifulSoup import UnicodeDammit
 
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod

scrapy/log/__init__.py

@@ -7,7 +7,7 @@ import sys
 from traceback import format_exc
 
 from twisted.python import log
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.conf import settings
 from scrapy.utils.python import unicode_to_str

scrapy/management/web.py

@@ -9,7 +9,7 @@ import socket
 from datetime import datetime
 
 from twisted.web import server, resource
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core.engine import scrapyengine
 from scrapy.conf import settings

scrapy/stats/corestats.py

@@ -6,7 +6,7 @@ import getpass
 import socket
 import datetime
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.stats import stats

scrapy/stats/statscollector.py

@@ -3,7 +3,7 @@ Scrapy extension for collecting scraping stats
 """
 import pprint
 
-from pydispatch import dispatcher
+from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy import log

scrapy/tests/test_contrib_response_soup.py

@@ -1,6 +1,6 @@
 import unittest
 
-from BeautifulSoup import BeautifulSoup
+from scrapy.xlib.BeautifulSoup import BeautifulSoup
 
 from scrapy.http import Response
 from scrapy.contrib.response.soup import ResponseSoup

scrapy/tests/test_engine.py

@@ -58,7 +58,7 @@ class CrawlingSession(object):
             from scrapy.core import signals
             from scrapy.core.manager import scrapymanager
             from scrapy.core.engine import scrapyengine
-            from pydispatch import dispatcher
+            from scrapy.xlib.pydispatch import dispatcher
 
             dispatcher.connect(self.record_signal, signals.engine_started)
             dispatcher.connect(self.record_signal, signals.engine_stopped)

scrapy/tests/test_spidermonkey.py

@@ -4,7 +4,7 @@ class SpidermonkeyTest(unittest.TestCase):
 
     def setUp(self):
         try:
-            from spidermonkey import Runtime
+            from scrapy.xlib.spidermonkey import Runtime
             r = Runtime()
             self.cx = r.new_context()
         except ImportError:

scrapy/xlib/ClientForm.py

@@ -919,7 +919,7 @@ def _create_bs_classes(bs,
 try:
     if sys.version_info[:2] < (2, 2):
         raise ImportError  # BeautifulSoup uses generators
-    import BeautifulSoup
+    from scrapy.xlib import BeautifulSoup
 except ImportError:
     pass
 else:

scrapy/xlib/pydispatch/dispatcher.py

@@ -27,7 +27,7 @@ Internal attributes:
 """
 from __future__ import generators
 import types, weakref
-from pydispatch import saferef, robustapply, errors
+from scrapy.xlib.pydispatch import saferef, robustapply, errors
 
 __author__ = "Patrick K. O'Brien <pobrien@orbtech.com>"
 __cvsid__ = "$Id$"

scrapy/xlib/pydispatch/robust.py

@@ -1,6 +1,6 @@
 """Module implementing error-catching version of send (sendRobust)"""
-from pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers
-from pydispatch.robustapply import robustApply
+from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers
+from scrapy.xlib.pydispatch.robustapply import robustApply
 
 def sendRobust(
 	signal=Any,