From a8de5cef6e2d388046c33e0c0c14e4effeb663e1 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 9 Jul 2009 14:37:59 -0300 Subject: [PATCH] remove xlib hack that appends scrapy/xlib to sys.path --- scrapy/__init__.py | 3 --- scrapy/command/cmdline.py | 2 +- scrapy/contrib/closedomain.py | 2 +- scrapy/contrib/cluster/master/manager.py | 2 +- scrapy/contrib/cluster/master/web.py | 2 +- scrapy/contrib/delayedclosedomain.py | 2 +- scrapy/contrib/downloadermiddleware/cookies.py | 2 +- scrapy/contrib/downloadermiddleware/httpcache.py | 2 +- scrapy/contrib/downloadermiddleware/robotstxt.py | 2 +- scrapy/contrib/itemsampler.py | 2 +- scrapy/contrib/memdebug.py | 2 +- scrapy/contrib/memusage.py | 2 +- scrapy/contrib/pipeline/media.py | 2 +- scrapy/contrib/response/soup.py | 2 +- scrapy/contrib/spider/profiler.py | 2 +- scrapy/contrib/spider/reloader.py | 2 +- scrapy/contrib/spidermiddleware/requestlimit.py | 2 +- scrapy/contrib/statsmailer.py | 2 +- scrapy/contrib/web/service.py | 2 +- scrapy/contrib/webconsole/enginestatus.py | 2 +- scrapy/contrib/webconsole/livestats.py | 2 +- scrapy/contrib/webconsole/scheduler.py | 2 +- scrapy/contrib/webconsole/spiderctl.py | 2 +- scrapy/contrib/webconsole/spiderstats.py | 2 +- scrapy/contrib/webconsole/stats.py | 2 +- scrapy/contrib_exp/history/middleware.py | 2 +- scrapy/contrib_exp/pipeline/shoveitem.py | 2 +- scrapy/core/downloader/dnscache.py | 2 +- scrapy/core/engine.py | 2 +- scrapy/core/signals.py | 2 +- scrapy/http/request/form.py | 2 +- scrapy/http/response/text.py | 2 +- scrapy/log/__init__.py | 2 +- scrapy/management/web.py | 2 +- scrapy/stats/corestats.py | 2 +- scrapy/stats/statscollector.py | 2 +- scrapy/tests/test_contrib_response_soup.py | 2 +- scrapy/tests/test_engine.py | 2 +- scrapy/tests/test_spidermonkey.py | 2 +- scrapy/xlib/ClientForm.py | 2 +- scrapy/xlib/pydispatch/dispatcher.py | 2 +- scrapy/xlib/pydispatch/robust.py | 4 ++-- 42 files changed, 42 insertions(+), 45 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index d30a484b4..b873829a8 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -11,9 +11,6 @@ if sys.version_info < (2,5): print "Scrapy %s requires Python 2.5 or above" % __version__ sys.exit(1) -# add external python libraries bundled into scrapy -sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "xlib")) - # monkey patches to fix external library issues from scrapy.patches import monkeypatches monkeypatches.apply_patches() diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index 535dc8085..821f6f31f 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -143,7 +143,7 @@ def execute_with_args(argv): p.runctx('ret = cmd.run(args, opts)', globals(), loc) p.dump_stats(opts.profile) try: - import lsprofcalltree + from scrapy.xlib import lsprofcalltree fn = opts.profile + ".cachegrind" k = lsprofcalltree.KCacheGrind(p) with open(fn, 'w') as f: diff --git a/scrapy/contrib/closedomain.py b/scrapy/contrib/closedomain.py index 570075a88..1dbd5560b 100644 --- a/scrapy/contrib/closedomain.py +++ b/scrapy/contrib/closedomain.py @@ -6,7 +6,7 @@ time has expired. import datetime import pprint -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log diff --git a/scrapy/contrib/cluster/master/manager.py b/scrapy/contrib/cluster/master/manager.py index 1fa13f1eb..6166fb421 100644 --- a/scrapy/contrib/cluster/master/manager.py +++ b/scrapy/contrib/cluster/master/manager.py @@ -3,7 +3,7 @@ from __future__ import with_statement import datetime import cPickle as pickle -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from twisted.spread import pb from twisted.internet import reactor diff --git a/scrapy/contrib/cluster/master/web.py b/scrapy/contrib/cluster/master/web.py index f1c1e9a42..31b88d617 100644 --- a/scrapy/contrib/cluster/master/web.py +++ b/scrapy/contrib/cluster/master/web.py @@ -1,6 +1,6 @@ import datetime -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.spider import spiders from scrapy.management.web import banner, webconsole_discover_module diff --git a/scrapy/contrib/delayedclosedomain.py b/scrapy/contrib/delayedclosedomain.py index 78b9de4c9..84743a6fc 100644 --- a/scrapy/contrib/delayedclosedomain.py +++ b/scrapy/contrib/delayedclosedomain.py @@ -5,7 +5,7 @@ configurable amount of idle time is reached from datetime import datetime -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from collections import defaultdict from scrapy.core import signals diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py index 6e1d04238..e64fc8b00 100644 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ b/scrapy/contrib/downloadermiddleware/cookies.py @@ -1,7 +1,7 @@ import operator from itertools import groupby from collections import defaultdict -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.http import Response diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 1f1738d39..94b65c7c9 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -5,7 +5,7 @@ import os import hashlib import datetime import cPickle as pickle -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 8e4295804..f7671878e 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. import urlparse import robotparser -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.engine import scrapyengine diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index e0774be80..ee18c2605 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -25,7 +25,7 @@ from __future__ import with_statement import cPickle as pickle -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index 9d9c3983c..acfd18569 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -9,7 +9,7 @@ import gc import socket import libxml2 -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.exceptions import NotConfigured diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index d1a961397..f3d86fb8b 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -9,7 +9,7 @@ import os import pprint import socket -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 7fbdddbf7..326058d60 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -1,4 +1,4 @@ -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from twisted.internet import defer from scrapy.utils.defer import mustbe_deferred, defer_result diff --git a/scrapy/contrib/response/soup.py b/scrapy/contrib/response/soup.py index 1caa5c217..84051f103 100644 --- a/scrapy/contrib/response/soup.py +++ b/scrapy/contrib/response/soup.py @@ -4,7 +4,7 @@ ResponseSoup extension See documentation in docs/ref/extensions.rst """ -from BeautifulSoup import BeautifulSoup +from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response diff --git a/scrapy/contrib/spider/profiler.py b/scrapy/contrib/spider/profiler.py index 2c8638bd7..804358a6d 100644 --- a/scrapy/contrib/spider/profiler.py +++ b/scrapy/contrib/spider/profiler.py @@ -11,7 +11,7 @@ when needed. import datetime -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.extension import extensions from scrapy.core import signals diff --git a/scrapy/contrib/spider/reloader.py b/scrapy/contrib/spider/reloader.py index 1aa116f12..6b60b7534 100644 --- a/scrapy/contrib/spider/reloader.py +++ b/scrapy/contrib/spider/reloader.py @@ -4,7 +4,7 @@ Reload spider modules once they are finished scraping This is to release any resources held on to by scraping spiders. """ import sys -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py index 48a3085ff..73779ae55 100644 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ b/scrapy/contrib/spidermiddleware/requestlimit.py @@ -8,7 +8,7 @@ setting "REQUESTS_QUEUE_SIZE". If not specified (or 0), no limit will be applied. """ from itertools import imap -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.engine import scrapyengine diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index e93799d49..c876714f6 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address import pprint -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.stats import stats from scrapy.mail import MailSender diff --git a/scrapy/contrib/web/service.py b/scrapy/contrib/web/service.py index 7859a6e4e..ac70c4cc0 100644 --- a/scrapy/contrib/web/service.py +++ b/scrapy/contrib/web/service.py @@ -10,7 +10,7 @@ from scrapy.item import ScrapedItem from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings from scrapy.utils.misc import memoize, arg_to_iter -from lrucache import LRUCache +from scrapy.xlib.lrucache import LRUCache from .site import WebSite, WebResource from .http import HttpResponse diff --git a/scrapy/contrib/webconsole/enginestatus.py b/scrapy/contrib/webconsole/enginestatus.py index e90b9ab30..f34d3a375 100644 --- a/scrapy/contrib/webconsole/enginestatus.py +++ b/scrapy/contrib/webconsole/enginestatus.py @@ -1,7 +1,7 @@ """ Scheduler information module for Scrapy webconsole """ -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import scrapyengine from scrapy.management.web import banner diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index e69035680..0fe8b84f0 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -2,7 +2,7 @@ Live statistics extension """ from datetime import datetime -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.engine import scrapyengine from scrapy.management.web import banner, webconsole_discover_module diff --git a/scrapy/contrib/webconsole/scheduler.py b/scrapy/contrib/webconsole/scheduler.py index 472a138f9..805e3423c 100644 --- a/scrapy/contrib/webconsole/scheduler.py +++ b/scrapy/contrib/webconsole/scheduler.py @@ -4,7 +4,7 @@ Scheduler queue web console module See documentation in docs/ref/extensions.rst """ -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import scrapyengine from scrapy.management.web import banner, webconsole_discover_module diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index f6964b64b..bf7c3fdb6 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -2,7 +2,7 @@ Extensions for allowing spider control from web console """ -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.manager import scrapymanager diff --git a/scrapy/contrib/webconsole/spiderstats.py b/scrapy/contrib/webconsole/spiderstats.py index df8d56515..cbf2bfb8a 100644 --- a/scrapy/contrib/webconsole/spiderstats.py +++ b/scrapy/contrib/webconsole/spiderstats.py @@ -4,7 +4,7 @@ Extension for persistent statistics by domain import re import pprint -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.spider import spiders from scrapy.core.exceptions import NotConfigured diff --git a/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py index f58cb50e8..0ce81e0be 100644 --- a/scrapy/contrib/webconsole/stats.py +++ b/scrapy/contrib/webconsole/stats.py @@ -1,6 +1,6 @@ import pprint -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.stats import stats from scrapy.management.web import banner, webconsole_discover_module diff --git a/scrapy/contrib_exp/history/middleware.py b/scrapy/contrib_exp/history/middleware.py index 21b6d5aa6..4baf3767f 100644 --- a/scrapy/contrib_exp/history/middleware.py +++ b/scrapy/contrib_exp/history/middleware.py @@ -1,7 +1,7 @@ import hashlib from datetime import datetime -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.misc import load_object from scrapy.core import signals diff --git a/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py index 6504bc429..5bd377769 100644 --- a/scrapy/contrib_exp/pipeline/shoveitem.py +++ b/scrapy/contrib_exp/pipeline/shoveitem.py @@ -8,7 +8,7 @@ http://pypi.python.org/pypi/shove from string import Template from shove import Shove -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy import log from scrapy.core import signals diff --git a/scrapy/core/downloader/dnscache.py b/scrapy/core/downloader/dnscache.py index 1a8418391..5ad6be7e6 100644 --- a/scrapy/core/downloader/dnscache.py +++ b/scrapy/core/downloader/dnscache.py @@ -5,7 +5,7 @@ crawler, reducing the dns lookups. """ import socket from scrapy.core import signals -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher class DNSCache(object): """ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index acc933980..dc58e78e0 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,7 +9,7 @@ from datetime import datetime from twisted.internet import reactor, task from twisted.internet.error import CannotListenError from twisted.python.failure import Failure -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy import log from scrapy.conf import settings diff --git a/scrapy/core/signals.py b/scrapy/core/signals.py index 341c07fa3..7a6f0ef19 100644 --- a/scrapy/core/signals.py +++ b/scrapy/core/signals.py @@ -5,7 +5,7 @@ These signals are documented in docs/ref/signals.rst. Please don't add new signals here without documenting them there. """ -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy import log diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 252c193a4..55e5ea1fa 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/ref/request-response.rst import urllib from cStringIO import StringIO -from ClientForm import ParseFile +from scrapy.xlib.ClientForm import ParseFile from scrapy.http.request import Request from scrapy.utils.python import unicode_to_str diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 67227ba86..dc8a96d27 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -7,7 +7,7 @@ See documentation in docs/ref/request-response.rst import re -from BeautifulSoup import UnicodeDammit +from scrapy.xlib.BeautifulSoup import UnicodeDammit from scrapy.http.response import Response from scrapy.utils.python import memoizemethod diff --git a/scrapy/log/__init__.py b/scrapy/log/__init__.py index 9668fe377..384116ade 100644 --- a/scrapy/log/__init__.py +++ b/scrapy/log/__init__.py @@ -7,7 +7,7 @@ import sys from traceback import format_exc from twisted.python import log -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.conf import settings from scrapy.utils.python import unicode_to_str diff --git a/scrapy/management/web.py b/scrapy/management/web.py index 7073273f2..9078e6f35 100644 --- a/scrapy/management/web.py +++ b/scrapy/management/web.py @@ -9,7 +9,7 @@ import socket from datetime import datetime from twisted.web import server, resource -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import scrapyengine from scrapy.conf import settings diff --git a/scrapy/stats/corestats.py b/scrapy/stats/corestats.py index 200993daf..0988807f4 100644 --- a/scrapy/stats/corestats.py +++ b/scrapy/stats/corestats.py @@ -6,7 +6,7 @@ import getpass import socket import datetime -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.stats import stats diff --git a/scrapy/stats/statscollector.py b/scrapy/stats/statscollector.py index e9f21483b..5394e215d 100644 --- a/scrapy/stats/statscollector.py +++ b/scrapy/stats/statscollector.py @@ -3,7 +3,7 @@ Scrapy extension for collecting scraping stats """ import pprint -from pydispatch import dispatcher +from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log diff --git a/scrapy/tests/test_contrib_response_soup.py b/scrapy/tests/test_contrib_response_soup.py index 92bca7f04..e95239d68 100644 --- a/scrapy/tests/test_contrib_response_soup.py +++ b/scrapy/tests/test_contrib_response_soup.py @@ -1,6 +1,6 @@ import unittest -from BeautifulSoup import BeautifulSoup +from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response from scrapy.contrib.response.soup import ResponseSoup diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index e28e7143a..c2970c90f 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -58,7 +58,7 @@ class CrawlingSession(object): from scrapy.core import signals from scrapy.core.manager import scrapymanager from scrapy.core.engine import scrapyengine - from pydispatch import dispatcher + from scrapy.xlib.pydispatch import dispatcher dispatcher.connect(self.record_signal, signals.engine_started) dispatcher.connect(self.record_signal, signals.engine_stopped) diff --git a/scrapy/tests/test_spidermonkey.py b/scrapy/tests/test_spidermonkey.py index 52b753d32..fb8664895 100644 --- a/scrapy/tests/test_spidermonkey.py +++ b/scrapy/tests/test_spidermonkey.py @@ -4,7 +4,7 @@ class SpidermonkeyTest(unittest.TestCase): def setUp(self): try: - from spidermonkey import Runtime + from scrapy.xlib.spidermonkey import Runtime r = Runtime() self.cx = r.new_context() except ImportError: diff --git a/scrapy/xlib/ClientForm.py b/scrapy/xlib/ClientForm.py index a622de7b6..7e9203d21 100644 --- a/scrapy/xlib/ClientForm.py +++ b/scrapy/xlib/ClientForm.py @@ -919,7 +919,7 @@ def _create_bs_classes(bs, try: if sys.version_info[:2] < (2, 2): raise ImportError # BeautifulSoup uses generators - import BeautifulSoup + from scrapy.xlib import BeautifulSoup except ImportError: pass else: diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py index 413100b26..d5c5b0034 100644 --- a/scrapy/xlib/pydispatch/dispatcher.py +++ b/scrapy/xlib/pydispatch/dispatcher.py @@ -27,7 +27,7 @@ Internal attributes: """ from __future__ import generators import types, weakref -from pydispatch import saferef, robustapply, errors +from scrapy.xlib.pydispatch import saferef, robustapply, errors __author__ = "Patrick K. O'Brien " __cvsid__ = "$Id$" diff --git a/scrapy/xlib/pydispatch/robust.py b/scrapy/xlib/pydispatch/robust.py index 929f550c6..149b070d7 100644 --- a/scrapy/xlib/pydispatch/robust.py +++ b/scrapy/xlib/pydispatch/robust.py @@ -1,6 +1,6 @@ """Module implementing error-catching version of send (sendRobust)""" -from pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers -from pydispatch.robustapply import robustApply +from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers +from scrapy.xlib.pydispatch.robustapply import robustApply def sendRobust( signal=Any,