From 776064aee19b545f80c37c45ba64a6332199f583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 25 Jul 2015 17:38:07 +0200 Subject: [PATCH] PY3 port signals --- requirements.txt | 1 + scrapy/signalmanager.py | 2 +- scrapy/utils/signal.py | 12 +- scrapy/xlib/pydispatch/__init__.py | 6 - scrapy/xlib/pydispatch/dispatcher.py | 511 -------------------------- scrapy/xlib/pydispatch/errors.py | 15 - scrapy/xlib/pydispatch/license.txt | 34 -- scrapy/xlib/pydispatch/robust.py | 54 --- scrapy/xlib/pydispatch/robustapply.py | 58 --- scrapy/xlib/pydispatch/saferef.py | 180 --------- setup.py | 1 + tests/py3-ignores.txt | 1 - tests/test_engine.py | 2 +- tests/test_utils_signal.py | 2 +- 14 files changed, 14 insertions(+), 865 deletions(-) delete mode 100644 scrapy/xlib/pydispatch/__init__.py delete mode 100644 scrapy/xlib/pydispatch/dispatcher.py delete mode 100644 scrapy/xlib/pydispatch/errors.py delete mode 100644 scrapy/xlib/pydispatch/license.txt delete mode 100644 scrapy/xlib/pydispatch/robust.py delete mode 100644 scrapy/xlib/pydispatch/robustapply.py delete mode 100644 scrapy/xlib/pydispatch/saferef.py diff --git a/requirements.txt b/requirements.txt index 005b8f4f5..03e5329c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ cssselect>=0.9 w3lib>=1.8.0 queuelib six>=1.5.2 +PyDispatcher>=2.0.5 diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index bcc0df2c6..fd79905e9 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,5 +1,5 @@ from __future__ import absolute_import -from scrapy.xlib.pydispatch import dispatcher +from pydispatch import dispatcher from scrapy.utils import signal as _signal diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index d9a59e161..de00bac49 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -5,19 +5,23 @@ import logging from twisted.internet.defer import maybeDeferred, DeferredList, Deferred from twisted.python.failure import Failure -from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, \ +from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \ getAllReceivers, disconnect -from scrapy.xlib.pydispatch.robustapply import robustApply +from pydispatch.robustapply import robustApply from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) +class _IgnoredException(Exception): + pass + + def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = named.pop('dont_log', None) + dont_log = named.pop('dont_log', _IgnoredException) spider = named.get('spider', None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): @@ -39,6 +43,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): responses.append((receiver, result)) return responses + def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were @@ -65,6 +70,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): d.addCallback(lambda out: [x[1] for x in out]) return d + def disconnect_all(signal=Any, sender=Any): """Disconnect all signal handlers. Useful for cleaning up after running tests diff --git a/scrapy/xlib/pydispatch/__init__.py b/scrapy/xlib/pydispatch/__init__.py deleted file mode 100644 index 78511a0d8..000000000 --- a/scrapy/xlib/pydispatch/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Multi-consumer multi-producer dispatching mechanism -""" -__version__ = "2.0.0" -__author__ = "Patrick K. O'Brien" -__license__ = "BSD-style, see license.txt for details" - diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py deleted file mode 100644 index 7d588dd20..000000000 --- a/scrapy/xlib/pydispatch/dispatcher.py +++ /dev/null @@ -1,511 +0,0 @@ -"""Multiple-producer-multiple-consumer signal-dispatching - -dispatcher is the core of the PyDispatcher system, -providing the primary API and the core logic for the -system. - -Module attributes of note: - - Any -- Singleton used to signal either "Any Sender" or - "Any Signal". See documentation of the _Any class. - Anonymous -- Singleton used to signal "Anonymous Sender" - See documentation of the _Anonymous class. - -Internal attributes: - WEAKREF_TYPES -- tuple of types/classes which represent - weak references to receivers, and thus must be de- - referenced on retrieval to retrieve the callable - object - connections -- { senderkey (id) : { signal : [receivers...]}} - senders -- { senderkey (id) : weakref(sender) } - used for cleaning up sender references on sender - deletion - sendersBack -- { receiverkey (id) : [senderkey (id)...] } - used for cleaning up receiver references on receiver - deletion, (considerably speeds up the cleanup process - vs. the original code.) -""" -from __future__ import generators -import types, weakref, six -from scrapy.xlib.pydispatch import saferef, robustapply, errors - -__author__ = "Patrick K. O'Brien " -__cvsid__ = "$Id: dispatcher.py,v 1.1.1.1 2006/07/07 15:59:38 mcfletch Exp $" -__version__ = "$Revision: 1.1.1.1 $"[11:-2] - - -class _Parameter: - """Used to represent default parameter values.""" - - def __repr__(self): - return self.__class__.__name__ - - -class _Any(_Parameter): - """Singleton used to signal either "Any Sender" or "Any Signal" - - The Any object can be used with connect, disconnect, - send, or sendExact to signal that the parameter given - Any should react to all senders/signals, not just - a particular sender/signal. - """ - - -Any = _Any() - - -class _Anonymous(_Parameter): - """Singleton used to signal "Anonymous Sender" - - The Anonymous object is used to signal that the sender - of a message is not specified (as distinct from being - "any sender"). Registering callbacks for Anonymous - will only receive messages sent without senders. Sending - with anonymous will only send messages to those receivers - registered for Any or Anonymous. - - Note: - The default sender for connect is Any, while the - default sender for send is Anonymous. This has - the effect that if you do not specify any senders - in either function then all messages are routed - as though there was a single sender (Anonymous) - being used everywhere. - """ - - -Anonymous = _Anonymous() - -WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref) - -connections = {} -senders = {} -sendersBack = {} - - -def connect(receiver, signal=Any, sender=Any, weak=True): - """Connect receiver to sender for signal - - receiver -- a callable Python object which is to receive - messages/signals/events. Receivers must be hashable - objects. - - if weak is True, then receiver must be weak-referencable - (more precisely saferef.safeRef() must be able to create - a reference to the receiver). - - Receivers are fairly flexible in their specification, - as the machinery in the robustApply module takes care - of most of the details regarding figuring out appropriate - subsets of the sent arguments to apply to a given - receiver. - - Note: - if receiver is itself a weak reference (a callable), - it will be de-referenced by the system's machinery, - so *generally* weak references are not suitable as - receivers, though some use might be found for the - facility whereby a higher-level library passes in - pre-weakrefed receiver references. - - signal -- the signal to which the receiver should respond - - if Any, receiver will receive any signal from the - indicated sender (which might also be Any, but is not - necessarily Any). - - Otherwise must be a hashable Python object other than - None (DispatcherError raised on None). - - sender -- the sender to which the receiver should respond - - if Any, receiver will receive the indicated signals - from any sender. - - if Anonymous, receiver will only receive indicated - signals from send/sendExact which do not specify a - sender, or specify Anonymous explicitly as the sender. - - Otherwise can be any python object. - - weak -- whether to use weak references to the receiver - By default, the module will attempt to use weak - references to the receiver objects. If this parameter - is false, then strong references will be used. - - returns None, may raise DispatcherTypeError - """ - if signal is None: - raise errors.DispatcherTypeError( - 'Signal cannot be None (receiver=%r sender=%r)' % ( - receiver, sender) - ) - if weak: - receiver = saferef.safeRef(receiver, onDelete=_removeReceiver) - senderkey = id(sender) - if senderkey in connections: - signals = connections[senderkey] - else: - connections[senderkey] = signals = {} - # Keep track of senders for cleanup. - # Is Anonymous something we want to clean up? - if sender not in (None, Anonymous, Any): - def remove(object, senderkey=senderkey): - _removeSender(senderkey=senderkey) - - # Skip objects that can not be weakly referenced, which means - # they won't be automatically cleaned up, but that's too bad. - try: - weakSender = weakref.ref(sender, remove) - senders[senderkey] = weakSender - except: - pass - - receiverID = id(receiver) - # get current set, remove any current references to - # this receiver in the set, including back-references - if signal in signals: - receivers = signals[signal] - _removeOldBackRefs(senderkey, signal, receiver, receivers) - else: - receivers = signals[signal] = [] - try: - current = sendersBack.get(receiverID) - if current is None: - sendersBack[receiverID] = current = [] - if senderkey not in current: - current.append(senderkey) - except: - pass - - receivers.append(receiver) - - -def disconnect(receiver, signal=Any, sender=Any, weak=True): - """Disconnect receiver from sender for signal - - receiver -- the registered receiver to disconnect - signal -- the registered signal to disconnect - sender -- the registered sender to disconnect - weak -- the weakref state to disconnect - - disconnect reverses the process of connect, - the semantics for the individual elements are - logically equivalent to a tuple of - (receiver, signal, sender, weak) used as a key - to be deleted from the internal routing tables. - (The actual process is slightly more complex - but the semantics are basically the same). - - Note: - Using disconnect is not required to cleanup - routing when an object is deleted, the framework - will remove routes for deleted objects - automatically. It's only necessary to disconnect - if you want to stop routing to a live object. - - returns None, may raise DispatcherTypeError or - DispatcherKeyError - """ - if signal is None: - raise errors.DispatcherTypeError( - 'Signal cannot be None (receiver=%r sender=%r)' % ( - receiver, sender) - ) - if weak: receiver = saferef.safeRef(receiver) - senderkey = id(sender) - try: - signals = connections[senderkey] - receivers = signals[signal] - except KeyError: - raise errors.DispatcherKeyError( - """No receivers found for signal %r from sender %r""" % ( - signal, - sender - ) - ) - try: - # also removes from receivers - _removeOldBackRefs(senderkey, signal, receiver, receivers) - except ValueError: - raise errors.DispatcherKeyError( - """No connection to receiver %s for signal %s from sender %s""" % ( - receiver, - signal, - sender - ) - ) - _cleanupConnections(senderkey, signal) - - -def getReceivers(sender=Any, signal=Any): - """Get list of receivers from global tables - - This utility function allows you to retrieve the - raw list of receivers from the connections table - for the given sender and signal pair. - - Note: - there is no guarantee that this is the actual list - stored in the connections table, so the value - should be treated as a simple iterable/truth value - rather than, for instance a list to which you - might append new records. - - Normally you would use liveReceivers( getReceivers( ...)) - to retrieve the actual receiver objects as an iterable - object. - """ - try: - return connections[id(sender)][signal] - except KeyError: - return [] - - -def liveReceivers(receivers): - """Filter sequence of receivers to get resolved, live receivers - - This is a generator which will iterate over - the passed sequence, checking for weak references - and resolving them, then returning all live - receivers. - """ - for receiver in receivers: - if isinstance(receiver, WEAKREF_TYPES): - # Dereference the weak reference. - receiver = receiver() - if receiver is not None: - yield receiver - else: - yield receiver - - -def getAllReceivers(sender=Any, signal=Any): - """Get list of all receivers from global tables - - This gets all receivers which should receive - the given signal from sender, each receiver should - be produced only once by the resulting generator - """ - receivers = {} - for set in ( - # Get receivers that receive *this* signal from *this* sender. - getReceivers(sender, signal), - # Add receivers that receive *any* signal from *this* sender. - getReceivers(sender, Any), - # Add receivers that receive *this* signal from *any* sender. - getReceivers(Any, signal), - # Add receivers that receive *any* signal from *any* sender. - getReceivers(Any, Any), - ): - for receiver in set: - if receiver: # filter out dead instance-method weakrefs - try: - if receiver not in receivers: - receivers[receiver] = 1 - yield receiver - except TypeError: - # dead weakrefs raise TypeError on hash... - pass - - -def send(signal=Any, sender=Anonymous, *arguments, **named): - """Send signal from sender to all connected receivers. - - signal -- (hashable) signal value, see connect for details - - sender -- the sender of the signal - - if Any, only receivers registered for Any will receive - the message. - - if Anonymous, only receivers registered to receive - messages from Anonymous or Any will receive the message - - Otherwise can be any python object (normally one - registered with a connect if you actually want - something to occur). - - arguments -- positional arguments which will be passed to - *all* receivers. Note that this may raise TypeErrors - if the receivers do not allow the particular arguments. - Note also that arguments are applied before named - arguments, so they should be used with care. - - named -- named arguments which will be filtered according - to the parameters of the receivers to only provide those - acceptable to the receiver. - - Return a list of tuple pairs [(receiver, response), ... ] - - if any receiver raises an error, the error propagates back - through send, terminating the dispatch loop, so it is quite - possible to not have all receivers called if a raises an - error. - """ - # Call each receiver with whatever arguments it can accept. - # Return a list of tuple pairs [(receiver, response), ... ]. - responses = [] - for receiver in liveReceivers(getAllReceivers(sender, signal)): - response = robustapply.robustApply( - receiver, - signal=signal, - sender=sender, - *arguments, - **named - ) - responses.append((receiver, response)) - return responses - - -def sendExact(signal=Any, sender=Anonymous, *arguments, **named): - """Send signal only to those receivers registered for exact message - - sendExact allows for avoiding Any/Anonymous registered - handlers, sending only to those receivers explicitly - registered for a particular signal on a particular - sender. - """ - responses = [] - for receiver in liveReceivers(getReceivers(sender, signal)): - response = robustapply.robustApply( - receiver, - signal=signal, - sender=sender, - *arguments, - **named - ) - responses.append((receiver, response)) - return responses - - -def _removeReceiver(receiver): - """Remove receiver from connections.""" - if not sendersBack: - # During module cleanup the mapping will be replaced with None - return False - backKey = id(receiver) - try: - backSet = sendersBack.pop(backKey) - except KeyError as err: - return False - else: - for senderkey in backSet: - try: - signals = connections[senderkey].keys() - except KeyError as err: - pass - else: - for signal in signals: - try: - receivers = connections[senderkey][signal] - except KeyError: - pass - else: - try: - receivers.remove(receiver) - except Exception as err: - pass - _cleanupConnections(senderkey, signal) - - -def _cleanupConnections(senderkey, signal): - """Delete any empty signals for senderkey. Delete senderkey if empty.""" - try: - receivers = connections[senderkey][signal] - except: - pass - else: - if not receivers: - # No more connected receivers. Therefore, remove the signal. - try: - signals = connections[senderkey] - except KeyError: - pass - else: - del signals[signal] - if not signals: - # No more signal connections. Therefore, remove the sender. - _removeSender(senderkey) - - -def _removeSender(senderkey): - """Remove senderkey from connections.""" - _removeBackrefs(senderkey) - try: - del connections[senderkey] - except KeyError: - pass - # Senderkey will only be in senders dictionary if sender - # could be weakly referenced. - try: - del senders[senderkey] - except: - pass - - -def _removeBackrefs(senderkey): - """Remove all back-references to this senderkey""" - try: - signals = connections[senderkey] - except KeyError: - signals = None - else: - items = signals.items() - - def allReceivers(): - for signal, set in items: - for item in set: - yield item - - for receiver in allReceivers(): - _killBackref(receiver, senderkey) - - -def _removeOldBackRefs(senderkey, signal, receiver, receivers): - """Kill old sendersBack references from receiver - - This guards against multiple registration of the same - receiver for a given signal and sender leaking memory - as old back reference records build up. - - Also removes old receiver instance from receivers - """ - try: - index = receivers.index(receiver) - # need to scan back references here and remove senderkey - except ValueError: - return False - else: - oldReceiver = receivers[index] - del receivers[index] - found = 0 - signals = connections.get(signal) - if signals is not None: - for sig, recs in six.iteritems(connections.get(signal, {})): - if sig != signal: - for rec in recs: - if rec is oldReceiver: - found = 1 - break - if not found: - _killBackref(oldReceiver, senderkey) - return True - return False - - -def _killBackref(receiver, senderkey): - """Do the actual removal of back reference from receiver to senderkey""" - receiverkey = id(receiver) - set = sendersBack.get(receiverkey, ()) - while senderkey in set: - try: - set.remove(senderkey) - except: - break - if not set: - try: - del sendersBack[receiverkey] - except KeyError: - pass - return True diff --git a/scrapy/xlib/pydispatch/errors.py b/scrapy/xlib/pydispatch/errors.py deleted file mode 100644 index c5540d8f7..000000000 --- a/scrapy/xlib/pydispatch/errors.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Error types for dispatcher mechanism -""" - - -class DispatcherError(Exception): - """Base class for all Dispatcher errors""" - - -class DispatcherKeyError(KeyError, DispatcherError): - """Error raised when unknown (sender,signal) set specified""" - - -class DispatcherTypeError(TypeError, DispatcherError): - """Error raised when inappropriate signal-type specified (None)""" - diff --git a/scrapy/xlib/pydispatch/license.txt b/scrapy/xlib/pydispatch/license.txt deleted file mode 100644 index 0960a6073..000000000 --- a/scrapy/xlib/pydispatch/license.txt +++ /dev/null @@ -1,34 +0,0 @@ -PyDispatcher License - - Copyright (c) 2001-2006, Patrick K. O'Brien and Contributors - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - - The name of Patrick K. O'Brien, or the name of any Contributor, - may not be used to endorse or promote products derived from this - software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/scrapy/xlib/pydispatch/robust.py b/scrapy/xlib/pydispatch/robust.py deleted file mode 100644 index a28f127e2..000000000 --- a/scrapy/xlib/pydispatch/robust.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Module implementing error-catching version of send (sendRobust)""" -from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers -from scrapy.xlib.pydispatch.robustapply import robustApply - - -def sendRobust(signal=Any, sender=Anonymous, *arguments, **named): - """Send signal from sender to all connected receivers catching errors - - signal -- (hashable) signal value, see connect for details - - sender -- the sender of the signal - - if Any, only receivers registered for Any will receive - the message. - - if Anonymous, only receivers registered to receive - messages from Anonymous or Any will receive the message - - Otherwise can be any python object (normally one - registered with a connect if you actually want - something to occur). - - arguments -- positional arguments which will be passed to - *all* receivers. Note that this may raise TypeErrors - if the receivers do not allow the particular arguments. - Note also that arguments are applied before named - arguments, so they should be used with care. - - named -- named arguments which will be filtered according - to the parameters of the receivers to only provide those - acceptable to the receiver. - - Return a list of tuple pairs [(receiver, response), ... ] - - if any receiver raises an error (specifically any subclass of Exception), - the error instance is returned as the result for that receiver. - """ - # Call each receiver with whatever arguments it can accept. - # Return a list of tuple pairs [(receiver, response), ... ]. - responses = [] - for receiver in liveReceivers(getAllReceivers(sender, signal)): - try: - response = robustApply( - receiver, - signal=signal, - sender=sender, - *arguments, - **named - ) - except Exception as err: - responses.append((receiver, err)) - else: - responses.append((receiver, response)) - return responses diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py deleted file mode 100644 index 9977ac6b7..000000000 --- a/scrapy/xlib/pydispatch/robustapply.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Robust apply mechanism - -Provides a function "call", which can sort out -what arguments a given callable object can take, -and subset the given arguments to match only -those which are acceptable. -""" -import inspect - - -def function(receiver): - """Get function-like callable object for given receiver - - returns (function_or_method, codeObject, fromMethod) - - If fromMethod is true, then the callable already - has its first argument bound - """ - if inspect.isclass(receiver) and hasattr(receiver, '__call__'): - # receiver is a class instance; assume it is callable. - # Reassign receiver to the actual method that will be called. - if hasattr(receiver.__call__, 'im_func') or \ - hasattr(receiver.__call__, 'im_code'): - receiver = receiver.__call__ - - if hasattr(receiver, 'im_func'): - # an instance-method... - return receiver, receiver.im_func.func_code, 1 - elif not hasattr(receiver, 'func_code'): - raise ValueError( - 'unknown receiver type %s %s' % (receiver, type(receiver))) - - return receiver, receiver.func_code, 0 - - -def robustApply(receiver, *arguments, **named): - """Call receiver with arguments and an appropriate subset of named - """ - receiver, codeObject, startIndex = function(receiver) - acceptable = codeObject.co_varnames[ - startIndex + len(arguments):codeObject.co_argcount] - for name in codeObject.co_varnames[startIndex:startIndex + len(arguments)]: - if name in named: - raise TypeError( - """Argument %r specified both positionally and as a keyword for calling %r""" % ( - name, receiver, - ) - ) - - if not (codeObject.co_flags & 8): - # fc does not have a **kwds type parameter, therefore - # remove unacceptable arguments. - for arg in named.keys(): - if arg not in acceptable: - del named[arg] - - return receiver(*arguments, **named) - diff --git a/scrapy/xlib/pydispatch/saferef.py b/scrapy/xlib/pydispatch/saferef.py deleted file mode 100644 index bd9659673..000000000 --- a/scrapy/xlib/pydispatch/saferef.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Refactored "safe reference" from dispatcher.py""" -from __future__ import print_function -import weakref, traceback - - -def safeRef(target, onDelete=None): - """Return a *safe* weak reference to a callable target - - target -- the object to be weakly referenced, if it's a - bound method reference, will create a BoundMethodWeakref, - otherwise creates a simple weakref. - onDelete -- if provided, will have a hard reference stored - to the callable to be called after the safe reference - goes out of scope with the reference object, (either a - weakref or a BoundMethodWeakref) as argument. - """ - if hasattr(target, 'im_self'): - if target.im_self is not None: - # Turn a bound method into a BoundMethodWeakref instance. - # Keep track of these instances for lookup by disconnect(). - assert hasattr(target, 'im_func'), """safeRef target %r has im_self, but no im_func, don't know how to create reference"""%( target,) - reference = BoundMethodWeakref( - target=target, - onDelete=onDelete - ) - return reference - if onDelete is not None: - return weakref.ref(target, onDelete) - else: - return weakref.ref(target) - - -class BoundMethodWeakref(object): - """'Safe' and reusable weak references to instance methods - - BoundMethodWeakref objects provide a mechanism for - referencing a bound method without requiring that the - method object itself (which is normally a transient - object) is kept alive. Instead, the BoundMethodWeakref - object keeps weak references to both the object and the - function which together define the instance method. - - Attributes: - key -- the identity key for the reference, calculated - by the class's calculateKey method applied to the - target instance method - deletionMethods -- sequence of callable objects taking - single argument, a reference to this object which - will be called when *either* the target object or - target function is garbage collected (i.e. when - this object becomes invalid). These are specified - as the onDelete parameters of safeRef calls. - weakSelf -- weak reference to the target object - weakFunc -- weak reference to the target function - - Class Attributes: - _allInstances -- class attribute pointing to all live - BoundMethodWeakref objects indexed by the class's - calculateKey(target) method applied to the target - objects. This weak value dictionary is used to - short-circuit creation so that multiple references - to the same (object, function) pair produce the - same BoundMethodWeakref instance. - - """ - _allInstances = weakref.WeakValueDictionary() - - def __new__(cls, target, onDelete=None, *arguments, **named): - """Create new instance or return current instance - - Basically this method of construction allows us to - short-circuit creation of references to already- - referenced instance methods. The key corresponding - to the target is calculated, and if there is already - an existing reference, that is returned, with its - deletionMethods attribute updated. Otherwise the - new instance is created and registered in the table - of already-referenced methods. - """ - key = cls.calculateKey(target) - current = cls._allInstances.get(key) - if current is not None: - current.deletionMethods.append(onDelete) - return current - else: - base = super(BoundMethodWeakref, cls).__new__(cls) - cls._allInstances[key] = base - base.__init__(target, onDelete, *arguments, **named) - return base - - def __init__(self, target, onDelete=None): - """Return a weak-reference-like instance for a bound method - - target -- the instance-method target for the weak - reference, must have im_self and im_func attributes - and be reconstructable via: - target.im_func.__get__( target.im_self ) - which is true of built-in instance methods. - onDelete -- optional callback which will be called - when this weak reference ceases to be valid - (i.e. either the object or the function is garbage - collected). Should take a single argument, - which will be passed a pointer to this object. - """ - - def remove(weak, self=self): - """Set self.isDead to true when method or instance is destroyed""" - methods = self.deletionMethods[:] - del self.deletionMethods[:] - try: - del self.__class__._allInstances[self.key] - except KeyError: - pass - for function in methods: - try: - if callable(function): - function(self) - except Exception as e: - try: - traceback.print_exc() - except AttributeError as err: - print( - '''Exception during saferef %s cleanup function %s: %s''' % ( - self, function, e - )) - - self.deletionMethods = [onDelete] - self.key = self.calculateKey(target) - self.weakSelf = weakref.ref(target.im_self, remove) - self.weakFunc = weakref.ref(target.im_func, remove) - self.selfName = target.im_self.__class__.__name__ - self.funcName = str(target.im_func.__name__) - - def calculateKey(cls, target): - """Calculate the reference key for this reference - - Currently this is a two-tuple of the id()'s of the - target object and the target function respectively. - """ - return (id(target.im_self), id(target.im_func)) - - calculateKey = classmethod(calculateKey) - - def __str__(self): - """Give a friendly representation of the object""" - return """%s( %s.%s )""" % ( - self.__class__.__name__, - self.selfName, - self.funcName, - ) - - __repr__ = __str__ - - def __nonzero__(self): - """Whether we are still a valid reference""" - return self() is not None - - def __cmp__(self, other): - """Compare with another reference""" - if not isinstance(other, self.__class__): - return cmp(self.__class__, type(other)) - return cmp(self.key, other.key) - - def __call__(self): - """Return a strong reference to the bound method - - If the target cannot be retrieved, then will - return None, otherwise returns a bound instance - method for our object and function. - - Note: - You may call this method any number of times, - as it does not invalidate the reference. - """ - target = self.weakSelf() - if target is not None: - function = self.weakFunc() - if function is not None: - return function.__get__(target) - return None diff --git a/setup.py b/setup.py index bacdca358..6dc1cfd94 100644 --- a/setup.py +++ b/setup.py @@ -44,5 +44,6 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', + 'PyDispatcher>=2.0.5', ], ) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 6c14adf43..9d84e7634 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -49,7 +49,6 @@ tests/test_utils_iterators.py tests/test_utils_log.py tests/test_utils_reqser.py tests/test_utils_response.py -tests/test_utils_signal.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_engine.py b/tests/test_engine.py index d7ad88abb..e14957eae 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -20,7 +20,7 @@ from twisted.trial import unittest from scrapy import signals from scrapy.utils.test import get_crawler -from scrapy.xlib.pydispatch import dispatcher +from pydispatch import dispatcher from tests import tests_datadir from scrapy.spiders import Spider from scrapy.item import Item, Field diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 571073911..b7de85049 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -2,8 +2,8 @@ from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure from twisted.internet import defer, reactor +from pydispatch import dispatcher -from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.signal import send_catch_log, send_catch_log_deferred