diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py index f5a9a611b..7d588dd20 100644 --- a/scrapy/xlib/pydispatch/dispatcher.py +++ b/scrapy/xlib/pydispatch/dispatcher.py @@ -6,24 +6,24 @@ system. Module attributes of note: - Any -- Singleton used to signal either "Any Sender" or - "Any Signal". See documentation of the _Any class. - Anonymous -- Singleton used to signal "Anonymous Sender" - See documentation of the _Anonymous class. + Any -- Singleton used to signal either "Any Sender" or + "Any Signal". See documentation of the _Any class. + Anonymous -- Singleton used to signal "Anonymous Sender" + See documentation of the _Anonymous class. Internal attributes: - WEAKREF_TYPES -- tuple of types/classes which represent - weak references to receivers, and thus must be de- - referenced on retrieval to retrieve the callable - object - connections -- { senderkey (id) : { signal : [receivers...]}} - senders -- { senderkey (id) : weakref(sender) } - used for cleaning up sender references on sender - deletion - sendersBack -- { receiverkey (id) : [senderkey (id)...] } - used for cleaning up receiver references on receiver - deletion, (considerably speeds up the cleanup process - vs. the original code.) + WEAKREF_TYPES -- tuple of types/classes which represent + weak references to receivers, and thus must be de- + referenced on retrieval to retrieve the callable + object + connections -- { senderkey (id) : { signal : [receivers...]}} + senders -- { senderkey (id) : weakref(sender) } + used for cleaning up sender references on sender + deletion + sendersBack -- { receiverkey (id) : [senderkey (id)...] } + used for cleaning up receiver references on receiver + deletion, (considerably speeds up the cleanup process + vs. the original code.) """ from __future__ import generators import types, weakref, six @@ -33,39 +33,47 @@ __author__ = "Patrick K. O'Brien " __cvsid__ = "$Id: dispatcher.py,v 1.1.1.1 2006/07/07 15:59:38 mcfletch Exp $" __version__ = "$Revision: 1.1.1.1 $"[11:-2] + class _Parameter: - """Used to represent default parameter values.""" - def __repr__(self): - return self.__class__.__name__ + """Used to represent default parameter values.""" + + def __repr__(self): + return self.__class__.__name__ + class _Any(_Parameter): - """Singleton used to signal either "Any Sender" or "Any Signal" + """Singleton used to signal either "Any Sender" or "Any Signal" + + The Any object can be used with connect, disconnect, + send, or sendExact to signal that the parameter given + Any should react to all senders/signals, not just + a particular sender/signal. + """ + - The Any object can be used with connect, disconnect, - send, or sendExact to signal that the parameter given - Any should react to all senders/signals, not just - a particular sender/signal. - """ Any = _Any() + class _Anonymous(_Parameter): - """Singleton used to signal "Anonymous Sender" + """Singleton used to signal "Anonymous Sender" + + The Anonymous object is used to signal that the sender + of a message is not specified (as distinct from being + "any sender"). Registering callbacks for Anonymous + will only receive messages sent without senders. Sending + with anonymous will only send messages to those receivers + registered for Any or Anonymous. + + Note: + The default sender for connect is Any, while the + default sender for send is Anonymous. This has + the effect that if you do not specify any senders + in either function then all messages are routed + as though there was a single sender (Anonymous) + being used everywhere. + """ - The Anonymous object is used to signal that the sender - of a message is not specified (as distinct from being - "any sender"). Registering callbacks for Anonymous - will only receive messages sent without senders. Sending - with anonymous will only send messages to those receivers - registered for Any or Anonymous. - Note: - The default sender for connect is Any, while the - default sender for send is Anonymous. This has - the effect that if you do not specify any senders - in either function then all messages are routed - as though there was a single sender (Anonymous) - being used everywhere. - """ Anonymous = _Anonymous() WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref) @@ -76,417 +84,428 @@ sendersBack = {} def connect(receiver, signal=Any, sender=Any, weak=True): - """Connect receiver to sender for signal + """Connect receiver to sender for signal - receiver -- a callable Python object which is to receive - messages/signals/events. Receivers must be hashable - objects. + receiver -- a callable Python object which is to receive + messages/signals/events. Receivers must be hashable + objects. - if weak is True, then receiver must be weak-referencable - (more precisely saferef.safeRef() must be able to create - a reference to the receiver). - - Receivers are fairly flexible in their specification, - as the machinery in the robustApply module takes care - of most of the details regarding figuring out appropriate - subsets of the sent arguments to apply to a given - receiver. + if weak is True, then receiver must be weak-referencable + (more precisely saferef.safeRef() must be able to create + a reference to the receiver). - Note: - if receiver is itself a weak reference (a callable), - it will be de-referenced by the system's machinery, - so *generally* weak references are not suitable as - receivers, though some use might be found for the - facility whereby a higher-level library passes in - pre-weakrefed receiver references. + Receivers are fairly flexible in their specification, + as the machinery in the robustApply module takes care + of most of the details regarding figuring out appropriate + subsets of the sent arguments to apply to a given + receiver. - signal -- the signal to which the receiver should respond - - if Any, receiver will receive any signal from the - indicated sender (which might also be Any, but is not - necessarily Any). - - Otherwise must be a hashable Python object other than - None (DispatcherError raised on None). - - sender -- the sender to which the receiver should respond - - if Any, receiver will receive the indicated signals - from any sender. - - if Anonymous, receiver will only receive indicated - signals from send/sendExact which do not specify a - sender, or specify Anonymous explicitly as the sender. + Note: + if receiver is itself a weak reference (a callable), + it will be de-referenced by the system's machinery, + so *generally* weak references are not suitable as + receivers, though some use might be found for the + facility whereby a higher-level library passes in + pre-weakrefed receiver references. - Otherwise can be any python object. - - weak -- whether to use weak references to the receiver - By default, the module will attempt to use weak - references to the receiver objects. If this parameter - is false, then strong references will be used. + signal -- the signal to which the receiver should respond - returns None, may raise DispatcherTypeError - """ - if signal is None: - raise errors.DispatcherTypeError( - 'Signal cannot be None (receiver=%r sender=%r)'%( receiver,sender) - ) - if weak: - receiver = saferef.safeRef(receiver, onDelete=_removeReceiver) - senderkey = id(sender) - if senderkey in connections: - signals = connections[senderkey] - else: - connections[senderkey] = signals = {} - # Keep track of senders for cleanup. - # Is Anonymous something we want to clean up? - if sender not in (None, Anonymous, Any): - def remove(object, senderkey=senderkey): - _removeSender(senderkey=senderkey) - # Skip objects that can not be weakly referenced, which means - # they won't be automatically cleaned up, but that's too bad. - try: - weakSender = weakref.ref(sender, remove) - senders[senderkey] = weakSender - except: - pass - - receiverID = id(receiver) - # get current set, remove any current references to - # this receiver in the set, including back-references - if signal in signals: - receivers = signals[signal] - _removeOldBackRefs(senderkey, signal, receiver, receivers) - else: - receivers = signals[signal] = [] - try: - current = sendersBack.get( receiverID ) - if current is None: - sendersBack[ receiverID ] = current = [] - if senderkey not in current: - current.append(senderkey) - except: - pass + if Any, receiver will receive any signal from the + indicated sender (which might also be Any, but is not + necessarily Any). - receivers.append(receiver) + Otherwise must be a hashable Python object other than + None (DispatcherError raised on None). + sender -- the sender to which the receiver should respond + + if Any, receiver will receive the indicated signals + from any sender. + + if Anonymous, receiver will only receive indicated + signals from send/sendExact which do not specify a + sender, or specify Anonymous explicitly as the sender. + + Otherwise can be any python object. + + weak -- whether to use weak references to the receiver + By default, the module will attempt to use weak + references to the receiver objects. If this parameter + is false, then strong references will be used. + + returns None, may raise DispatcherTypeError + """ + if signal is None: + raise errors.DispatcherTypeError( + 'Signal cannot be None (receiver=%r sender=%r)' % ( + receiver, sender) + ) + if weak: + receiver = saferef.safeRef(receiver, onDelete=_removeReceiver) + senderkey = id(sender) + if senderkey in connections: + signals = connections[senderkey] + else: + connections[senderkey] = signals = {} + # Keep track of senders for cleanup. + # Is Anonymous something we want to clean up? + if sender not in (None, Anonymous, Any): + def remove(object, senderkey=senderkey): + _removeSender(senderkey=senderkey) + + # Skip objects that can not be weakly referenced, which means + # they won't be automatically cleaned up, but that's too bad. + try: + weakSender = weakref.ref(sender, remove) + senders[senderkey] = weakSender + except: + pass + + receiverID = id(receiver) + # get current set, remove any current references to + # this receiver in the set, including back-references + if signal in signals: + receivers = signals[signal] + _removeOldBackRefs(senderkey, signal, receiver, receivers) + else: + receivers = signals[signal] = [] + try: + current = sendersBack.get(receiverID) + if current is None: + sendersBack[receiverID] = current = [] + if senderkey not in current: + current.append(senderkey) + except: + pass + + receivers.append(receiver) def disconnect(receiver, signal=Any, sender=Any, weak=True): - """Disconnect receiver from sender for signal + """Disconnect receiver from sender for signal - receiver -- the registered receiver to disconnect - signal -- the registered signal to disconnect - sender -- the registered sender to disconnect - weak -- the weakref state to disconnect + receiver -- the registered receiver to disconnect + signal -- the registered signal to disconnect + sender -- the registered sender to disconnect + weak -- the weakref state to disconnect - disconnect reverses the process of connect, - the semantics for the individual elements are - logically equivalent to a tuple of - (receiver, signal, sender, weak) used as a key - to be deleted from the internal routing tables. - (The actual process is slightly more complex - but the semantics are basically the same). + disconnect reverses the process of connect, + the semantics for the individual elements are + logically equivalent to a tuple of + (receiver, signal, sender, weak) used as a key + to be deleted from the internal routing tables. + (The actual process is slightly more complex + but the semantics are basically the same). - Note: - Using disconnect is not required to cleanup - routing when an object is deleted, the framework - will remove routes for deleted objects - automatically. It's only necessary to disconnect - if you want to stop routing to a live object. - - returns None, may raise DispatcherTypeError or - DispatcherKeyError - """ - if signal is None: - raise errors.DispatcherTypeError( - 'Signal cannot be None (receiver=%r sender=%r)'%( receiver,sender) - ) - if weak: receiver = saferef.safeRef(receiver) - senderkey = id(sender) - try: - signals = connections[senderkey] - receivers = signals[signal] - except KeyError: - raise errors.DispatcherKeyError( - """No receivers found for signal %r from sender %r""" %( - signal, - sender - ) - ) - try: - # also removes from receivers - _removeOldBackRefs(senderkey, signal, receiver, receivers) - except ValueError: - raise errors.DispatcherKeyError( - """No connection to receiver %s for signal %s from sender %s""" %( - receiver, - signal, - sender - ) - ) - _cleanupConnections(senderkey, signal) + Note: + Using disconnect is not required to cleanup + routing when an object is deleted, the framework + will remove routes for deleted objects + automatically. It's only necessary to disconnect + if you want to stop routing to a live object. -def getReceivers( sender = Any, signal = Any ): - """Get list of receivers from global tables + returns None, may raise DispatcherTypeError or + DispatcherKeyError + """ + if signal is None: + raise errors.DispatcherTypeError( + 'Signal cannot be None (receiver=%r sender=%r)' % ( + receiver, sender) + ) + if weak: receiver = saferef.safeRef(receiver) + senderkey = id(sender) + try: + signals = connections[senderkey] + receivers = signals[signal] + except KeyError: + raise errors.DispatcherKeyError( + """No receivers found for signal %r from sender %r""" % ( + signal, + sender + ) + ) + try: + # also removes from receivers + _removeOldBackRefs(senderkey, signal, receiver, receivers) + except ValueError: + raise errors.DispatcherKeyError( + """No connection to receiver %s for signal %s from sender %s""" % ( + receiver, + signal, + sender + ) + ) + _cleanupConnections(senderkey, signal) - This utility function allows you to retrieve the - raw list of receivers from the connections table - for the given sender and signal pair. - Note: - there is no guarantee that this is the actual list - stored in the connections table, so the value - should be treated as a simple iterable/truth value - rather than, for instance a list to which you - might append new records. +def getReceivers(sender=Any, signal=Any): + """Get list of receivers from global tables + + This utility function allows you to retrieve the + raw list of receivers from the connections table + for the given sender and signal pair. + + Note: + there is no guarantee that this is the actual list + stored in the connections table, so the value + should be treated as a simple iterable/truth value + rather than, for instance a list to which you + might append new records. + + Normally you would use liveReceivers( getReceivers( ...)) + to retrieve the actual receiver objects as an iterable + object. + """ + try: + return connections[id(sender)][signal] + except KeyError: + return [] - Normally you would use liveReceivers( getReceivers( ...)) - to retrieve the actual receiver objects as an iterable - object. - """ - try: - return connections[id(sender)][signal] - except KeyError: - return [] def liveReceivers(receivers): - """Filter sequence of receivers to get resolved, live receivers + """Filter sequence of receivers to get resolved, live receivers - This is a generator which will iterate over - the passed sequence, checking for weak references - and resolving them, then returning all live - receivers. - """ - for receiver in receivers: - if isinstance( receiver, WEAKREF_TYPES): - # Dereference the weak reference. - receiver = receiver() - if receiver is not None: - yield receiver - else: - yield receiver + This is a generator which will iterate over + the passed sequence, checking for weak references + and resolving them, then returning all live + receivers. + """ + for receiver in receivers: + if isinstance(receiver, WEAKREF_TYPES): + # Dereference the weak reference. + receiver = receiver() + if receiver is not None: + yield receiver + else: + yield receiver +def getAllReceivers(sender=Any, signal=Any): + """Get list of all receivers from global tables -def getAllReceivers( sender = Any, signal = Any ): - """Get list of all receivers from global tables + This gets all receivers which should receive + the given signal from sender, each receiver should + be produced only once by the resulting generator + """ + receivers = {} + for set in ( + # Get receivers that receive *this* signal from *this* sender. + getReceivers(sender, signal), + # Add receivers that receive *any* signal from *this* sender. + getReceivers(sender, Any), + # Add receivers that receive *this* signal from *any* sender. + getReceivers(Any, signal), + # Add receivers that receive *any* signal from *any* sender. + getReceivers(Any, Any), + ): + for receiver in set: + if receiver: # filter out dead instance-method weakrefs + try: + if receiver not in receivers: + receivers[receiver] = 1 + yield receiver + except TypeError: + # dead weakrefs raise TypeError on hash... + pass - This gets all receivers which should receive - the given signal from sender, each receiver should - be produced only once by the resulting generator - """ - receivers = {} - for set in ( - # Get receivers that receive *this* signal from *this* sender. - getReceivers( sender, signal ), - # Add receivers that receive *any* signal from *this* sender. - getReceivers( sender, Any ), - # Add receivers that receive *this* signal from *any* sender. - getReceivers( Any, signal ), - # Add receivers that receive *any* signal from *any* sender. - getReceivers( Any, Any ), - ): - for receiver in set: - if receiver: # filter out dead instance-method weakrefs - try: - if receiver not in receivers: - receivers[receiver] = 1 - yield receiver - except TypeError: - # dead weakrefs raise TypeError on hash... - pass def send(signal=Any, sender=Anonymous, *arguments, **named): - """Send signal from sender to all connected receivers. - - signal -- (hashable) signal value, see connect for details + """Send signal from sender to all connected receivers. - sender -- the sender of the signal - - if Any, only receivers registered for Any will receive - the message. + signal -- (hashable) signal value, see connect for details - if Anonymous, only receivers registered to receive - messages from Anonymous or Any will receive the message + sender -- the sender of the signal - Otherwise can be any python object (normally one - registered with a connect if you actually want - something to occur). + if Any, only receivers registered for Any will receive + the message. - arguments -- positional arguments which will be passed to - *all* receivers. Note that this may raise TypeErrors - if the receivers do not allow the particular arguments. - Note also that arguments are applied before named - arguments, so they should be used with care. + if Anonymous, only receivers registered to receive + messages from Anonymous or Any will receive the message - named -- named arguments which will be filtered according - to the parameters of the receivers to only provide those - acceptable to the receiver. + Otherwise can be any python object (normally one + registered with a connect if you actually want + something to occur). - Return a list of tuple pairs [(receiver, response), ... ] + arguments -- positional arguments which will be passed to + *all* receivers. Note that this may raise TypeErrors + if the receivers do not allow the particular arguments. + Note also that arguments are applied before named + arguments, so they should be used with care. - if any receiver raises an error, the error propagates back - through send, terminating the dispatch loop, so it is quite - possible to not have all receivers called if a raises an - error. - """ - # Call each receiver with whatever arguments it can accept. - # Return a list of tuple pairs [(receiver, response), ... ]. - responses = [] - for receiver in liveReceivers(getAllReceivers(sender, signal)): - response = robustapply.robustApply( - receiver, - signal=signal, - sender=sender, - *arguments, - **named - ) - responses.append((receiver, response)) - return responses -def sendExact( signal=Any, sender=Anonymous, *arguments, **named ): - """Send signal only to those receivers registered for exact message + named -- named arguments which will be filtered according + to the parameters of the receivers to only provide those + acceptable to the receiver. + + Return a list of tuple pairs [(receiver, response), ... ] + + if any receiver raises an error, the error propagates back + through send, terminating the dispatch loop, so it is quite + possible to not have all receivers called if a raises an + error. + """ + # Call each receiver with whatever arguments it can accept. + # Return a list of tuple pairs [(receiver, response), ... ]. + responses = [] + for receiver in liveReceivers(getAllReceivers(sender, signal)): + response = robustapply.robustApply( + receiver, + signal=signal, + sender=sender, + *arguments, + **named + ) + responses.append((receiver, response)) + return responses + + +def sendExact(signal=Any, sender=Anonymous, *arguments, **named): + """Send signal only to those receivers registered for exact message + + sendExact allows for avoiding Any/Anonymous registered + handlers, sending only to those receivers explicitly + registered for a particular signal on a particular + sender. + """ + responses = [] + for receiver in liveReceivers(getReceivers(sender, signal)): + response = robustapply.robustApply( + receiver, + signal=signal, + sender=sender, + *arguments, + **named + ) + responses.append((receiver, response)) + return responses - sendExact allows for avoiding Any/Anonymous registered - handlers, sending only to those receivers explicitly - registered for a particular signal on a particular - sender. - """ - responses = [] - for receiver in liveReceivers(getReceivers(sender, signal)): - response = robustapply.robustApply( - receiver, - signal=signal, - sender=sender, - *arguments, - **named - ) - responses.append((receiver, response)) - return responses - def _removeReceiver(receiver): - """Remove receiver from connections.""" - if not sendersBack: - # During module cleanup the mapping will be replaced with None - return False - backKey = id(receiver) - try: - backSet = sendersBack.pop(backKey) - except KeyError as err: - return False - else: - for senderkey in backSet: - try: - signals = connections[senderkey].keys() - except KeyError as err: - pass - else: - for signal in signals: - try: - receivers = connections[senderkey][signal] - except KeyError: - pass - else: - try: - receivers.remove( receiver ) - except Exception as err: - pass - _cleanupConnections(senderkey, signal) + """Remove receiver from connections.""" + if not sendersBack: + # During module cleanup the mapping will be replaced with None + return False + backKey = id(receiver) + try: + backSet = sendersBack.pop(backKey) + except KeyError as err: + return False + else: + for senderkey in backSet: + try: + signals = connections[senderkey].keys() + except KeyError as err: + pass + else: + for signal in signals: + try: + receivers = connections[senderkey][signal] + except KeyError: + pass + else: + try: + receivers.remove(receiver) + except Exception as err: + pass + _cleanupConnections(senderkey, signal) + def _cleanupConnections(senderkey, signal): - """Delete any empty signals for senderkey. Delete senderkey if empty.""" - try: - receivers = connections[senderkey][signal] - except: - pass - else: - if not receivers: - # No more connected receivers. Therefore, remove the signal. - try: - signals = connections[senderkey] - except KeyError: - pass - else: - del signals[signal] - if not signals: - # No more signal connections. Therefore, remove the sender. - _removeSender(senderkey) + """Delete any empty signals for senderkey. Delete senderkey if empty.""" + try: + receivers = connections[senderkey][signal] + except: + pass + else: + if not receivers: + # No more connected receivers. Therefore, remove the signal. + try: + signals = connections[senderkey] + except KeyError: + pass + else: + del signals[signal] + if not signals: + # No more signal connections. Therefore, remove the sender. + _removeSender(senderkey) + def _removeSender(senderkey): - """Remove senderkey from connections.""" - _removeBackrefs(senderkey) - try: - del connections[senderkey] - except KeyError: - pass - # Senderkey will only be in senders dictionary if sender - # could be weakly referenced. - try: - del senders[senderkey] - except: - pass + """Remove senderkey from connections.""" + _removeBackrefs(senderkey) + try: + del connections[senderkey] + except KeyError: + pass + # Senderkey will only be in senders dictionary if sender + # could be weakly referenced. + try: + del senders[senderkey] + except: + pass -def _removeBackrefs( senderkey): - """Remove all back-references to this senderkey""" - try: - signals = connections[senderkey] - except KeyError: - signals = None - else: - items = signals.items() - def allReceivers( ): - for signal,set in items: - for item in set: - yield item - for receiver in allReceivers(): - _killBackref( receiver, senderkey ) +def _removeBackrefs(senderkey): + """Remove all back-references to this senderkey""" + try: + signals = connections[senderkey] + except KeyError: + signals = None + else: + items = signals.items() + + def allReceivers(): + for signal, set in items: + for item in set: + yield item + + for receiver in allReceivers(): + _killBackref(receiver, senderkey) + def _removeOldBackRefs(senderkey, signal, receiver, receivers): - """Kill old sendersBack references from receiver + """Kill old sendersBack references from receiver - This guards against multiple registration of the same - receiver for a given signal and sender leaking memory - as old back reference records build up. + This guards against multiple registration of the same + receiver for a given signal and sender leaking memory + as old back reference records build up. - Also removes old receiver instance from receivers - """ - try: - index = receivers.index(receiver) - # need to scan back references here and remove senderkey - except ValueError: - return False - else: - oldReceiver = receivers[index] - del receivers[index] - found = 0 - signals = connections.get(signal) - if signals is not None: - for sig, recs in six.iteritems(connections.get(signal,{})): - if sig != signal: - for rec in recs: - if rec is oldReceiver: - found = 1 - break - if not found: - _killBackref( oldReceiver, senderkey ) - return True - return False - - -def _killBackref( receiver, senderkey ): - """Do the actual removal of back reference from receiver to senderkey""" - receiverkey = id(receiver) - set = sendersBack.get( receiverkey, () ) - while senderkey in set: - try: - set.remove( senderkey ) - except: - break - if not set: - try: - del sendersBack[ receiverkey ] - except KeyError: - pass - return True + Also removes old receiver instance from receivers + """ + try: + index = receivers.index(receiver) + # need to scan back references here and remove senderkey + except ValueError: + return False + else: + oldReceiver = receivers[index] + del receivers[index] + found = 0 + signals = connections.get(signal) + if signals is not None: + for sig, recs in six.iteritems(connections.get(signal, {})): + if sig != signal: + for rec in recs: + if rec is oldReceiver: + found = 1 + break + if not found: + _killBackref(oldReceiver, senderkey) + return True + return False + + +def _killBackref(receiver, senderkey): + """Do the actual removal of back reference from receiver to senderkey""" + receiverkey = id(receiver) + set = sendersBack.get(receiverkey, ()) + while senderkey in set: + try: + set.remove(senderkey) + except: + break + if not set: + try: + del sendersBack[receiverkey] + except KeyError: + pass + return True diff --git a/scrapy/xlib/pydispatch/errors.py b/scrapy/xlib/pydispatch/errors.py index a2eb32ed7..c5540d8f7 100644 --- a/scrapy/xlib/pydispatch/errors.py +++ b/scrapy/xlib/pydispatch/errors.py @@ -1,10 +1,15 @@ """Error types for dispatcher mechanism """ -class DispatcherError(Exception): - """Base class for all Dispatcher errors""" -class DispatcherKeyError(KeyError, DispatcherError): - """Error raised when unknown (sender,signal) set specified""" -class DispatcherTypeError(TypeError, DispatcherError): - """Error raised when inappropriate signal-type specified (None)""" + +class DispatcherError(Exception): + """Base class for all Dispatcher errors""" + + +class DispatcherKeyError(KeyError, DispatcherError): + """Error raised when unknown (sender,signal) set specified""" + + +class DispatcherTypeError(TypeError, DispatcherError): + """Error raised when inappropriate signal-type specified (None)""" diff --git a/scrapy/xlib/pydispatch/robust.py b/scrapy/xlib/pydispatch/robust.py index f829dbfd9..a28f127e2 100644 --- a/scrapy/xlib/pydispatch/robust.py +++ b/scrapy/xlib/pydispatch/robust.py @@ -2,56 +2,53 @@ from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers from scrapy.xlib.pydispatch.robustapply import robustApply -def sendRobust( - signal=Any, - sender=Anonymous, - *arguments, **named -): - """Send signal from sender to all connected receivers catching errors - - signal -- (hashable) signal value, see connect for details - sender -- the sender of the signal - - if Any, only receivers registered for Any will receive - the message. +def sendRobust(signal=Any, sender=Anonymous, *arguments, **named): + """Send signal from sender to all connected receivers catching errors - if Anonymous, only receivers registered to receive - messages from Anonymous or Any will receive the message + signal -- (hashable) signal value, see connect for details - Otherwise can be any python object (normally one - registered with a connect if you actually want - something to occur). + sender -- the sender of the signal - arguments -- positional arguments which will be passed to - *all* receivers. Note that this may raise TypeErrors - if the receivers do not allow the particular arguments. - Note also that arguments are applied before named - arguments, so they should be used with care. + if Any, only receivers registered for Any will receive + the message. - named -- named arguments which will be filtered according - to the parameters of the receivers to only provide those - acceptable to the receiver. + if Anonymous, only receivers registered to receive + messages from Anonymous or Any will receive the message - Return a list of tuple pairs [(receiver, response), ... ] + Otherwise can be any python object (normally one + registered with a connect if you actually want + something to occur). - if any receiver raises an error (specifically any subclass of Exception), - the error instance is returned as the result for that receiver. - """ - # Call each receiver with whatever arguments it can accept. - # Return a list of tuple pairs [(receiver, response), ... ]. - responses = [] - for receiver in liveReceivers(getAllReceivers(sender, signal)): - try: - response = robustApply( - receiver, - signal=signal, - sender=sender, - *arguments, - **named - ) - except Exception as err: - responses.append((receiver, err)) - else: - responses.append((receiver, response)) - return responses + arguments -- positional arguments which will be passed to + *all* receivers. Note that this may raise TypeErrors + if the receivers do not allow the particular arguments. + Note also that arguments are applied before named + arguments, so they should be used with care. + + named -- named arguments which will be filtered according + to the parameters of the receivers to only provide those + acceptable to the receiver. + + Return a list of tuple pairs [(receiver, response), ... ] + + if any receiver raises an error (specifically any subclass of Exception), + the error instance is returned as the result for that receiver. + """ + # Call each receiver with whatever arguments it can accept. + # Return a list of tuple pairs [(receiver, response), ... ]. + responses = [] + for receiver in liveReceivers(getAllReceivers(sender, signal)): + try: + response = robustApply( + receiver, + signal=signal, + sender=sender, + *arguments, + **named + ) + except Exception as err: + responses.append((receiver, err)) + else: + responses.append((receiver, response)) + return responses diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py index 5deda3c42..9977ac6b7 100644 --- a/scrapy/xlib/pydispatch/robustapply.py +++ b/scrapy/xlib/pydispatch/robustapply.py @@ -5,9 +5,9 @@ what arguments a given callable object can take, and subset the given arguments to match only those which are acceptable. """ - import inspect + def function(receiver): """Get function-like callable object for given receiver @@ -20,32 +20,35 @@ def function(receiver): # receiver is a class instance; assume it is callable. # Reassign receiver to the actual method that will be called. if hasattr(receiver.__call__, 'im_func') or \ - hasattr(receiver.__call__, 'im_code'): + hasattr(receiver.__call__, 'im_code'): receiver = receiver.__call__ - if hasattr( receiver, 'im_func' ): + if hasattr(receiver, 'im_func'): # an instance-method... return receiver, receiver.im_func.func_code, 1 elif not hasattr(receiver, 'func_code'): - raise ValueError('unknown receiver type %s %s'%(receiver, type(receiver))) + raise ValueError( + 'unknown receiver type %s %s' % (receiver, type(receiver))) return receiver, receiver.func_code, 0 + def robustApply(receiver, *arguments, **named): """Call receiver with arguments and an appropriate subset of named """ receiver, codeObject, startIndex = function(receiver) - acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount] - for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]: + acceptable = codeObject.co_varnames[ + startIndex + len(arguments):codeObject.co_argcount] + for name in codeObject.co_varnames[startIndex:startIndex + len(arguments)]: if name in named: raise TypeError( - """Argument %r specified both positionally and as a keyword for calling %r"""% ( + """Argument %r specified both positionally and as a keyword for calling %r""" % ( name, receiver, ) ) if not (codeObject.co_flags & 8): - # fc does not have a **kwds type parameter, therefore + # fc does not have a **kwds type parameter, therefore # remove unacceptable arguments. for arg in named.keys(): if arg not in acceptable: diff --git a/scrapy/xlib/pydispatch/saferef.py b/scrapy/xlib/pydispatch/saferef.py index f1b8b1f9b..bd9659673 100644 --- a/scrapy/xlib/pydispatch/saferef.py +++ b/scrapy/xlib/pydispatch/saferef.py @@ -2,165 +2,179 @@ from __future__ import print_function import weakref, traceback -def safeRef(target, onDelete = None): - """Return a *safe* weak reference to a callable target - target -- the object to be weakly referenced, if it's a - bound method reference, will create a BoundMethodWeakref, - otherwise creates a simple weakref. - onDelete -- if provided, will have a hard reference stored - to the callable to be called after the safe reference - goes out of scope with the reference object, (either a - weakref or a BoundMethodWeakref) as argument. - """ - if hasattr(target, 'im_self'): - if target.im_self is not None: - # Turn a bound method into a BoundMethodWeakref instance. - # Keep track of these instances for lookup by disconnect(). - assert hasattr(target, 'im_func'), """safeRef target %r has im_self, but no im_func, don't know how to create reference"""%( target,) - reference = BoundMethodWeakref( - target=target, - onDelete=onDelete - ) - return reference - if onDelete is not None: - return weakref.ref(target, onDelete) - else: - return weakref.ref( target ) +def safeRef(target, onDelete=None): + """Return a *safe* weak reference to a callable target + + target -- the object to be weakly referenced, if it's a + bound method reference, will create a BoundMethodWeakref, + otherwise creates a simple weakref. + onDelete -- if provided, will have a hard reference stored + to the callable to be called after the safe reference + goes out of scope with the reference object, (either a + weakref or a BoundMethodWeakref) as argument. + """ + if hasattr(target, 'im_self'): + if target.im_self is not None: + # Turn a bound method into a BoundMethodWeakref instance. + # Keep track of these instances for lookup by disconnect(). + assert hasattr(target, 'im_func'), """safeRef target %r has im_self, but no im_func, don't know how to create reference"""%( target,) + reference = BoundMethodWeakref( + target=target, + onDelete=onDelete + ) + return reference + if onDelete is not None: + return weakref.ref(target, onDelete) + else: + return weakref.ref(target) + class BoundMethodWeakref(object): - """'Safe' and reusable weak references to instance methods + """'Safe' and reusable weak references to instance methods - BoundMethodWeakref objects provide a mechanism for - referencing a bound method without requiring that the - method object itself (which is normally a transient - object) is kept alive. Instead, the BoundMethodWeakref - object keeps weak references to both the object and the - function which together define the instance method. + BoundMethodWeakref objects provide a mechanism for + referencing a bound method without requiring that the + method object itself (which is normally a transient + object) is kept alive. Instead, the BoundMethodWeakref + object keeps weak references to both the object and the + function which together define the instance method. - Attributes: - key -- the identity key for the reference, calculated - by the class's calculateKey method applied to the - target instance method - deletionMethods -- sequence of callable objects taking - single argument, a reference to this object which - will be called when *either* the target object or - target function is garbage collected (i.e. when - this object becomes invalid). These are specified - as the onDelete parameters of safeRef calls. - weakSelf -- weak reference to the target object - weakFunc -- weak reference to the target function + Attributes: + key -- the identity key for the reference, calculated + by the class's calculateKey method applied to the + target instance method + deletionMethods -- sequence of callable objects taking + single argument, a reference to this object which + will be called when *either* the target object or + target function is garbage collected (i.e. when + this object becomes invalid). These are specified + as the onDelete parameters of safeRef calls. + weakSelf -- weak reference to the target object + weakFunc -- weak reference to the target function - Class Attributes: - _allInstances -- class attribute pointing to all live - BoundMethodWeakref objects indexed by the class's - calculateKey(target) method applied to the target - objects. This weak value dictionary is used to - short-circuit creation so that multiple references - to the same (object, function) pair produce the - same BoundMethodWeakref instance. + Class Attributes: + _allInstances -- class attribute pointing to all live + BoundMethodWeakref objects indexed by the class's + calculateKey(target) method applied to the target + objects. This weak value dictionary is used to + short-circuit creation so that multiple references + to the same (object, function) pair produce the + same BoundMethodWeakref instance. - """ - _allInstances = weakref.WeakValueDictionary() - def __new__( cls, target, onDelete=None, *arguments,**named ): - """Create new instance or return current instance + """ + _allInstances = weakref.WeakValueDictionary() - Basically this method of construction allows us to - short-circuit creation of references to already- - referenced instance methods. The key corresponding - to the target is calculated, and if there is already - an existing reference, that is returned, with its - deletionMethods attribute updated. Otherwise the - new instance is created and registered in the table - of already-referenced methods. - """ - key = cls.calculateKey(target) - current =cls._allInstances.get(key) - if current is not None: - current.deletionMethods.append( onDelete) - return current - else: - base = super( BoundMethodWeakref, cls).__new__( cls ) - cls._allInstances[key] = base - base.__init__( target, onDelete, *arguments,**named) - return base - def __init__(self, target, onDelete=None): - """Return a weak-reference-like instance for a bound method + def __new__(cls, target, onDelete=None, *arguments, **named): + """Create new instance or return current instance - target -- the instance-method target for the weak - reference, must have im_self and im_func attributes - and be reconstructable via: - target.im_func.__get__( target.im_self ) - which is true of built-in instance methods. - onDelete -- optional callback which will be called - when this weak reference ceases to be valid - (i.e. either the object or the function is garbage - collected). Should take a single argument, - which will be passed a pointer to this object. - """ - def remove(weak, self=self): - """Set self.isDead to true when method or instance is destroyed""" - methods = self.deletionMethods[:] - del self.deletionMethods[:] - try: - del self.__class__._allInstances[ self.key ] - except KeyError: - pass - for function in methods: - try: - if callable( function ): - function( self ) - except Exception as e: - try: - traceback.print_exc() - except AttributeError as err: - print('''Exception during saferef %s cleanup function %s: %s'''%( - self, function, e - )) - self.deletionMethods = [onDelete] - self.key = self.calculateKey( target ) - self.weakSelf = weakref.ref(target.im_self, remove) - self.weakFunc = weakref.ref(target.im_func, remove) - self.selfName = target.im_self.__class__.__name__ - self.funcName = str(target.im_func.__name__) - def calculateKey( cls, target ): - """Calculate the reference key for this reference + Basically this method of construction allows us to + short-circuit creation of references to already- + referenced instance methods. The key corresponding + to the target is calculated, and if there is already + an existing reference, that is returned, with its + deletionMethods attribute updated. Otherwise the + new instance is created and registered in the table + of already-referenced methods. + """ + key = cls.calculateKey(target) + current = cls._allInstances.get(key) + if current is not None: + current.deletionMethods.append(onDelete) + return current + else: + base = super(BoundMethodWeakref, cls).__new__(cls) + cls._allInstances[key] = base + base.__init__(target, onDelete, *arguments, **named) + return base - Currently this is a two-tuple of the id()'s of the - target object and the target function respectively. - """ - return (id(target.im_self),id(target.im_func)) - calculateKey = classmethod( calculateKey ) - def __str__(self): - """Give a friendly representation of the object""" - return """%s( %s.%s )"""%( - self.__class__.__name__, - self.selfName, - self.funcName, - ) - __repr__ = __str__ - def __nonzero__( self ): - """Whether we are still a valid reference""" - return self() is not None - def __cmp__( self, other ): - """Compare with another reference""" - if not isinstance (other,self.__class__): - return cmp( self.__class__, type(other) ) - return cmp( self.key, other.key) - def __call__(self): - """Return a strong reference to the bound method + def __init__(self, target, onDelete=None): + """Return a weak-reference-like instance for a bound method - If the target cannot be retrieved, then will - return None, otherwise returns a bound instance - method for our object and function. + target -- the instance-method target for the weak + reference, must have im_self and im_func attributes + and be reconstructable via: + target.im_func.__get__( target.im_self ) + which is true of built-in instance methods. + onDelete -- optional callback which will be called + when this weak reference ceases to be valid + (i.e. either the object or the function is garbage + collected). Should take a single argument, + which will be passed a pointer to this object. + """ - Note: - You may call this method any number of times, - as it does not invalidate the reference. - """ - target = self.weakSelf() - if target is not None: - function = self.weakFunc() - if function is not None: - return function.__get__(target) - return None + def remove(weak, self=self): + """Set self.isDead to true when method or instance is destroyed""" + methods = self.deletionMethods[:] + del self.deletionMethods[:] + try: + del self.__class__._allInstances[self.key] + except KeyError: + pass + for function in methods: + try: + if callable(function): + function(self) + except Exception as e: + try: + traceback.print_exc() + except AttributeError as err: + print( + '''Exception during saferef %s cleanup function %s: %s''' % ( + self, function, e + )) + + self.deletionMethods = [onDelete] + self.key = self.calculateKey(target) + self.weakSelf = weakref.ref(target.im_self, remove) + self.weakFunc = weakref.ref(target.im_func, remove) + self.selfName = target.im_self.__class__.__name__ + self.funcName = str(target.im_func.__name__) + + def calculateKey(cls, target): + """Calculate the reference key for this reference + + Currently this is a two-tuple of the id()'s of the + target object and the target function respectively. + """ + return (id(target.im_self), id(target.im_func)) + + calculateKey = classmethod(calculateKey) + + def __str__(self): + """Give a friendly representation of the object""" + return """%s( %s.%s )""" % ( + self.__class__.__name__, + self.selfName, + self.funcName, + ) + + __repr__ = __str__ + + def __nonzero__(self): + """Whether we are still a valid reference""" + return self() is not None + + def __cmp__(self, other): + """Compare with another reference""" + if not isinstance(other, self.__class__): + return cmp(self.__class__, type(other)) + return cmp(self.key, other.key) + + def __call__(self): + """Return a strong reference to the bound method + + If the target cannot be retrieved, then will + return None, otherwise returns a bound instance + method for our object and function. + + Note: + You may call this method any number of times, + as it does not invalidate the reference. + """ + target = self.weakSelf() + if target is not None: + function = self.weakFunc() + if function is not None: + return function.__get__(target) + return None