/// <summary> /// Returns true if this Broker has been marked as failed and it is now time to /// start a recovery attempt. /// </summary> public bool IsTimeForRecovery(DiscoveredServiceData service) { lock (service.SyncRoot) { if (!service.Failed) { return(false); } int maxReconnectAttempts = MaxReconnectAttempts; // Are we done trying to recover this guy? if (maxReconnectAttempts > 0 && service.FailureCount > maxReconnectAttempts) { Tracer.DebugFormat("Max reconnect attempts of the {0} service has been reached.", service.ServiceName); return(false); } // Is it not yet time? if (DateTime.Now < service.RecoveryTime) { return(false); } Tracer.DebugFormat("Resuming event advertisement of the {0} service.", service.ServiceName); service.Failed = false; return(true); } }
internal bool MarkFailed(DiscoveredServiceData service) { lock (service.SyncRoot) { if (!service.Failed) { service.Failed = true; service.FailureCount++; long reconnectDelay = 0; if (!UseExponentialBackOff) { reconnectDelay = InitialReconnectDelay; } else { reconnectDelay = (long)Math.Pow(BackOffMultiplier, service.FailureCount); reconnectDelay = Math.Min(reconnectDelay, MaxReconnectDelay); } Tracer.DebugFormat("Remote failure of {0} while still receiving multicast advertisements. " + "Advertising events will be suppressed for {1} ms, the current " + "failure count is: {2}", service.ServiceName, reconnectDelay, service.FailureCount); service.RecoveryTime = DateTime.Now + TimeSpan.FromMilliseconds(reconnectDelay); return(true); } } return(false); }
protected void ProcessLiveService(string brokerName, string service) { if (SelfService == null || !service.Equals(SelfService)) { DiscoveredServiceData remoteBroker = null; lock (discoveredServicesLock) { discoveredServices.TryGetValue(service, out remoteBroker); } if (remoteBroker == null) { remoteBroker = new DiscoveredServiceData(brokerName, service); discoveredServices.Add(service, remoteBroker); FireServiceAddEvent(remoteBroker); DoAdvertizeSelf(); } else { UpdateHeartBeat(remoteBroker); if (IsTimeForRecovery(remoteBroker)) { FireServiceAddEvent(remoteBroker); } } } }
public void ServiceFailed(DiscoveryEvent failedEvent) { DiscoveredServiceData data = null; discoveredServices.TryGetValue(failedEvent.ServiceName, out data); if (data != null && MarkFailed(data)) { FireServiceRemoveEvent(data); } }
internal void UpdateHeartBeat(DiscoveredServiceData service) { lock (service.SyncRoot) { service.LastHeartBeat = DateTime.Now; // Consider that the broker recovery has succeeded if it has not failed in 60 seconds. if (!service.Failed && service.FailureCount > 0 && (service.LastHeartBeat - service.RecoveryTime) > TimeSpan.FromMilliseconds(1000 * 60)) { Tracer.DebugFormat("I now think that the {0} service has recovered.", service.ServiceName); service.FailureCount = 0; service.RecoveryTime = DateTime.MinValue; } } }
protected void ProcessDeadService(string service) { if (!service.Equals(SelfService)) { DiscoveredServiceData remoteBroker = null; lock (discoveredServicesLock) { discoveredServices.TryGetValue(service, out remoteBroker); if (remoteBroker != null) { discoveredServices.Remove(service); } } if (remoteBroker != null && !remoteBroker.Failed) { FireServiceRemoveEvent(remoteBroker); } } }
private void DoExpireOldServices() { DateTime expireTime = DateTime.Now - TimeSpan.FromMilliseconds(KeepAliveInterval * HEARTBEAT_MISS_BEFORE_DEATH); DiscoveredServiceData[] services = null; lock (discoveredServicesLock) { services = new DiscoveredServiceData[this.discoveredServices.Count]; this.discoveredServices.Values.CopyTo(services, 0); } foreach (DiscoveredServiceData service in services) { if (service.LastHeartBeat < expireTime) { ProcessDeadService(service.ServiceName); } } }