private void ChangeState(CircuitBreakerState state)
{
this.State = state;
this.OnCircuitBreakerStateChanged(new EventArgs()
{
});
}
public CircuitBreaker(int threshold, int openCircuitTimeoutInMilliseconds, IList <Type> ignoredExceptionTypes)
{
if (threshold <= 0)
{
throw new ArgumentOutOfRangeException("threshold", threshold, "The threshold must be greater than 0.");
}
if (openCircuitTimeoutInMilliseconds < 1)
{
throw new ArgumentOutOfRangeException("openCircuitTimeoutInMilliseconds", openCircuitTimeoutInMilliseconds, "The timeout of an open circuit should be greater than 1 millisecond.");
}
if (ignoredExceptionTypes == null)
{
throw new ArgumentNullException("ignoredExceptionTypes", "The list of ignored exception type should not be null.");
}
this.disposed = false;
this.threshold = threshold;
this.failureCount = 0;
this.state = CircuitBreakerState.Closed;
this.ignoredExceptionTypes = ignoredExceptionTypes;
this.openCircuitTimer = new Timer(openCircuitTimeoutInMilliseconds);
this.openCircuitTimer.Elapsed += this.OpenCircuitTimerElapsed;
}
public void ChangeState(CircuitBreakerState state)
{
State = state;
OnCircuitBreakerStateChanged(new EventArgs()
{
});
}
public async Task Trip(Exception exception)
{
if (ShouldTripCircuitBreaker(exception) == false)
{
return;
}
var timeStamp = rebusTime.Now;
_errorDates.TryAdd(timeStamp.Ticks, timeStamp);
var errorsInPeriod = _errorDates
.Where(x => x.Key > timeStamp.Ticks - settings.TrackingPeriod.Ticks)
.Take(settings.Attempts)
.ToList();
var numberOfErrorsInPeriod = errorsInPeriod.Count;
if (IsInRecoveringState(numberOfErrorsInPeriod))
{
State = CircuitBreakerState.Open;
return;
}
// Do the tripping
if (numberOfErrorsInPeriod >= settings.Attempts)
{
State = CircuitBreakerState.Open;
}
RemoveOutOfPeriodErrors(errorsInPeriod);
await Task.FromResult(0);
}
public async Task Reset()
{
if (IsClosed)
{
return;
}
var latestError = _errorDates
.OrderBy(x => x.Key)
.Take(1)
.FirstOrDefault();
if (latestError.Equals(default(KeyValuePair <int, DateTimeOffset>)))
{
return;
}
var currentTime = rebusTime.Now;
if (currentTime > latestError.Value + settings.HalfOpenResetInterval)
{
State = CircuitBreakerState.HalfOpen;
}
if (currentTime > latestError.Value + settings.CloseResetInterval)
{
State = CircuitBreakerState.Closed;
}
}
private static async Task <CircuitBreakerState> RunScenario(int[] openingRates, Rating[] analyseResponses)
{
var circuitBreakerPeriod = new CircuitBreakerPeriod {
PeriodMs = 10
};
var circuitBreakerOpeningRates = A.Fake <ICircuitBreakerOpeningRates>();
var circuitBreakerAnalyser = new StubCircuitBreakerAnalyser(analyseResponses);
var testSubject = new CircuitBreakerState("policy", circuitBreakerAnalyser, circuitBreakerOpeningRates, circuitBreakerPeriod);
A.CallTo(() => circuitBreakerOpeningRates.OpeningRates).Returns(openingRates);
var monitorTask = Task.Run(() => testSubject.MonitorAsync(CancellationToken.None));
// Wait long enough for the monitor task to throw an exception, due to no more analyser responses
await Task.Delay(100);
try
{
await monitorTask;
}
catch (InvalidOperationException)
{
}
return(testSubject);
}
public void when_a_circuitbreaker_exists_then_setting_forces_the_state_to_be_that_of_the_message(
[Values(CircuitBreakerStatus.Open, CircuitBreakerStatus.HalfOpen, CircuitBreakerStatus.Closed, CircuitBreakerStatus.ForcedClosed, CircuitBreakerStatus.ForcedOpen)]
CircuitBreakerStatus startingState,
[Values(CircuitBreakerStatus.Open, CircuitBreakerStatus.HalfOpen, CircuitBreakerStatus.Closed, CircuitBreakerStatus.ForcedClosed, CircuitBreakerStatus.ForcedOpen)]
CircuitBreakerStatus desiredState)
{
CircuitBreakerState state = Helper.MakeBreakerState(1, startingState);
CircuitBreakerState originalState = _set.ProcessCircuitStateMessage(state);
CircuitBreakerState result = _set.SetCircuitState(originalState.CircuitBreakerId, desiredState, SystemTime.Now());
CircuitBreakerState fetchedState = _set.TryGetCircuit(originalState.CircuitBreakerId);
Assert.That(result, Is.Not.Null);
Assert.That(result.Status, Is.EqualTo(desiredState));
Assert.That(result.MessageSequenceNumber, Is.EqualTo(originalState.MessageSequenceNumber)); // important that the sequence number does not change -> otherwise the next inbound messages will be discards by the dashboard.
Assert.That(fetchedState, Is.Not.Null);
Assert.That(fetchedState.Status, Is.EqualTo(desiredState));
Assert.That(fetchedState.MessageSequenceNumber, Is.EqualTo(originalState.MessageSequenceNumber));
Assert.That(_set.Circuits(), Contains.Item(originalState.CircuitBreakerId));
}
public static string UpdateSpecificCircuit(CircuitBreakerSet set, string circuitId, int sequenceNumber)
{
CircuitBreakerState input = MakeBreakerState(messageSequenceNumber: sequenceNumber, id: circuitId);
set.ProcessCircuitStateMessage(input);
return(circuitId);
}
private void ResetCircuitBreaker()
{
lock (padLock)
{
_state = CircuitBreakerState.Closed;
}
}
public void circuit_should_be_accepting_all_requests_on_creation()
{
var testSubject = new CircuitBreakerState("policy", A.Fake <ICircuitBreakerAnalyser>(), A.Fake <ICircuitBreakerOpeningRates>(), A.Fake <ICircuitBreakerPeriod>());
var accepts = Enumerable.Range(0, 100).Select(c => testSubject.ShouldAccept()).ToArray();
Assert.That(accepts.All(c => c), Is.True);
}
public static string AddRandomCircuit(CircuitBreakerSet set)
{
CircuitBreakerState input = MakeBreakerState(messageSequenceNumber: 15);
var circuitId = input.CircuitBreakerId;
set.ProcessCircuitStateMessage(input);
return(circuitId);
}
/// <summary>
/// Handles state change logic.
/// </summary>
/// <param name="newState"></param>
private void ChangeState(CircuitBreakerState newState)
{
// Change the circuit breaker state
this.state = newState;
// Raise changed event
this.OnCircuitBreakerStateChanged(new EventArgs());
}
public async Task <CircuitBreakerState> GetStateAsync(string serviceName, string functionName)
{
var table = _cloudStorageAccount.CreateCloudTableClient().GetTableReference(TABLE_STATE);
CircuitBreakerFunctionState functionState = (await table.ExecuteAsync(TableOperation.Retrieve <CircuitBreakerFunctionState>(serviceName, functionName))).Result as CircuitBreakerFunctionState;
return(functionState == null ? null : CircuitBreakerState.FromState(functionState));
}
public CircuitBreaker(uint threshold, TimeSpan resetTimeout)
{
this._threshold = threshold;
_failureCount = 0;
_state = CircuitBreakerState.Closed;
_resetTimer = new Timer(resetTimeout.TotalMilliseconds);
_resetTimer.Elapsed += ResetTimerElapsed;
}
public void when_a_circuitbreaker_does_not_exist_then_setting_a_state_returns_null()
{
// the rationale behind this logic is that the circuit has been garbage collected because it was offline long enough.
// therefore a 'set' command from the UI should not create it, but ignore it. A null is returned rather than an exception.
CircuitBreakerState result = _set.SetCircuitState(Guid.NewGuid().ToString("N"), CircuitBreakerStatus.HalfOpen, SystemTime.Now());
Assert.That(result, Is.Null);
Assert.That(_set.Circuits(), Is.Not.Contains("a"));
}
public CircuitBreaker(uint threshold, uint timeout)
{
this.threshold = threshold;
this.failureCount = 0;
this.state = CircuitBreakerState.Closed;
this.ignoredExceptionTypes = new List <Type>();
this.timer = new System.Timers.Timer(timeout);
this.timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
}
public CircuitBreaker(uint threshold, uint timeout)
{
this.threshold = threshold;
this.failureCount = 0;
this.state = CircuitBreakerState.Closed;
this.ignoredExceptionTypes = new List<Type>();
this.timer = new System.Timers.Timer(timeout);
this.timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
}
private void ProcessException(Exception ex)
{
lock (padLock)
{
lastFailureTime = DateTime.Now;
lastException = ex;
_state = CircuitBreakerState.Open;
}
throw new CircuitBreakerOpenException(lastException);
}
public CircuitBreaker(int threshold, int timeout)
{
_threshold = threshold;
_failureCount = 0;
_state = CircuitBreakerState.Closed;
_ignoredExceptionTypes = new List <Type>();
_timer = new Timer(timeout);
_timer.Elapsed += TimerElapsed;
}
/// <summary>
/// <para> Builds a <see cref="Policy"/> that will function like a Circuit Breaker.</para>
/// <para>The circuit will break after <paramref name="exceptionsAllowedBeforeBreaking"/>
/// exceptions that are handled by this policy are raised. The circuit will stay
/// broken for the <paramref name="durationOfBreak"/>. Any attempt to execute this policy
/// while the circuit is broken, will immediately throw a <see cref="BrokenCircuitException"/> containing the exception
/// that broke the cicuit.
/// </para>
/// <para>If the first action after the break duration period results in an exception, the circuit will break
/// again for another <paramref name="durationOfBreak"/>, otherwise it will reset.
/// </para>
/// </summary>
/// <param name="policyBuilder">The policy builder.</param>
/// <param name="exceptionsAllowedBeforeBreaking">The number of exceptions that are allowed before opening the circuit.</param>
/// <param name="durationOfBreak">The duration the circuit will stay open before resetting.</param>
/// <returns>The policy instance.</returns>
/// <remarks>(see "Release It!" by Michael T. Nygard fi)</remarks>
/// <exception cref="System.ArgumentOutOfRangeException">exceptionsAllowedBeforeBreaking;Value must be greater than zero.</exception>
public static Policy CircuitBreaker(this PolicyBuilder policyBuilder, int exceptionsAllowedBeforeBreaking, TimeSpan durationOfBreak)
{
if (exceptionsAllowedBeforeBreaking <= 0)
{
throw new ArgumentOutOfRangeException("exceptionsAllowedBeforeBreaking", "Value must be greater than zero.");
}
var policyState = new CircuitBreakerState(exceptionsAllowedBeforeBreaking, durationOfBreak);
return(new Policy(action => CircuitBreakerPolicy.Implementation(action, policyBuilder.ExceptionPredicates, policyState)));
}
/// <summary>
/// <para>Builds the policy that will "break the circuit" after <paramref name="countBeforeBreaking"/>
/// exceptions that could be handled by the <paramref name="syntax"/> being built. The circuit
/// stays broken for the <paramref name="duration"/>. Any attempt to
/// invoke method within the policy, while the circuit is broken, will immediately re-throw
/// the last exception. </para>
/// <para>If the action fails within the policy after the block period, then the breaker
/// is blocked again for the next <paramref name="duration"/>.
/// It will be reset, otherwise.</para>
/// </summary>
/// <param name="syntax">The syntax.</param>
/// <param name="duration">How much time the breaker will stay open before resetting</param>
/// <param name="countBeforeBreaking">How many exceptions are needed to break the circuit</param>
/// <returns>shared policy instance</returns>
/// <remarks>(see "ReleaseIT!" for the details)</remarks>
public static ActionPolicyWithState CircuitBreaker(this Syntax <ExceptionHandler> syntax, TimeSpan duration,
int countBeforeBreaking)
{
Enforce.Argument(() => syntax);
Enforce.Argument(() => countBeforeBreaking, Is.GreaterThan(0));
Enforce.Argument(() => duration, Is.NotDefault);
var state = new CircuitBreakerState(duration, countBeforeBreaking);
var syncLock = new CircuitBreakerStateLock(state);
return(new ActionPolicyWithState(action => CircuitBreakerPolicy.Implementation(action, syntax.Target, syncLock)));
}
/// <summary>
/// <para> Builds a <see cref="Policy"/> that will function like a Circuit Breaker.</para>
/// <para>The circuit will break after <paramref name="exceptionsAllowedBeforeBreaking"/>
/// exceptions that are handled by this policy are raised. The circuit will stay
/// broken for the <paramref name="durationOfBreak"/>. Any attempt to execute this policy
/// while the circuit is broken, will immediately throw a <see cref="BrokenCircuitException"/> containing the exception
/// that broke the cicuit.
/// </para>
/// <para>If the first action after the break duration period results in an exception, the circuit will break
/// again for another <paramref name="durationOfBreak"/>, otherwise it will reset.
/// </para>
/// </summary>
/// <param name="policyBuilder">The policy builder.</param>
/// <param name="exceptionsAllowedBeforeBreaking">The number of exceptions that are allowed before opening the circuit.</param>
/// <param name="durationOfBreak">The duration the circuit will stay open before resetting.</param>
/// <returns>The policy instance.</returns>
/// <remarks>(see "Release It!" by Michael T. Nygard fi)</remarks>
/// <exception cref="System.ArgumentOutOfRangeException">exceptionsAllowedBeforeBreaking;Value must be greater than zero.</exception>
public static Policy CircuitBreakerAsync(this PolicyBuilder policyBuilder, int exceptionsAllowedBeforeBreaking, TimeSpan durationOfBreak)
{
if (exceptionsAllowedBeforeBreaking <= 0)
{
throw new ArgumentOutOfRangeException("exceptionsAllowedBeforeBreaking", "Value must be greater than zero.");
}
var policyState = new CircuitBreakerState(exceptionsAllowedBeforeBreaking, durationOfBreak);
return(new Policy(
(action, cancellationToken, continueOnCapturedContext) => CircuitBreakerPolicy.ImplementationAsync(action, cancellationToken, policyBuilder.ExceptionPredicates, policyState, continueOnCapturedContext),
policyBuilder.ExceptionPredicates
));
}
public void adding_an_unseen_circuit_returns_a_passthrough_message_stamped_with_current_time()
{
SystemTime.Is(millisecond: 100);
CircuitBreakerState input = Helper.MakeBreakerState(messageSequenceNumber: 15);
SystemTime.Is(millisecond: 200);
CircuitBreakerState result = _set.ProcessCircuitStateMessage(input);
Assert.That(result.CircuitBreakerId, Is.EqualTo(input.CircuitBreakerId), "CircuitBreakerId");
Assert.That(result.MessageSequenceNumber, Is.EqualTo(input.MessageSequenceNumber), "RequestMessageSequenceNumber");
Assert.That(result.ProcessId, Is.EqualTo(input.ProcessId), "ProcessID");
Assert.That(result.Status, Is.EqualTo(input.Status), "Status");
Assert.That(result.MachineName, Is.EqualTo(input.MachineName), "MachineName");
Assert.That(result.LastModifiedDate, Is.EqualTo(input.LastModifiedDate.AddMilliseconds(100)), "LastModifiedDate");
Assert.That(result.LastModifiedDate, Is.EqualTo(SystemTime.Now()), "LastModifiedDate should be now");
}
public void an_older_message_for_existing_breaker_is_ignored_by_update_and_returns_the_more_recent_state()
{
SystemTime.Is(second: 1, millisecond: 100);
CircuitBreakerState original = Helper.MakeBreakerState("a", messageSequenceNumber: 15);
CircuitBreakerState originalResult = _set.ProcessCircuitStateMessage(original);
CircuitBreakerState ignoredResult =
_set.ProcessCircuitStateMessage(Helper.MakeBreakerState("a", messageSequenceNumber: 14));
Assert.That(ignoredResult.CircuitBreakerId, Is.EqualTo(originalResult.CircuitBreakerId), "CircuitBreakerId");
Assert.That(ignoredResult.MessageSequenceNumber, Is.EqualTo(originalResult.MessageSequenceNumber), "RequestMessageSequenceNumber");
Assert.That(ignoredResult.ProcessId, Is.EqualTo(originalResult.ProcessId), "ProcessID");
Assert.That(ignoredResult.Status, Is.EqualTo(originalResult.Status), "Status");
Assert.That(ignoredResult.MachineName, Is.EqualTo(originalResult.MachineName), "MachineName");
Assert.That(ignoredResult.LastModifiedDate, Is.EqualTo(SystemTime.Now()), "LastModifiedDate should be now");
}
public void a_newer_message_number_for_existing_breaker_is_processed_for_update()
{
SystemTime.Is(second: 1, millisecond: 100);
CircuitBreakerState original = Helper.MakeBreakerState("a", messageSequenceNumber: 15);
CircuitBreakerState originalResult = _set.ProcessCircuitStateMessage(original);
SystemTime.Is(second: 2, millisecond: 200);
CircuitBreakerState newResult =
_set.ProcessCircuitStateMessage(Helper.MakeBreakerState("a", messageSequenceNumber: 16));
Assert.That(newResult.CircuitBreakerId, Is.EqualTo(originalResult.CircuitBreakerId), "CircuitBreakerId");
Assert.That(newResult.MessageSequenceNumber, Is.EqualTo(16), "RequestMessageSequenceNumber");
Assert.That(newResult.ProcessId, Is.EqualTo(originalResult.ProcessId), "ProcessID");
Assert.That(newResult.Status, Is.EqualTo(originalResult.Status), "Status");
Assert.That(newResult.MachineName, Is.EqualTo(originalResult.MachineName), "MachineName");
Assert.That(newResult.LastModifiedDate, Is.EqualTo(SystemTime.Now()), "LastModifiedDate should be now");
}
public CircuitBreakerState Trip(Exception exception, CircuitBreakerState currentState)
{
var now = ticksFn();
var horizon = now - interval;
int count;
lock (wsync)
{
errors.Add(now);
count = errors.CountItems(x => x > horizon);
}
if (count > maxErrorsInInterval)
{
return(CircuitBreakerState.Open);
}
return(count > 0 ? CircuitBreakerState.HalfOpen : CircuitBreakerState.Closed);
}
public CircuitBreaker(int threshold = 5, int timeout = 60000)
{
if (threshold <= 0)
{
throw new ArgumentOutOfRangeException($"{threshold} deve ser maior que zero");
}
if (timeout <= 0)
{
throw new ArgumentOutOfRangeException($"{timeout} deve ser maior que zero");
}
this.ThresHold = threshold;
this.Timeout = timeout;
this.State = CircuitBreakerState.Closed;
this.Timer = new Timer(timeout);
this.Timer.Enabled = false;
this.Timer.Elapsed += Timer_Elapsed;
}
public async Task setup_scenario_base()
{
ClosedPcts = new List <double>();
ShouldAccepts = new List <bool>();
var circuitBreakerPeriod = new CircuitBreakerPeriod {
PeriodMs = 100
};
var circuitBreakerOpeningRates = new BinaryCircuitBreakerOpeningRates();
var circuitBreakerAnalyser = new DefaultCircuitBreakerAnalyser();
TestSubject = new CircuitBreakerState("policy", circuitBreakerAnalyser, circuitBreakerOpeningRates, circuitBreakerPeriod);
var cancellationTokenSource = new CancellationTokenSource();
var monitorTask = Task.Run(() => TestSubject.MonitorAsync(cancellationTokenSource.Token));
await RunScenario();
cancellationTokenSource.Cancel();
await monitorTask;
}
void ChangeCircuitBreakerState(CircuitBreakerState previousState, CircuitBreakerState currentState)
{
_log.Info("Circuit breaker changed from {PreviousState} to {State}", previousState, currentState);
_circuitBreakerEvents.RaiseCircuitBreakerChanged(currentState);
if (currentState == CircuitBreakerState.Closed)
{
SetNumberOfWorkers(_configuredNumberOfWorkers);
return;
}
if (currentState == CircuitBreakerState.HalfOpen)
{
SetNumberOfWorkers(1);
return;
}
if (currentState == CircuitBreakerState.Open)
{
SetNumberOfWorkers(0);
return;
}
}
public CircuitBreaker()
{
_state = CircuitBreakerState.Closed;
padLock = new object();
}
public override void Update(float elapsedSeconds)
{
SetClosingAuthorization(TCSClosingAuthorization() && DriverClosingAuthorization() && CurrentPantographState() == PantographState.Up);
switch (CurrentState())
{
case CircuitBreakerState.Closed:
if (!ClosingAuthorization())
{
SetCurrentState(CircuitBreakerState.Open);
}
break;
case CircuitBreakerState.Closing:
if (ClosingAuthorization() && (DriverClosingOrder() || TCSClosingOrder()))
{
if (!ClosingTimer.Started)
{
ClosingTimer.Start();
}
if (ClosingTimer.Triggered)
{
ClosingTimer.Stop();
SetCurrentState(CircuitBreakerState.Closed);
}
}
else
{
ClosingTimer.Stop();
SetCurrentState(CircuitBreakerState.Open);
}
break;
case CircuitBreakerState.Open:
if (ClosingAuthorization() && (DriverClosingOrder() || TCSClosingOrder()))
{
SetCurrentState(CircuitBreakerState.Closing);
}
break;
}
if (PreviousState != CurrentState())
{
switch (CurrentState())
{
case CircuitBreakerState.Open:
SignalEvent(Event.CircuitBreakerOpen);
break;
case CircuitBreakerState.Closing:
SignalEvent(Event.CircuitBreakerClosing);
break;
case CircuitBreakerState.Closed:
SignalEvent(Event.CircuitBreakerClosed);
break;
}
}
PreviousState = CurrentState();
}
/// <summary>
/// Handles state change logic.
/// </summary>
/// <param name="newState"></param>
private void ChangeState(CircuitBreakerState newState)
{
// Change the circuit breaker state
this.state = newState;
// Raise changed event
this.OnCircuitBreakerStateChanged(new EventArgs());
}
private void ResetCircuitBreaker()
{
lock (padLock)
{
_state = CircuitBreakerState.Closed;
}
}
private void ProcessException(Exception ex)
{
lock (padLock)
{
lastFailureTime = DateTime.Now;
lastException = ex;
_state = CircuitBreakerState.Open;
}
throw new CircuitBreakerOpenException(lastException);
}
public CircuitBreaker()
{
_state = CircuitBreakerState.Closed;
padLock = new object();
}
