public async Task AddFailure(FailureRequest req)
{
if (State == CircuitState.Open)
{
_log.LogInformation($"Tried to add additional failure to {Entity.Current.EntityKey} that is already open.");
return;
}
FailureWindow.Add(req.RequestId, req);
var cutoff = req.FailureTime.Subtract(windowSize);
// Filter the window only to exceptions within the cutoff timespan
FailureWindow = FailureWindow.Where(p => p.Value.FailureTime >= cutoff).ToDictionary(p => p.Key, p => p.Value);
if (FailureWindow.Count >= failureThreshold)
{
_log.LogCritical($"Break this circuit for entity {Entity.Current.EntityKey}!");
await _durableClient.StartNewAsync(nameof(OpenCircuitOrchestrator.OpenCircuit), req.InstanceId);
// Mark the circuit as "open" (circuit is broken)
State = CircuitState.Open;
}
else
{
_log.LogInformation($"The circuit {Entity.Current.EntityKey} currently has {FailureWindow.Count} exceptions in the window of {windowSize.ToString()}");
}
}
public void CircuitStateConstructorTest()
{
int reserveState = 3;
CircuitState target = new CircuitState(reserveState);
Assert.AreEqual(reserveState, target.Count);
}
public Boolean Allow()
{
if ((this.state == CircuitState.OPEN) && (this.nextCheck < DateTime.Now.Ticks / 1000L))
this.state = CircuitState.HALF_OPEN;
return (this.state == CircuitState.CLOSED) || (this.state == CircuitState.HALF_OPEN);
}
public CircuitStatus(string clientId, string circuitName, CircuitState newState, DateTimeOffset dateCreated)
{
ClientId = clientId;
CircuitName = circuitName;
NewState = newState;
DateCreated = dateCreated;
}
public TResult Execute <TResult>(Func <TResult> action)
{
Console.WriteLine("--------------------");
if (action == null)
{
throw new ArgumentNullException("action");
}
if (_state == CircuitState.Closed)
{
return(PerformAction(action));
}
else if (_state == CircuitState.Open)
{
bool timeoutExpired = DateTime.UtcNow.Ticks >= _blockedTill;
if (timeoutExpired)
{
Console.WriteLine("HALF-Open at " + DateTime.UtcNow.ToString());
_state = CircuitState.HalfOpen;
return(PerformAction(action));
}
else
{
Console.WriteLine("OpenCircuitExeption at " + DateTime.UtcNow.ToString());
throw new OpenCircuitException();
}
}
else // half-open
{
return(PerformAction(action));
}
}
/// <summary>
/// Integrate a state variable
/// Note that the integrated quantity will/should be stored at the next index!
/// </summary>
/// <param name="state">The state of the circuit</param>
/// <param name="geq">The Geq parameter</param>
/// <param name="ceq">The Ceq parameter</param>
/// <param name="index">The index of the variable to be integrated</param>
/// <param name="cap">The capacitance</param>
public void Integrate(CircuitState state, out double geq, out double ceq, int index, double cap)
{
var result = Integrate(state, index, cap);
geq = result.Geq;
ceq = result.Ceq;
}
void CloseBreaker()
{
Console.WriteLine("Close breaker");
_state = CircuitState.Closed;
_currentFailureCount = 0;
_blockedTill = DateTime.MinValue.Ticks;
}
/// <summary>
/// Integrate a state variable
/// </summary>
/// <param name="ckt">The circuit</param>
/// <param name="index">The state index to integrate</param>
/// <param name="cap">The capacitance</param>
/// <returns></returns>
public override Result Integrate(CircuitState state, int qcap, double cap)
{
int ccap = qcap + 1;
switch (Order)
{
case 1:
state.States[0][ccap] = ag[0] * state.States[0][qcap] + ag[1] * state.States[1][qcap];
break;
case 2:
state.States[0][ccap] = -state.States[1][ccap] * ag[1] + ag[0] * (state.States[0][qcap] - state.States[1][qcap]);
break;
default:
throw new CircuitException("Invalid order");
}
// Create the returned object
Result result = new Result();
result.Ceq = state.States[0][ccap] - ag[0] * state.States[0][qcap];
result.Geq = ag[0] * cap;
return(result);
}
public void CircuitStateEndDefinitionTest()
{
ProjectTester.InitResources();
CircuitProject project = CircuitProject.Create(null);
CircuitButton button = null;
CircuitSymbol buttonSymbol = null;
Gate led = null;
CircuitSymbol ledSymbol = null;
project.InTransaction(() => {
button = project.CircuitButtonSet.Create("b", false, PinSide.Right);
buttonSymbol = project.CircuitSymbolSet.Create(button, project.ProjectSet.Project.LogicalCircuit, 1, 1);
led = project.GateSet.Gate(GateType.Led, 1, false);
ledSymbol = project.CircuitSymbolSet.Create(led, project.ProjectSet.Project.LogicalCircuit, 5, 1);
});
CircuitState target = new CircuitState(3);
int buttonResult = target.ReserveState();
FunctionButton functionButton = new FunctionButton(target, new CircuitSymbol[] { buttonSymbol }, buttonResult);
FunctionLed functionLed = new FunctionLed(target, new CircuitSymbol[] { ledSymbol }, buttonResult);
target.EndDefinition();
Assert.IsTrue(functionButton.Dependent != null && functionButton.Dependent.Length == 1 && functionButton.Dependent[0] == functionLed);
Assert.IsTrue(functionLed.Dependent == null);
}
public TimeSegmentChanges(TimeSpan interval, CircuitState startState, BinaryVarValue[] changes, List <TimeStepChanges> timeSteps)
{
this.TimeInterval = interval;
this.StartState = startState;
this.Changes = changes;
this.StepChanges = timeSteps;
}
static void AdvancedCircuitBreaker()
{
Action <Exception, TimeSpan, Context> onBreak = (exception, timespan, context) =>
{
Console.WriteLine("onBreak Running,the exception.Message : " + exception.Message);
};
Action <Context> onReset = context =>
{
Console.WriteLine("Job Back to normal");
};
var breaker = Policy.Handle <ArgumentException>()
.AdvancedCircuitBreaker(
failureThreshold: 0.5, // Break on >=50% actions result in handled exceptions...
samplingDuration: TimeSpan.FromSeconds(3), // ... over any 10 second period
minimumThroughput: 3, // ... provided at least 8 actions in the 10 second period.
durationOfBreak: TimeSpan.FromSeconds(5), // Break for 30 seconds.
onBreak: onBreak, onReset: onReset);
ISyncPolicy policy = Policy.Handle <ArgumentException>()
.WaitAndRetry(new[] { TimeSpan.FromMilliseconds(200),
TimeSpan.FromMilliseconds(400) }, (ex, timesapn, context) =>
{
// Monitor the circuit state, for example for health reporting.
CircuitState state = breaker.CircuitState;
Console.WriteLine(state.ToString());
Console.WriteLine($"Runing Retry,Exception :{ex.Message},timesapn:{timesapn}");
});
while (true)
{
try
{
var policyWrap = Policy.Wrap(policy, breaker);
// (wraps the policies around any executed delegate: fallback outermost ... bulkhead innermost)
policyWrap.Execute(() =>
{
Console.WriteLine("Job Start");
if (DateTime.Now.Second % 2 == 0)
{
throw new ArgumentException("Hello Polly!");
}
});
}
catch (Exception ex)
{
// 手动打开熔断器，阻止执行
//breaker.Isolate();
}
Thread.Sleep(500);
// 恢复操作，启动执行
//breaker.Reset();
}
}
public static void ChangeCircuitState(string circuitName, CircuitState newState)
{
lock (StateLocker)
{
if (_circuits.ContainsKey(circuitName))
_circuits[circuitName].SetState(newState);
}
}
private void CheckDistributedRecovery(CircuitState state)
{
if (this.internalState == InternalState.DistributedOpen && state == CircuitState.Closed)
{
this.internalState = InternalState.RecoveredFromDistributedOpen;
this.healEndTimeTicks = DateTimeOffset.UtcNow.Add(this.healDuration).Ticks;
}
}
public void Update(CircuitState state)
{
// check whether we are in a recovery phase
this.CheckDistributedRecovery(state);
this.localCircuitState = state;
this.distributedStates.AddOrUpdate(this.key, state, (s, circuitState) => state);
}
private void Trip()
{
if (this.state != CircuitState.OPEN)
{
this.state = CircuitState.OPEN;
this.nextCheck = (DateTime.Now.Ticks / 1000L + this.timeoutInSecs);
}
}
void ResetInternal_NeedsLock()
{
_count = 0;
_blockedTill = DateTime.MinValue;
_circuitState = CircuitState.Closed;
_lastException = new InvalidOperationException("This exception should never be thrown");
}
static void FallbackWrap()
{
Action <Exception, TimeSpan, Context> onBreak = (exception, timespan, context) =>
{
Console.WriteLine("onBreak Running,the exception.Message : " + exception.Message);
};
Action <Context> onReset = context =>
{
Console.WriteLine("Job Back to normal");
};
ISyncPolicy fallback = Policy.Handle <Polly.CircuitBreaker.BrokenCircuitException>()
.Or <System.ArgumentException>()
.Fallback(() =>
{
Console.WriteLine("Error occured,runing fallback");
},
(ex) => { Console.WriteLine($"Fallback exception:{ex.GetType().ToString()},Message:{ex.Message}"); });
ISyncPolicy retry = Policy.Handle <ArgumentException>()
.WaitAndRetry(new[] { TimeSpan.FromMilliseconds(200),
TimeSpan.FromMilliseconds(300) }, (ex, timesapn, context) =>
{
Console.WriteLine($"Runing Retry,Exception :{ex.Message},timesapn:{timesapn}");
});
var breaker = Policy.Handle <ArgumentException>()
.AdvancedCircuitBreaker(
failureThreshold: 0.5, // Break on >=50% actions result in handled exceptions...
samplingDuration: TimeSpan.FromSeconds(3), // ... over any 10 second period
minimumThroughput: 3, // ... provided at least 8 actions in the 10 second period.
durationOfBreak: TimeSpan.FromSeconds(8), // Break for 30 seconds.
onBreak: onBreak, onReset: onReset);
// Monitor the circuit state, for example for health reporting.
CircuitState state = breaker.CircuitState;
Console.WriteLine(state.ToString());
while (true)
{
var policyWrap = Policy.Wrap(fallback, retry, breaker);
// (wraps the policies around any executed delegate: fallback outermost ... bulkhead innermost)
policyWrap.Execute(() =>
{
Console.WriteLine("Job Start");
if (DateTime.Now.Second % 2 == 0)
{
throw new ArgumentException("Hello Polly!");
}
});
Thread.Sleep(300);
}
}
/// <summary>
/// Management bus - Send the circuit status change.
/// </summary>
/// <param name="circuitName"></param>
/// <param name="newState"></param>
public static void ManagementBusCircuitStateChangedNotifier(string circuitName, CircuitState newState)
{
if (_managementBusClient == null)
return;
var stateChange = new CircuitStatus(_managementBusClient.ClientId, circuitName, newState, DateTimeOffset.Now);
_managementBusClient.SendMessage(ManagementBus_StateChange_Topic, stateChange);
}
private void CircuitMapCleanUpTest(CircuitProject circuitProject, string logicalCircuitName, int expectedFunctions)
{
ProjectTester.SwitchTo(circuitProject, logicalCircuitName);
CircuitMap map = new CircuitMap(circuitProject.ProjectSet.Project.LogicalCircuit);
CircuitState state = map.Apply(CircuitRunner.HistorySize);
Assert.AreEqual(expectedFunctions, state.Functions.Count(), "wrong number of functions");
}
public void Isolate()
{
using (TimedLock.Lock(_lock))
{
_lastOutcome = new DelegateResult <TResult>(new IsolatedCircuitException("The circuit is manually held open and is not allowing calls."));
BreakFor_NeedsLock(TimeSpan.MaxValue, Context.None);
_circuitState = CircuitState.Isolated;
}
}
public void Isolate()
{
using (TimedLock.Lock(_lock))
{
_lastException = new IsolatedCircuitException("The circuit is manually held open and is not allowing calls.");
BreakFor_NeedsLock(TimeSpan.MaxValue, Context.Empty);
_circuitState = CircuitState.Isolated;
}
}
public IEnumerable <CircuitState> GetAllDistinctStates()
{
CircuitState state = StartState.Copy();
foreach (var step in StepChanges)
{
state.AddChanges(step.GetChanges(Changes), step.Time);
yield return(state);
}
}
protected CircuitStateController(TimeSpan durationOfBreak, Action <Exception, TimeSpan, Context> onBreak, Action <Context> onReset, Action onHalfOpen)
{
_durationOfBreak = durationOfBreak;
_onBreak = onBreak;
_onReset = onReset;
_onHalfOpen = onHalfOpen;
_circuitState = CircuitState.Closed;
Reset();
}
private void BreakFor_NeedsLock(TimeSpan durationOfBreak, Context context)
{
bool willDurationTakeUsPastDateTimeMaxValue = durationOfBreak > DateTime.MaxValue - SystemClock.UtcNow();
_blockedTill = willDurationTakeUsPastDateTimeMaxValue
? DateTime.MaxValue.Ticks
: (SystemClock.UtcNow() + durationOfBreak).Ticks;
_circuitState = CircuitState.Open;
_onBreak(_lastOutcome, durationOfBreak, context);
}
private void CircuitBreakerOnBreak(Exception exception, CircuitState state, TimeSpan timeSpan, Context context)
{
if (exception != null)
{
_logger.LogWarning(exception, "Circuit breaker [{0}:BREAK] due to exception", context.PolicyKey);
}
else
{
_logger.LogWarning("Circuit breaker [{0}:BREAK] due to manual", context.PolicyKey);
}
}
void BreakFor_NeedsLock(TimeSpan durationOfBreak, Context context)
{
bool willDurationTakeUsPastDateTimeMaxValue = durationOfBreak > DateTime.MaxValue - SystemClock.UtcNow();
_blockedTill = willDurationTakeUsPastDateTimeMaxValue
? DateTime.MaxValue
: SystemClock.UtcNow() + durationOfBreak;
_circuitState = CircuitState.Open;
_onBreak(_lastException, durationOfBreak, context ?? Context.Empty);
}
public Circuit(int threshold, TimeSpan? timeout, CircuitState state)
{
Threshold = threshold;
if (timeout.HasValue)
{
Timeout = timeout.Value;
}
State = state ?? new CircuitState { Position = CircuitPosition.Closed };
ExceptionFilters = new List<Func<Exception, bool>>();
}
private void WaitForState(CircuitState desired_state)
{
Monitor.Enter(_stateChangeEvent);
while (_state != desired_state)
{
Monitor.Exit(_stateChangeEvent);
_stateChangeEvent.WaitOne(Timeout.Infinite);
Monitor.Enter(_stateChangeEvent);
}
return;
}
static void CircuitBreaker()
{
Action <Exception, TimeSpan, Context> onBreak = (exception, timespan, context) =>
{
Console.WriteLine("onBreak Running : " + exception.Message);
};
Action <Context> onReset = context =>
{
Console.WriteLine("Job Back to normal");
};
CircuitBreakerPolicy breaker = Policy
.Handle <AggregateException>()
.CircuitBreaker(3, TimeSpan.FromSeconds(10), onBreak, onReset);
// Monitor the circuit state, for example for health reporting.
CircuitState state = breaker.CircuitState;
ISyncPolicy policy = Policy.Handle <ArgumentException>()
.Retry(3, (ex, retryCount, context) =>
{
Console.WriteLine($"Runing fallback,Exception :{ex.Message},RetryCount:{retryCount}");
});
while (true)
{
try
{
var policyWrap = Policy.Wrap(policy, breaker);
// (wraps the policies around any executed delegate: fallback outermost ... bulkhead innermost)
policyWrap.Execute(() =>
{
Console.WriteLine("Job Start");
if (DateTime.Now.Second % 3 == 0)
{
throw new ArgumentException("Hello Polly!");
}
});
}
catch (Exception ex)
{
// 手动打开熔断器，阻止执行
breaker.Isolate();
}
Thread.Sleep(1000);
// 恢复操作，启动执行
breaker.Reset();
}
}
protected void ResetInternal_NeedsLock(Context context)
{
_blockedTill = DateTime.MinValue.Ticks;
_lastOutcome = null;
CircuitState priorState = _circuitState;
_circuitState = CircuitState.Closed;
if (priorState != CircuitState.Closed)
{
_onReset(context);
}
}
protected void ResetInternal_NeedsLock(Context context)
{
_blockedTill = DateTime.MinValue;
_lastOutcome = new DelegateResult <TResult>(new InvalidOperationException("This exception should never be thrown"));
CircuitState priorState = _circuitState;
_circuitState = CircuitState.Closed;
if (priorState != CircuitState.Closed)
{
_onReset(context);
}
}
protected void ResetInternal_NeedsLock(Context context)
{
_blockedTill = DateTime.MinValue;
_lastException = new InvalidOperationException("This exception should never be thrown");
CircuitState priorState = _circuitState;
_circuitState = CircuitState.Closed;
if (priorState != CircuitState.Closed)
{
_onReset(context ?? Context.Empty);
}
}
private void Reset(Context context)
{
using (TimedLock.Lock(_lock))
{
CircuitState priorState = _circuitState;
ResetInternal_NeedsLock();
if (priorState != CircuitState.Closed)
{
_onReset(context ?? Context.Empty);
}
}
}
public void CircuitStateItemTest()
{
CircuitState target = new CircuitState(3);
target.EndDefinition();
Assert.AreEqual <int>(3, target.Count);
Assert.AreEqual <State>(State.Off, target[0]);
Assert.AreEqual <State>(State.Off, target[1]);
Assert.AreEqual <State>(State.Off, target[2]);
target[0] = State.On0;
target[1] = State.On1;
Assert.AreEqual <State>(State.On0, target[0]);
Assert.AreEqual <State>(State.On1, target[1]);
}
public void CircuitStateEvaluateTest()
{
CircuitState target = new CircuitState(3);
OneBitConst c1 = new OneBitConst(target, State.On0, 0);
OneBitConst c2 = new OneBitConst(target, State.On1, 1);
FunctionAnd and = FunctionAnd.Create(target, new int[] { 0, 1 }, 2);
target.EndDefinition();
bool success = target.Evaluate(true);
Assert.IsTrue(success);
Assert.AreEqual <State>(State.On0, target[0]);
Assert.AreEqual <State>(State.On1, target[1]);
Assert.AreEqual <State>(State.On0, target[2]);
}
private void CircuitMapPerfTest(string project, string initialCircuit, int maxCount, int maxSeconds)
{
CircuitProject circuitProject = ProjectTester.Load(this.TestContext, project, initialCircuit);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < maxCount; i++)
{
CircuitMap circuitMap = new CircuitMap(circuitProject.ProjectSet.Project.LogicalCircuit);
CircuitState circuitState = circuitMap.Apply(CircuitRunner.HistorySize);
Assert.IsNotNull(circuitState);
circuitMap.TurnOn();
}
stopwatch.Stop();
this.TestContext.WriteLine("{0} CircuitMap(s) created and applied in {1} - {2:N2} sec per each map", maxCount, stopwatch.Elapsed, stopwatch.Elapsed.TotalSeconds / maxCount);
Assert.IsTrue(stopwatch.Elapsed < new TimeSpan(0, 0, maxSeconds), "CircuitMap was created and applied successfully but too slow");
}
public void ResetCircuitState()
{
if (m_CircuitState == null)
{
m_CircuitState = new CircuitState();
}
m_CircuitState.fSqCheckpointDist = 0.0f;
m_CircuitState.iCheckPoint = -1;
m_CircuitState.iLaps = 0;
}
private void Reset()
{
this.state = CircuitState.CLOSED;
this.failureRate = 0;
}
protected void HandleException(Exception exception, CircuitState state)
{
if (ExceptionFilters.Any(filter => filter(exception)))
{
return;
}
if (ExcludedExceptions == null || !ExcludedExceptions.Contains(exception.GetType()))
{
state.ResetTime = DateTime.UtcNow.Add(Timeout);
state.Increment();
if (state.Position == CircuitPosition.HalfOpen || state.CurrentIteration >= Threshold)
{
state.Position = CircuitPosition.Open;
}
if (Logger != null)
{
Logger.Invoke(exception.Message);
}
}
}
public ChangeCircuitState(string circuitName, CircuitState newState)
{
CircuitName = circuitName;
NewState = newState;
}
