public void When_tasks_get_deleted_after_scheduling_System_will_not_execute_them()
{
var successTasks = new[] { 0.5, 1.5, 2.5 };
var tasksToRemove = new[] { 1.0, 2.0 };
foreach (var task in successTasks.Concat(tasksToRemove))
{
var text = task.ToString(CultureInfo.InvariantCulture);
var testMessage = new SuccessCommand(text);
_scheduler.Tell(new ScheduleCommand(testMessage, new ScheduleKey(Guid.Empty, text, text), CreateOptions(task, Timeout)));
}
var successTaskIds = successTasks.Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray();
var tasksToRemoveTaskIds = tasksToRemove.Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray();
foreach (var taskId in tasksToRemoveTaskIds)
{
_scheduler.Tell(new Unschedule(new ScheduleKey(Guid.Empty, taskId, taskId)));
}
Throttle.Assert(() =>
{
ResultHolder.Contains(successTaskIds);
Assert.True(tasksToRemoveTaskIds.All(x => ResultHolder.Get(x) == null));
}, minTimeout: TimeSpan.FromSeconds(4));
}
public async void TestCircuitClosedAfterSuccess()
{
String key = "cmd-G";
try
{
int sleepWindow = 100;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 1, sleepWindow);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key, sleepWindow);
cmd4.Execute();
// everything has failed in the test window so we should return false now
Time.Wait(150);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen);
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 60, sleepWindow);
IObservable <bool> asyncResult = cmd5.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest);
await asyncResult.SingleAsync();
// all requests should be open again
Time.Wait(150);
output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void When_some_of_scheduled_jobs_fail_System_still_executes_others()
{
var successTasks = new[] { 0.5, 1.5, 2.5 };
var failTasks = new[] { 1.0, 2.0 };
foreach (var task in successTasks)
{
var text = task.ToString(CultureInfo.InvariantCulture);
var testCommand = new SuccessCommand(text);
_scheduler.Tell(new ScheduleCommand(testCommand, new ScheduleKey(Guid.Empty, text, text), CreateOptions(task, Timeout)));
}
foreach (var failTask in failTasks)
{
var text = failTask.ToString(CultureInfo.InvariantCulture);
var failTaskCommand = new FailCommand();
_scheduler.Tell(new ScheduleCommand(failTaskCommand, new ScheduleKey(Guid.Empty, text, text), CreateOptions(failTask, Timeout)));
}
var successTaskIds = successTasks.Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray();
var failTaskIds = failTasks.Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray();
Throttle.Assert(() =>
{
ResultHolder.Contains(successTaskIds);
Assert.True(failTaskIds.All(x => ResultHolder.Get(x) == null));
}, minTimeout: TimeSpan.FromSeconds(3));
}
public void TestGetErrorPercentage()
{
string key = "cmd-metrics-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1);
HystrixCommandMetrics metrics = cmd1._metrics;
cmd1.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1);
cmd2.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
cmd3.Execute();
cmd4.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(25, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd5.Execute();
cmd6.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd8 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd9 = new SuccessCommand(key, 1);
cmd7.Execute();
cmd8.Execute();
cmd9.Execute();
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// latent
HystrixCommand <bool> cmd10 = new SuccessCommand(key, 60);
cmd10.Execute();
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Time.Wait(200);
Assert.Equal(30, metrics.Healthcounts.ErrorPercentage);
}
public async Task CanCompleteSuccessfully()
{
CommandBase command = new SuccessCommand();
CommandResult result = await command.ExecuteAsync(CancellationToken.None);
Assert.Equal(0, result.ExitCode);
}
public async Task TestCircuitClosedAfterSuccess()
{
string key = "cmd-G";
int sleepWindow = 100;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 1, sleepWindow);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1, sleepWindow);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1, sleepWindow);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow);
_ = await cmd4.ExecuteAsync();
// everything has failed in the test window so we should return false now
Time.Wait(sleepWindow);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow);
IObservable <bool> asyncResult = cmd5.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
await asyncResult.SingleAsync();
// all requests should be open again
Time.Wait(sleepWindow + 50);
output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public void TestBadRequestsDoNotAffectErrorPercentage()
{
string key = "cmd-metrics-B";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
HystrixCommandMetrics metrics = cmd1._metrics;
Assert.True(WaitForHealthCountToUpdate(key, 1000), "Health count stream took to long");
cmd1.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
cmd2.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd3 = new BadRequestCommand(key, 0);
HystrixCommand <bool> cmd4 = new BadRequestCommand(key, 0);
try
{
cmd3.Execute();
}
catch (HystrixBadRequestException)
{
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + "Caught expected HystrixBadRequestException from cmd3");
}
try
{
cmd4.Execute();
}
catch (HystrixBadRequestException)
{
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + "Caught expected HystrixBadRequestException from cmd4");
}
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd5 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 0);
cmd5.Execute();
cmd6.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(75, metrics.Healthcounts.ErrorPercentage);
}
public void When_client_tries_to_add_two_task_with_same_id_Then_only_one_gets_executed()
{
var testMessage = new SuccessCommand(Name);
_scheduler.Tell(new ScheduleCommand(testMessage, new ScheduleKey(Guid.Empty, Name, Group), CreateOptions(0.5, Timeout)));
_scheduler.Tell(new ScheduleCommand(testMessage, new ScheduleKey(Guid.Empty, Name, Group), CreateOptions(1, Timeout)));
Throttle.Assert(() => Assert.True(ResultHolder.Count == 1), minTimeout: TimeSpan.FromSeconds(2));
}
public void Succeed()
{
if (SuccessCommand != null)
{
SuccessCommand.Data = new Dictionary <String, object>();
SuccessCommand.Data.Add("Task", this);
SuccessCommand.Execute();
}
}
public void TestBadRequestsDoNotAffectErrorPercentage()
{
string key = "cmd-metrics-B";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1);
HystrixCommandMetrics metrics = cmd1._metrics;
cmd1.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1);
cmd2.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd3 = new BadRequestCommand(key, 1);
HystrixCommand <bool> cmd4 = new BadRequestCommand(key, 1);
try
{
cmd3.Execute();
}
catch (HystrixBadRequestException)
{
output.WriteLine("Caught expected HystrixBadRequestException from cmd3");
}
try
{
cmd4.Execute();
}
catch (HystrixBadRequestException)
{
output.WriteLine("Caught expected HystrixBadRequestException from cmd4");
}
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd5 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 1);
cmd5.Execute();
cmd6.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(75, metrics.Healthcounts.ErrorPercentage);
}
public async Task TestTripCircuitOnFailuresAboveThreshold()
{
var key = "cmd-B";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
var cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
_ = await cmd4.ExecuteAsync();
HystrixCommand <bool> cmd5 = new FailureCommand(key, 0);
_ = await cmd5.ExecuteAsync();
HystrixCommand <bool> cmd6 = new SuccessCommand(key, 0);
_ = await cmd6.ExecuteAsync();
HystrixCommand <bool> cmd7 = new FailureCommand(key, 0);
_ = await cmd7.ExecuteAsync();
HystrixCommand <bool> cmd8 = new FailureCommand(key, 0);
_ = await cmd8.ExecuteAsync();
// Let window pass, this should trip the circuit as the error percentage is above the threshold
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async Task When_client_tries_to_add_two_task_with_same_id_Then_only_one_gets_executed()
{
var testMessage = new SuccessCommand("yes!");
Scheduler.Tell(new ScheduleCommandExecution(testMessage, new ScheduleKey(Name, Group), CreateOptions(0.5)));
Scheduler.Tell(new ScheduleCommandExecution(testMessage, new ScheduleKey(Name, Group), CreateOptions(1)));
await Task.Delay(2000);
Assert.True(ResultHolder.Count == 1);
}
public void TestCurrentConcurrentExecutionCount()
{
String key = "cmd-metrics-C";
HystrixCommandMetrics metrics = null;
List <IObservable <bool> > cmdResults = new List <IObservable <bool> >();
int NUM_CMDS = 8;
for (int i = 0; i < NUM_CMDS; i++)
{
HystrixCommand <Boolean> cmd = new SuccessCommand(key, 900);
if (metrics == null)
{
metrics = cmd.metrics;
}
IObservable <bool> eagerObservable = cmd.Observe();
cmdResults.Add(eagerObservable);
}
try
{
Time.Wait(200);
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
output.WriteLine("ReqLog: " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(NUM_CMDS, metrics.CurrentConcurrentExecutionCount);
CountdownEvent latch = new CountdownEvent(1);
Observable.Merge(cmdResults).Subscribe(
(n) =>
{
},
(e) =>
{
output.WriteLine("Error duing command execution");
output.WriteLine(e.ToString());
latch.SignalEx();
},
() =>
{
output.WriteLine("All commands done");
latch.SignalEx();
});
latch.Wait(10000);
Assert.Equal(0, metrics.CurrentConcurrentExecutionCount);
}
public async Task TestTripCircuitAsync()
{
var key = "cmd-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
var cb = cmd1._circuitBreaker;
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
_ = await cmd1.ExecuteAsync();
_ = await cmd2.ExecuteAsync();
_ = await cmd3.ExecuteAsync();
_ = await cmd4.ExecuteAsync();
// this should still allow requests as everything has been successful
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// fail
HystrixCommand <bool> cmd5 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd7 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd8 = new FailureCommand(key, 0);
Assert.False(await cmd5.ExecuteAsync());
Assert.False(await cmd6.ExecuteAsync());
Assert.False(await cmd7.ExecuteAsync());
Assert.False(await cmd8.ExecuteAsync());
// make sure window has passed
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void When_there_are_several_scheduled_jobs_System_executes_all_of_them()
{
var tasks = new[] { 0.5, 0.6, 0.7, 0.8, 1 };
foreach (var task in tasks)
{
var text = task.ToString(CultureInfo.InvariantCulture);
var testMessage = new SuccessCommand(text);
_scheduler.Tell(new ScheduleCommand(testMessage, new ScheduleKey(Guid.Empty, text, text), CreateOptions(task, Timeout)));
}
var taskIds = tasks.Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray();
Throttle.Assert(() => ResultHolder.Contains(taskIds));
}
public async Task TestTripCircuitOnFailuresAboveThreshold()
{
string key = "cmd-B";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
_ = await cmd4.ExecuteAsync();
HystrixCommand <bool> cmd5 = new FailureCommand(key, 1);
_ = await cmd5.ExecuteAsync();
HystrixCommand <bool> cmd6 = new SuccessCommand(key, 1);
_ = await cmd6.ExecuteAsync();
HystrixCommand <bool> cmd7 = new FailureCommand(key, 1);
_ = await cmd7.ExecuteAsync();
HystrixCommand <bool> cmd8 = new FailureCommand(key, 1);
_ = await cmd8.ExecuteAsync();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(200);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async Task TestCircuitDoesNotTripOnFailuresBelowThreshold()
{
string key = "cmd-C";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
ICircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
await cmd4.ExecuteAsync();
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 0);
await cmd5.ExecuteAsync();
HystrixCommand <bool> cmd6 = new FailureCommand(key, 0);
await cmd6.ExecuteAsync();
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0);
await cmd7.ExecuteAsync();
HystrixCommand <bool> cmd8 = new FailureCommand(key, 0);
await cmd8.ExecuteAsync();
// Allow window to pass, this should remain closed as the failure threshold is below the percentage limit
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
/// <summary>
/// Initializes a new instance of the <see cref="ExportImageViewModel"/> class.
/// </summary>
/// <param name="dialogService">Dialog service for opening dialogs from view model.</param>
/// <param name="height">Default height of the image.</param>
/// <param name="width">Default width of the image.</param>
/// <param name="dpi">Default DPI of the image.</param>
/// <param name="plotModel">Plot model to export</param>
public ExportImageViewModel(IDialogService dialogService, int height = 0, int width = 0, int dpi = 0, PlotModel plotModel = null)
{
this.dialogService = dialogService;
Height = height;
Width = width;
Dpi = dpi;
ExportCommand = ReactiveCommand.CreateCombined(new [] { SuccessCommand },
this.WhenAnyValue(x => x.FilePath, x => x.Height, x => x.Width, x => x.Dpi)
.Select(
x => !string.IsNullOrWhiteSpace(x.Item1) &&
x.Item2 >= 0 && x.Item3 >= 0 && x.Item4 >= 0));
BrowseFilesCommand = ReactiveCommand.Create(() => FilePath = this.dialogService.SaveFile(".png", @"Png Files (*.png)|*.png"));
SuccessCommand.Where(_ => plotModel != null).Subscribe(_ => ExportPlotModel(plotModel));
}
public async Task TestTripCircuitOnTimeoutsAboveThreshold()
{
var key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
var cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
var taskList = new List <Task>
{
cmd1.ExecuteAsync(),
cmd2.ExecuteAsync(),
cmd3.ExecuteAsync(),
cmd4.ExecuteAsync(),
cmd5.ExecuteAsync(),
cmd6.ExecuteAsync(),
cmd7.ExecuteAsync(),
cmd8.ExecuteAsync(),
cmd9.ExecuteAsync(),
};
await Task.WhenAll(taskList);
// Allow window to pass, this should trip the circuit as the error percentage is above the threshold
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void When_scheduler_is_restarted_Then_scheduled_jobs_still_get_executed()
{
var tasks = new[] { 0.5, 1, 1.5, 2, 2.5 };
foreach (var task in tasks)
{
var text = task.ToString(CultureInfo.InvariantCulture);
var testMessage = new SuccessCommand(text);
_scheduler.Tell(new ScheduleCommand(testMessage, new ScheduleKey(Guid.Empty, text, text), CreateOptions(task, Timeout)));
}
_quartzScheduler.Shutdown(false);
CreateScheduler();
var taskIds = tasks.Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray();
Throttle.Assert(() => ResultHolder.Contains(taskIds));
}
public void TestTripCircuit()
{
String key = "cmd-A";
try
{
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
cmd1.Execute();
cmd2.Execute();
cmd3.Execute();
cmd4.Execute();
// this should still allow requests as everything has been successful
Time.Wait(150);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// fail
HystrixCommand <bool> cmd5 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd7 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd8 = new FailureCommand(key, 1);
Assert.False(cmd5.Execute());
Assert.False(cmd6.Execute());
Assert.False(cmd7.Execute());
Assert.False(cmd8.Execute());
// everything has failed in the test window so we should return false now
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void Legacy_wire_data_can_run_with_latest_job_code()
{
ScheduleKey key = new ScheduleKey(Guid.NewGuid(), Name, Group);
Command command = new SuccessCommand("1232");
ExecutionOptions executionOptions = new ExecutionOptions(DateTime.Now.AddSeconds(1), typeof(ScheduledCommandSuccessfullyProcessed));
var serializedCommand = SerializeAsLegacy(command);
var serializedKey = SerializeAsLegacy(key);
var serializedOptions = SerializeAsLegacy(executionOptions);
var jobDataMap = new JobDataMap
{
{ QuartzJob.CommandKey, serializedCommand },
{ QuartzJob.ScheduleKey, serializedKey },
{ QuartzJob.ExecutionOptionsKey, serializedOptions }
};
var legacyJob = QuartzJob.CreateJob(key, jobDataMap);
var listener = new CallbackJobListener();
_quartzScheduler.ListenerManager.AddJobListener(listener, KeyMatcher <JobKey> .KeyEquals(legacyJob.Key));
var task = listener.TaskFinish.Task;
var trigger = TriggerBuilder.Create()
.WithIdentity(legacyJob.Key.Name, legacyJob.Key.Group)
.WithSimpleSchedule(x => x.WithMisfireHandlingInstructionFireNow()
.WithRepeatCount(0))
.StartAt(DateTimeOffset.Now.AddMilliseconds(200))
.Build();
_quartzScheduler.ScheduleJob(legacyJob, trigger);
if (!task.Wait(TimeSpan.FromSeconds(10000)))
{
Assert.Fail("Job execution timed out");
}
if (task.Result.Item2 != null)
{
Assert.Fail("Job threw an exception", task.Result.Item2);
}
}
public void TestTripCircuitOnTimeoutsAboveThreshold()
{
string key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
cmd5.Execute();
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd6.Execute();
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
cmd8.Execute();
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
cmd9.Execute();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
public void TestTripCircuitOnTimeoutsAboveThreshold()
{
String key = "cmd-E";
try
{
HystrixCommand <Boolean> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <Boolean> cmd5 = new TimeoutCommand(key);
cmd5.Execute();
HystrixCommand <Boolean> cmd6 = new TimeoutCommand(key);
cmd6.Execute();
HystrixCommand <Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <Boolean> cmd8 = new TimeoutCommand(key);
cmd8.Execute();
HystrixCommand <Boolean> cmd9 = new TimeoutCommand(key);
cmd9.Execute();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void TestCircuitDoesNotTripOnFailuresBelowThreshold()
{
String key = "cmd-C";
try
{
HystrixCommand <Boolean> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <Boolean> cmd5 = new SuccessCommand(key, 1);
cmd5.Execute();
HystrixCommand <Boolean> cmd6 = new FailureCommand(key, 1);
cmd6.Execute();
HystrixCommand <Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <Boolean> cmd8 = new FailureCommand(key, 1);
cmd8.Execute();
// this should remain closed as the failure threshold is below the percentage limit
Time.Wait(150);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public async Task TestCircuitDoesNotTripOnFailuresBelowThreshold()
{
string key = "cmd-C";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
await cmd4.ExecuteAsync();
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 1);
await cmd5.ExecuteAsync();
HystrixCommand <bool> cmd6 = new FailureCommand(key, 1);
await cmd6.ExecuteAsync();
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
await cmd7.ExecuteAsync();
HystrixCommand <bool> cmd8 = new FailureCommand(key, 1);
await cmd8.ExecuteAsync();
// this should remain closed as the failure threshold is below the percentage limit
Time.Wait(150);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public async Task TestTripCircuitAsync()
{
string key = "cmd-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
_ = await cmd1.ExecuteAsync();
_ = await cmd2.ExecuteAsync();
_ = await cmd3.ExecuteAsync();
_ = await cmd4.ExecuteAsync();
// this should still allow requests as everything has been successful
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// fail
HystrixCommand <bool> cmd5 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd7 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd8 = new FailureCommand(key, 0);
Assert.False(await cmd5.ExecuteAsync());
Assert.False(await cmd6.ExecuteAsync());
Assert.False(await cmd7.ExecuteAsync());
Assert.False(await cmd8.ExecuteAsync());
// everything has failed in the test window so we should return false now
Time.Wait(300);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void TestTripCircuit()
{
String key = "cmd-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
cmd1.Execute();
cmd2.Execute();
cmd3.Execute();
cmd4.Execute();
// this should still allow requests as everything has been successful
Time.Wait(150);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// fail
HystrixCommand <bool> cmd5 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd7 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd8 = new FailureCommand(key, 1);
Assert.False(cmd5.Execute());
Assert.False(cmd6.Execute());
Assert.False(cmd7.Execute());
Assert.False(cmd8.Execute());
// everything has failed in the test window so we should return false now
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
public void TestGetErrorPercentage()
{
string key = "cmd-metrics-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
HystrixCommandMetrics metrics = cmd1._metrics;
Assert.True(WaitForHealthCountToUpdate(key, 1000), "Health count stream took to long");
cmd1.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
cmd2.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
cmd3.Execute();
cmd4.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(25, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd5.Execute();
cmd6.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd8 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd9 = new SuccessCommand(key, 0);
cmd7.Execute();
cmd8.Execute();
cmd9.Execute();
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// latent
HystrixCommand <bool> cmd10 = new SuccessCommand(key, 60);
cmd10.Execute();
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
Assert.Equal(30, metrics.Healthcounts.ErrorPercentage);
}
public async Task TestMultipleTimeWindowRetriesBeforeClosingCircuit()
{
var key = "cmd-H";
var sleepWindow = 400;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0);
var cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed when expected!");
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
_ = await cmd4.ExecuteAsync();
// everything has failed in the test window so we should return false now
// Allow window to pass,
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine(Time.CurrentTimeMillis + " !!!! 1 4 failures, circuit will open on recalc");
// Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// wait for sleepWindow to pass
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2 Sleep window starting where all commands fail-fast");
Time.Wait(sleepWindow + 50);
output.WriteLine(Time.CurrentTimeMillis + " !!!! 3 Sleep window over, should allow singleTest()");
// but the circuit should still be open
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd5 = new FailureCommand(key, 50);
var asyncResult5 = cmd5.Observe();
output.WriteLine(Time.CurrentTimeMillis + " !!!! Kicked off the single-test");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
output.WriteLine(Time.CurrentTimeMillis + " !!!! Confirmed that no other requests go out during single-test");
await asyncResult5.SingleAsync();
output.WriteLine(Time.CurrentTimeMillis + " !!!! SingleTest just completed");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!");
// wait for sleepWindow to pass
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window START");
Time.Wait(sleepWindow + 50);
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window over");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd6 = new FailureCommand(key, 50);
var asyncResult6 = cmd6.Observe();
output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest just kicked off");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
Assert.False(await asyncResult6.SingleAsync());
output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest now over");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow);
// but the circuit should still be open
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 50);
var asyncResult7 = cmd7.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
await asyncResult7.SingleAsync();
// all requests should be open again
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (1) when expected!");
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (2) when expected!");
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (3) when expected!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
}
