public void ToJsonList_ReturnsExpected()
{
var stream = HystrixDashboardStream.GetInstance();
CountdownEvent latch = new CountdownEvent(1);
List <string> result = null;
var subscription = stream.Observe()
.SubscribeOn(NewThreadScheduler.Default)
.ObserveOn(NewThreadScheduler.Default)
.Subscribe(
(data) =>
{
result = Serialize.ToJsonList(data, null);
if (result.Count > 0)
{
latch.SignalEx();
}
},
(e) =>
{
latch.SignalEx();
},
() =>
{
latch.SignalEx();
});
MyCommand cmd = new MyCommand();
cmd.Execute();
latch.Wait(10000);
Assert.NotNull(result);
Assert.True(result.Count > 0);
var jsonObject = result[0];
var dict = JsonConvert.DeserializeObject <Dictionary <string, object> >(jsonObject);
Assert.NotNull(dict);
Assert.NotNull(dict["origin"]);
Assert.NotNull(dict["data"]);
JObject cmdData = (JObject)dict["data"];
Assert.NotNull(cmdData["type"]);
var type = cmdData["type"].Value <string>();
Assert.True("HystrixCommand".Equals(type) || "HystrixThreadPool".Equals(type));
Assert.NotNull(cmdData["name"]);
var name = cmdData["name"].Value <string>();;
Assert.True("MyCommand".Equals(name) || "MyCommandGroup".Equals(name));
subscription.Dispose();
}
public void TestTwoSubscribersOneSlowOneFast()
{
CountdownEvent latch = new CountdownEvent(1);
AtomicBoolean foundError = new AtomicBoolean(false);
IObservable <HystrixDashboardStream.DashboardData> fast = stream
.Observe()
.ObserveOn(NewThreadScheduler.Default);
IObservable <HystrixDashboardStream.DashboardData> slow = stream
.Observe()
.ObserveOn(NewThreadScheduler.Default)
.Map((n) =>
{
try
{
Time.Wait(100);
return(n);
}
catch (Exception)
{
return(n);
}
});
IObservable <bool> checkZippedEqual = Observable.Zip(fast, slow, (payload, payload2) =>
{
return(payload == payload2);
});
IDisposable s1 = checkZippedEqual
.Take(10000)
.Subscribe(
(b) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : OnNext : " + b);
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : OnError : " + e);
output.WriteLine(e.ToString());
foundError.Value = true;
latch.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : OnCompleted");
latch.SignalEx();
});
for (int i = 0; i < 50; i++)
{
HystrixCommand <int> cmd = Command.From(GroupKey, CommandKey, HystrixEventType.SUCCESS, 50);
cmd.Execute();
}
latch.Wait(10000);
Assert.False(foundError.Value);
s1.Dispose();
}
public void TestSharedSourceStream()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Health-N");
stream = HealthCountsStream.GetInstance(key, 10, 100);
var latch = new CountdownEvent(1);
var allEqual = new AtomicBoolean(false);
var o1 = stream
.Observe()
.Take(10)
.ObserveOn(TaskPoolScheduler.Default);
var o2 = stream
.Observe()
.Take(10)
.ObserveOn(TaskPoolScheduler.Default);
var zipped = Observable.Zip(o1, o2, (healthCounts, healthCounts2) =>
{
return(healthCounts == healthCounts2); // we want object equality
});
var reduced = zipped.Aggregate(true, (a, b) =>
{
return(a && b);
}).Select(n => n);
var rdisp = reduced.Subscribe(
(b) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " Reduced OnNext : " + b);
allEqual.Value = b;
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " Reduced OnError : " + e);
output.WriteLine(e.ToString());
latch.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " Reduced OnCompleted");
latch.SignalEx();
});
for (var i = 0; i < 10; i++)
{
HystrixCommand <int> cmd = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
cmd.Execute();
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.True(allEqual.Value);
rdisp.Dispose();
// we should be getting the same object from both streams. this ensures that multiple subscribers don't induce extra work
}
protected void AssertNonBlockingObserve(C command, Action <C> assertion, bool isSuccess)
{
//output.WriteLine("Running command.observe(), awaiting terminal state of Observable, then running assertions...");
CountdownEvent latch = new CountdownEvent(1);
IObservable <int> o = command.Observe();
o.Subscribe(
(n) =>
{
},
(e) =>
{
latch.SignalEx();
},
() =>
{
latch.SignalEx();
});
try
{
latch.Wait(3000);
assertion(command);
}
catch (Exception)
{
throw;
}
if (isSuccess)
{
try
{
o.ToList().Single();
}
catch (Exception)
{
throw;
}
}
else
{
try
{
o.ToList().Single();
Assert.True(false, "Expected a command failure!");
}
catch (Exception)
{
//output.WriteLine("Received expected ex : " + ex);
// ex.printStackTrace();
}
}
}
public void TestCurrentConcurrentExecutionCount()
{
String key = "cmd-metrics-C";
HystrixCommandMetrics metrics = null;
List <IObservable <bool> > cmdResults = new List <IObservable <bool> >();
int NUM_CMDS = 8;
for (int i = 0; i < NUM_CMDS; i++)
{
HystrixCommand <Boolean> cmd = new SuccessCommand(key, 900);
if (metrics == null)
{
metrics = cmd.metrics;
}
IObservable <bool> eagerObservable = cmd.Observe();
cmdResults.Add(eagerObservable);
}
try
{
Time.Wait(200);
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
output.WriteLine("ReqLog: " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(NUM_CMDS, metrics.CurrentConcurrentExecutionCount);
CountdownEvent latch = new CountdownEvent(1);
Observable.Merge(cmdResults).Subscribe(
(n) =>
{
},
(e) =>
{
output.WriteLine("Error duing command execution");
output.WriteLine(e.ToString());
latch.SignalEx();
},
() =>
{
output.WriteLine("All commands done");
latch.SignalEx();
});
latch.Wait(10000);
Assert.Equal(0, metrics.CurrentConcurrentExecutionCount);
}
protected void AssertNonBlockingObserve(C command, Action <C> assertion, bool isSuccess)
{
CountdownEvent latch = new CountdownEvent(1);
IObservable <int> o = command.Observe();
o.Subscribe(
(n) =>
{
},
(e) =>
{
latch.SignalEx();
},
() =>
{
latch.SignalEx();
});
try
{
latch.Wait(3000);
assertion(command);
}
catch (Exception)
{
throw;
}
if (isSuccess)
{
try
{
o.ToList().SingleAsync().Wait();
}
catch (Exception)
{
throw;
}
}
else
{
try
{
o.ToList().SingleAsync().Wait();
Assert.True(false, "Expected a command failure!");
}
catch (Exception)
{
}
}
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-A");
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Skip(10).Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
Assert.Equal(0, distribution.GetTotalCount());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public void TestThreadPoolRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-E");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 10 commands with latency should occupy the entire threadpool. execute those, then wait for bucket to roll
// next command should be a thread-pool rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 200));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() > 0)
{
AssertBetween(200, 250, distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Command threadPoolRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(40);
threadPoolRejected.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(200, 250, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(threadPoolRejected.IsResponseThreadPoolRejected, "Response NOT ThreadPoolRejected as expected");
}
public void TestMultipleBucketsBothGetStored()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-H");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() == 2)
{
AssertBetween(55, 90, distribution.GetMean());
}
if (distribution.GetTotalCount() == 5)
{
AssertBetween(60, 90, distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 100);
cmd1.Observe();
cmd2.Observe();
try
{
Time.Wait(500);
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Command cmd3 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
Command cmd4 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
Command cmd5 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 70);
cmd3.Observe();
cmd4.Observe();
cmd5.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
AssertBetween(55, 90, stream.LatestMean);
AssertBetween(10, 50, stream.GetLatestPercentile(0.0));
AssertBetween(100, 150, stream.GetLatestPercentile(100.0));
}
public void TestShortCircuitedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-D");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 3 failures is enough to trigger short-circuit. execute those, then wait for bucket to roll
// next command should be a short-circuit
List <Command> commands = new List <Command>();
for (int i = 0; i < 3; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.FAILURE, 0));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
AssertBetween(0, 30, distribution.GetMean());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Command shortCircuit = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(200);
shortCircuit.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(3, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(shortCircuit.IsResponseShortCircuited);
}
public void TestSingleBucketWithMultipleEventTypes()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-C");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + DateTime.Now.Ticks / 10000 + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() < 4 && distribution.GetTotalCount() > 0)
{ //buckets before timeout latency registers
AssertBetween(10, 50, (int)distribution.GetMean());
}
else if (distribution.GetTotalCount() == 4)
{
AssertBetween(150, 250, (int)distribution.GetMean()); //now timeout latency of 600ms is there
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.TIMEOUT); //latency = 600
Command cmd3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 30);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST, 40);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
cmd4.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
AssertBetween(150, 350, stream.LatestMean); //now timeout latency of 600ms is there
AssertBetween(10, 40, stream.GetLatestPercentile(0.0));
AssertBetween(600, 800, stream.GetLatestPercentile(100.0));
}
public void TestStreamHasData()
{
AtomicBoolean commandShowsUp = new AtomicBoolean(false);
AtomicBoolean threadPoolShowsUp = new AtomicBoolean(false);
CountdownEvent latch = new CountdownEvent(1);
int num = 10;
for (int i = 0; i < 2; i++)
{
HystrixCommand <int> cmd = Command.From(GroupKey, CommandKey, HystrixEventType.SUCCESS, 50);
cmd.Observe();
}
stream.Observe().Take(num).Subscribe(
(configuration) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Received data with : " + configuration.CommandConfig.Count + " commands");
if (configuration.CommandConfig.ContainsKey(CommandKey))
{
commandShowsUp.Value = true;
}
if (configuration.ThreadPoolConfig.Count != 0)
{
threadPoolShowsUp.Value = true;
}
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " OnError : " + e);
latch.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " OnCompleted");
latch.SignalEx();
});
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.True(commandShowsUp.Value);
Assert.True(threadPoolShowsUp.Value);
}
public void TestStreamHasData()
{
AtomicBoolean commandShowsUp = new AtomicBoolean(false);
AtomicBoolean threadPoolShowsUp = new AtomicBoolean(false);
CountdownEvent latch = new CountdownEvent(1);
int NUM = 10;
for (int i = 0; i < 2; i++)
{
HystrixCommand <int> cmd = Command.From(groupKey, commandKey, HystrixEventType.SUCCESS, 50);
cmd.Observe();
}
stream.Observe().Take(NUM).Subscribe(
(utilization) =>
{
output.WriteLine(DateTime.Now.Ticks / 10000 + " : Received data with : " + " : Received data with : " + utilization.CommandUtilizationMap.Count + " commands");
if (utilization.CommandUtilizationMap.ContainsKey(commandKey))
{
commandShowsUp.Value = true;
}
if (utilization.ThreadPoolUtilizationMap.Count != 0)
{
threadPoolShowsUp.Value = true;
}
},
(e) =>
{
output.WriteLine(DateTime.Now.Ticks / 10000 + " : " + Thread.CurrentThread.ManagedThreadId + " OnError : " + e);
latch.SignalEx();
},
() =>
{
output.WriteLine(DateTime.Now.Ticks / 10000 + " : " + Thread.CurrentThread.ManagedThreadId + " OnCompleted");
latch.SignalEx();
});
Assert.True(latch.Wait(10000));
Assert.True(commandShowsUp.Value);
Assert.True(threadPoolShowsUp.Value);
}
public void TestThreadSafety()
{
var time = new MockedTime();
var p = new HystrixRollingPercentile(time, 100, 25, 1000, true);
int num_threads = 1000; // .NET Core StackOverflow
int num_iterations = 1000000;
var latch = new CountdownEvent(num_threads);
var aggregateMetrics = new AtomicInteger(); // same as a blackhole
var r = new Random();
var cts = new CancellationTokenSource();
var metricsPoller = Task.Run(() =>
{
while (!cts.Token.IsCancellationRequested)
{
aggregateMetrics.AddAndGet(p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
}
});
for (int i = 0; i < num_threads; i++)
{
int threadId = i;
Task.Run(() =>
{
for (int j = 1; j < (num_iterations / num_threads) + 1; j++)
{
int nextInt = r.Next(100);
p.AddValue(nextInt);
if (threadId == 0)
{
time.Increment(1);
}
}
latch.SignalEx();
});
}
try
{
latch.Wait(TimeSpan.FromSeconds(100));
cts.Cancel();
}
catch (Exception)
{
Assert.True(false, "Timeout on all threads writing percentiles");
}
aggregateMetrics.AddAndGet(p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
output.WriteLine(p.Mean + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(75) + " : " + p.GetPercentile(90) + " : " + p.GetPercentile(95) + " : " + p.GetPercentile(99));
}
public void TestSingleBucketGetsStored()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-B");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() == 1)
{
AssertBetween(10, 50, (int)distribution.GetMean());
}
else if (distribution.GetTotalCount() == 2)
{
AssertBetween(300, 400, (int)distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.TIMEOUT); // latency = 600
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
catch (Exception)
{
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(false, "Interrupted ex");
}
AssertBetween(150, 400, stream.LatestMean);
AssertBetween(10, 50, stream.GetLatestPercentile(0.0));
AssertBetween(300, 800, stream.GetLatestPercentile(100.0));
}
public void TestResponseFromCacheDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-G");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// should get 1 SUCCESS and 1 RESPONSE_FROM_CACHE
List <Command> commands = Command.GetCommandsWithResponseFromCache(GroupKey, key);
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
Assert.True(distribution.GetTotalCount() <= 1);
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(1, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public void TestWriteThreadSafety()
{
var time = new MockedTime();
var p = new HystrixRollingPercentile(time, 100, 25, 1000, true);
int num_threads = 10;
int num_iterations = 1000;
var latch = new CountdownEvent(num_threads);
var r = new Random();
var added = new AtomicInteger(0);
for (int i = 0; i < num_threads; i++)
{
var t = new Task(
() =>
{
for (int j = 1; j < (num_iterations / num_threads) + 1; j++)
{
int nextInt = r.Next(100);
p.AddValue(nextInt);
added.GetAndIncrement();
}
latch.SignalEx();
},
CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
try
{
latch.Wait(TimeSpan.FromSeconds(100));
Assert.Equal(added.Value, p._buckets.PeekLast._data.Length);
}
catch (Exception)
{
Assert.True(false, "Timeout on all threads writing percentiles");
}
}
public void TestStreamHasData()
{
AtomicBoolean commandShowsUp = new AtomicBoolean(false);
CountdownEvent latch = new CountdownEvent(1);
int NUM = 10;
for (int i = 0; i < 2; i++)
{
HystrixCommand <int> cmd = Command.From(groupKey, commandKey, HystrixEventType.SUCCESS, 50);
cmd.Observe();
}
stream.Observe().Take(NUM).Subscribe(
(dashboardData) =>
{
output.WriteLine(DateTime.Now.Ticks / 10000 + " : Received data with : " + dashboardData.commandMetrics.Count + " commands");
foreach (HystrixCommandMetrics metrics in dashboardData.commandMetrics)
{
if (metrics.CommandKey.Equals(commandKey))
{
commandShowsUp.Value = true;
}
}
},
(e) =>
{
output.WriteLine(DateTime.Now.Ticks / 10000 + " : " + Thread.CurrentThread.ManagedThreadId + " OnError : " + e);
},
() =>
{
output.WriteLine(DateTime.Now.Ticks / 10000 + " : " + Thread.CurrentThread.ManagedThreadId + " OnCompleted");
latch.SignalEx();
});
Assert.True(latch.Wait(10000));
Assert.True(commandShowsUp.Value);
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-A");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + DateTime.Now.Ticks / 10000 + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + Thread.CurrentThread.ManagedThreadId);
Assert.Equal(0, distribution.GetTotalCount());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public void TestStreamHasData()
{
var commandShowsUp = new AtomicBoolean(false);
var latch = new CountdownEvent(1);
var num = 10;
for (var i = 0; i < 2; i++)
{
HystrixCommand <int> cmd = Command.From(GroupKey, CommandKey, HystrixEventType.SUCCESS, 50);
cmd.Observe();
}
stream.Observe().Take(num).Subscribe(
(dashboardData) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : Received data with : " + dashboardData.CommandMetrics.Count + " commands");
foreach (var metrics in dashboardData.CommandMetrics)
{
if (metrics.CommandKey.Equals(CommandKey))
{
commandShowsUp.Value = true;
}
}
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " OnError : " + e);
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " OnCompleted");
latch.SignalEx();
});
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.True(commandShowsUp.Value);
}
protected override void OnCompletedCore()
{
output.WriteLine("OnCompletedCore @ " + DateTime.Now.Ticks / 10000 + " : " + Thread.CurrentThread.ManagedThreadId);
latch.SignalEx();
}
public void TestTwoSubscribersOneUnsubscribes()
{
CountdownEvent latch1 = new CountdownEvent(1);
CountdownEvent latch2 = new CountdownEvent(1);
AtomicInteger payloads1 = new AtomicInteger(0);
AtomicInteger payloads2 = new AtomicInteger(0);
IDisposable s1 = stream
.Observe()
.Take(100)
.OnDispose(() =>
{
latch1.SignalEx();
})
.Subscribe(
(dashboardData) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 1 OnNext : " + dashboardData);
payloads1.IncrementAndGet();
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 1 OnError : " + e);
latch1.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 1 OnCompleted");
latch1.SignalEx();
});
IDisposable s2 = stream
.Observe()
.Take(100)
.OnDispose(() =>
{
latch2.SignalEx();
})
.Subscribe(
(dashboardData) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 2 OnNext : " + dashboardData);
payloads2.IncrementAndGet();
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 2 OnError : " + e);
latch2.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 2 OnCompleted");
latch2.SignalEx();
});
// execute 1 command, then unsubscribe from first stream. then execute the rest
for (int i = 0; i < 50; i++)
{
HystrixCommand <int> cmd = Command.From(GroupKey, CommandKey, HystrixEventType.SUCCESS, 50);
cmd.Execute();
if (i == 1)
{
s1.Dispose();
}
}
Assert.True(latch1.Wait(10000));
Assert.True(latch2.Wait(10000));
output.WriteLine("s1 got : " + payloads1.Value + ", s2 got : " + payloads2.Value);
Assert.True(payloads1.Value > 0); // "s1 got data"
Assert.True(payloads2.Value > 0); // "s2 got data"
Assert.True(payloads2.Value > payloads1.Value); // "s1 got less data than s2",
}
public void TestSemaphoreRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-F");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 10 commands with latency should occupy all semaphores. execute those, then wait for bucket to roll
// next command should be a semaphore rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 200, ExecutionIsolationStrategy.SEMAPHORE));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() > 0)
{
AssertBetween(200, 250, (int)distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
Task t = new Task(
() =>
{
cmd.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
Command semaphoreRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(40);
semaphoreRejected.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(200, 250, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(semaphoreRejected.IsResponseSemaphoreRejected);
}
public void TestTwoSubscribersBothUnsubscribe()
{
CountdownEvent latch1 = new CountdownEvent(1);
CountdownEvent latch2 = new CountdownEvent(1);
AtomicInteger payloads1 = new AtomicInteger(0);
AtomicInteger payloads2 = new AtomicInteger(0);
IDisposable s1 = stream
.Observe()
.Take(10)
.OnDispose(() =>
{
latch1.SignalEx();
})
.Subscribe(
(dashboardData) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 1 OnNext : " + dashboardData);
payloads1.IncrementAndGet();
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 1 OnError : " + e);
latch1.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 1 OnCompleted");
latch1.SignalEx();
});
IDisposable s2 = stream
.Observe()
.Take(10)
.OnDispose(() =>
{
latch2.SignalEx();
})
.Subscribe(
(dashboardData) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 2 OnNext : " + dashboardData);
payloads2.IncrementAndGet();
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 2 OnError : " + e);
latch2.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Dashboard 2 OnCompleted");
latch2.SignalEx();
});
// execute half the commands, then unsubscribe from both streams, then execute the rest
for (int i = 0; i < 50; i++)
{
HystrixCommand <int> cmd = Command.From(GroupKey, CommandKey, HystrixEventType.SUCCESS, 50);
cmd.Execute();
if (i == 25)
{
s1.Dispose();
s2.Dispose();
}
}
Assert.False(stream.IsSourceCurrentlySubscribed); // both subscriptions have been cancelled - source should be too
Assert.True(latch1.Wait(10000));
Assert.True(latch2.Wait(10000));
output.WriteLine("s1 got : " + payloads1.Value + ", s2 got : " + payloads2.Value);
Assert.True(payloads1.Value > 0); // "s1 got data",
Assert.True(payloads2.Value > 0); // "s2 got data",
}
public void TestTwoSubscribersOneUnsubscribes()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-O");
stream = HealthCountsStream.GetInstance(key, 10, 100);
CountdownEvent latch1 = new CountdownEvent(1);
CountdownEvent latch2 = new CountdownEvent(1);
AtomicInteger healthCounts1 = new AtomicInteger(0);
AtomicInteger healthCounts2 = new AtomicInteger(0);
IDisposable s1 = stream
.Observe()
.Take(10)
.ObserveOn(TaskPoolScheduler.Default)
.Finally(() =>
{
latch1.SignalEx();
})
.Subscribe(
(healthCounts) =>
{
output.WriteLine((DateTime.Now.Ticks / 10000) + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 1 OnNext : " + healthCounts);
healthCounts1.IncrementAndGet();
},
(e) =>
{
output.WriteLine((DateTime.Now.Ticks / 10000) + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 1 OnError : " + e);
latch1.SignalEx();
},
() =>
{
output.WriteLine((DateTime.Now.Ticks / 10000) + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 1 OnCompleted");
latch1.SignalEx();
});
IDisposable s2 = stream
.Observe()
.Take(10)
.ObserveOn(TaskPoolScheduler.Default)
.Finally(() =>
{
latch2.SignalEx();
})
.Subscribe(
(healthCounts) =>
{
output.WriteLine((DateTime.Now.Ticks / 10000) + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 2 OnNext : " + healthCounts + " : " + healthCounts2.Value);
healthCounts2.IncrementAndGet();
},
(e) =>
{
output.WriteLine((DateTime.Now.Ticks / 10000) + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 2 OnError : " + e);
latch2.SignalEx();
},
() =>
{
output.WriteLine((DateTime.Now.Ticks / 10000) + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 2 OnCompleted");
latch2.SignalEx();
});
// execute 5 commands, then unsubscribe from first stream. then execute the rest
for (int i = 0; i < 10; i++)
{
HystrixCommand <int> cmd = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
cmd.Execute();
if (i == 5)
{
s1.Dispose();
}
}
Assert.True(stream.IsSourceCurrentlySubscribed); // only 1/2 subscriptions has been cancelled
Assert.True(latch1.Wait(10000));
Assert.True(latch2.Wait(10000));
output.WriteLine("s1 got : " + healthCounts1.Value + ", s2 got : " + healthCounts2.Value);
Assert.True(healthCounts1.Value > 0);
Assert.True(healthCounts2.Value > 0);
Assert.True(healthCounts2.Value > healthCounts1.Value);
}
protected override void OnCompletedCore()
{
output.WriteLine("OnCompleted @ " + (DateTime.Now.Ticks / 10000));
latch.SignalEx();
}
protected override void OnCompletedCore()
{
latch.SignalEx();
}
protected override void OnCompletedCore()
{
output?.WriteLine("OnComplete @ " + Time.CurrentTimeMillis + " :" + Thread.CurrentThread.ManagedThreadId);
StreamRunning = false;
latch.SignalEx();
}
public void TestBatches()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-B");
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Collapser.From(output, key, 1).Observe();
Collapser.From(output, key, 2).Observe();
Collapser.From(output, key, 3).Observe();
try
{
Time.Wait(250);
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Collapser.From(output, key, 4).Observe();
try
{
Time.Wait(250);
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Collapser.From(output, key, 5).Observe();
Collapser.From(output, key, 6).Observe();
Collapser.From(output, key, 7).Observe();
Collapser.From(output, key, 8).Observe();
Collapser.From(output, key, 9).Observe();
try
{
Time.Wait(250);
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Collapser.From(output, key, 10).Observe();
Collapser.From(output, key, 11).Observe();
Collapser.From(output, key, 12).Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
// should have 4 batches: 3, 1, 5, 3
Assert.Equal(4, stream.Latest.GetTotalCount());
Assert.Equal(3, stream.LatestMean);
Assert.Equal(1, stream.GetLatestPercentile(0));
Assert.Equal(5, stream.GetLatestPercentile(100));
}
