public RollingCollapserBatchSizeDistributionStreamTest(ITestOutputHelper output)
: base()
{
this.output = output;
RollingCollapserBatchSizeDistributionStream.Reset();
HystrixCollapserEventStream.Reset();
}
public void Before()
{
context = HystrixRequestContext.InitializeContext();
HystrixCommandMetrics.Reset();
HystrixThreadPoolMetrics.Reset();
HystrixCollapserMetrics.Reset();
// clear collapsers
RequestCollapserFactory.Reset();
// clear circuit breakers
HystrixCircuitBreakerFactory.Reset();
HystrixPlugins.Reset();
HystrixOptionsFactory.Reset();
// clear up all streams
CumulativeCollapserEventCounterStream.Reset();
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-A");
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Skip(10).Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
Assert.Equal(0, distribution.GetTotalCount());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.Latest.GetTotalCount());
}
#pragma warning restore S1199 // Nested code blocks should not be used
internal static void Reset()
{
RollingCollapserEventCounterStream.Reset();
CumulativeCollapserEventCounterStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
Metrics.Clear();
}
public override void Dispose()
{
latchSubscription?.Dispose();
stream?.Unsubscribe();
latchSubscription = null;
stream = null;
base.Dispose();
}
HystrixCollapserMetrics(IHystrixCollapserKey key, IHystrixCollapserOptions properties) : base(null)
{
this.collapserKey = key;
this.properties = properties;
rollingCollapserEventCounterStream = RollingCollapserEventCounterStream.GetInstance(key, properties);
cumulativeCollapserEventCounterStream = CumulativeCollapserEventCounterStream.GetInstance(key, properties);
rollingCollapserBatchSizeDistributionStream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, properties);
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-A");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public static void ShutdownThreads()
{
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
HealthCountsStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestBatches()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-B");
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Collapser.From(output, key, 1).Observe();
Collapser.From(output, key, 2).Observe();
Collapser.From(output, key, 3).Observe();
try
{
Time.Wait(250);
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Collapser.From(output, key, 4).Observe();
try
{
Time.Wait(250);
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Collapser.From(output, key, 5).Observe();
Collapser.From(output, key, 6).Observe();
Collapser.From(output, key, 7).Observe();
Collapser.From(output, key, 8).Observe();
Collapser.From(output, key, 9).Observe();
try
{
Time.Wait(250);
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Collapser.From(output, key, 10).Observe();
Collapser.From(output, key, 11).Observe();
Collapser.From(output, key, 12).Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
// should have 4 batches: 3, 1, 5, 3
Assert.Equal(4, stream.Latest.GetTotalCount());
Assert.Equal(3, stream.LatestMean);
Assert.Equal(1, stream.GetLatestPercentile(0));
Assert.Equal(5, stream.GetLatestPercentile(100));
}
public void TestBatches()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// First collapser created with key will be used for all command creations
List <Task> tasks = new List <Task>();
var c1 = Collapser.From(output, key, 1);
tasks.Add(c1.ExecuteAsync());
var c2 = Collapser.From(output, key, 2);
tasks.Add(c2.ExecuteAsync());
var c3 = Collapser.From(output, key, 3);
tasks.Add(c3.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 1 too long to start");
c1.CommandCreated = false;
var c4 = Collapser.From(output, key, 4);
tasks.Add(c4.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 2 too long to start");
c1.CommandCreated = false;
var c5 = Collapser.From(output, key, 5);
tasks.Add(c5.ExecuteAsync());
var c6 = Collapser.From(output, key, 6);
tasks.Add(c6.ExecuteAsync());
var c7 = Collapser.From(output, key, 7);
tasks.Add(c7.ExecuteAsync());
var c8 = Collapser.From(output, key, 8);
tasks.Add(c8.ExecuteAsync());
var c9 = Collapser.From(output, key, 9);
tasks.Add(c9.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 3 too long to start");
c1.CommandCreated = false;
var c10 = Collapser.From(output, key, 10);
tasks.Add(c10.ExecuteAsync());
var c11 = Collapser.From(output, key, 11);
tasks.Add(c11.ExecuteAsync());
var c12 = Collapser.From(output, key, 12);
tasks.Add(c12.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 4 too long to start");
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
// should have 4 batches: 3, 1, 5, 3
Assert.Equal(4, stream.Latest.GetTotalCount());
Assert.Equal(3, stream.LatestMean);
Assert.Equal(1, stream.GetLatestPercentile(0));
Assert.Equal(5, stream.GetLatestPercentile(100));
}
public void TestBatchesAgeOut()
{
IHystrixCollapserKey key = HystrixCollapserKeyDefault.AsKey("Collapser-Batch-Size-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCollapserBatchSizeDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// First collapser created with key will be used for all command creations
List <Task> tasks = new List <Task>();
var c1 = Collapser.From(output, key, 1);
tasks.Add(c1.ExecuteAsync());
var c2 = Collapser.From(output, key, 2);
tasks.Add(c2.ExecuteAsync());
var c3 = Collapser.From(output, key, 3);
tasks.Add(c3.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 1 too long to start");
c1.CommandCreated = false;
var c4 = Collapser.From(output, key, 4);
tasks.Add(c4.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 2 too long to start");
c1.CommandCreated = false;
var c5 = Collapser.From(output, key, 5);
tasks.Add(c5.ExecuteAsync());
var c6 = Collapser.From(output, key, 6);
tasks.Add(c6.ExecuteAsync());
var c7 = Collapser.From(output, key, 7);
tasks.Add(c7.ExecuteAsync());
var c8 = Collapser.From(output, key, 8);
tasks.Add(c8.ExecuteAsync());
var c9 = Collapser.From(output, key, 9);
tasks.Add(c9.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 3 too long to start");
c1.CommandCreated = false;
var c10 = Collapser.From(output, key, 10);
tasks.Add(c10.ExecuteAsync());
var c11 = Collapser.From(output, key, 11);
tasks.Add(c11.ExecuteAsync());
var c12 = Collapser.From(output, key, 12);
tasks.Add(c12.ExecuteAsync());
Assert.True(Time.WaitUntil(() => c1.CommandCreated, 500), "Batch 4 too long to start");
Task.WaitAll(tasks.ToArray());
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.Latest.GetTotalCount());
Assert.Equal(0, stream.LatestMean);
}
