public void CounterAggregatorDoesNothingWithoutData()
{
var aggregator = new CounterAggregator(DefaultDimensions);
Assert.IsTrue(!aggregator.Samples.Any());
Assert.IsTrue(!aggregator.TimeMergedSamples.Any());
}
public void CounterAggregatorRejectsPerMachinePercentiles()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = new CounterQueryResponse
{
HttpResponseCode = 200,
Samples = new List <DataSample>
{
new DataSample
{
Name = "bob",
StartTime = DateTime.Now.ToMillisecondTimestamp(),
EndTime = DateTime.Now.ToMillisecondTimestamp(),
Dimensions =
new Dictionary <string, string> {
{ AnyDimensionName, "tacos" }
},
SampleType = DataSampleType.Percentile,
Percentile = 40,
PercentileValue = 11,
}
}
};
Assert.Throws <ArgumentException>(() => aggregator.AddMachineResponse(response));
}
public void CounterAggregatorFiltersQueryParametersProperly()
{
var aggregator = new CounterAggregator(DefaultDimensions);
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(null));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string>()));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "PERCENTILE", "50.0" }
}));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "percentile", "50.0" }
}));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "Percentile", "50.0" }
}));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "PErcenTILe", "50.0" }
}));
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "percentile", "MAX" }
}).Count());
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "percentile", "Average" }
}).Count());
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "tacos", "delicious" }
}).Count());
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "percentile", "i am a key tree!" }
}).Count());
}
public void CounterAggregatorCanSmashAllBucketsTogether()
{
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 10));
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
this.VerifySample(superSamples.First(), 10);
Assert.AreEqual(1, superSamples.Count());
}
public void CounterAggregatorMergesCaseInsenstively()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var now = DateTime.Now;
aggregator.AddMachineResponse(this.CreateResponse(now, 1, 0, 1, 1, "taco"));
aggregator.AddMachineResponse(this.CreateResponse(now, 1, 0, 1, 1, "TACO"));
Assert.AreEqual(1, aggregator.Samples.Count());
}
public void AddAggregator(CounterAggregator aggregator)
{
lock (Aggregators)
{
if (!Aggregators.Contains(aggregator))
{
aggregator.BeginAggregation(ObservableEvents);
Aggregators.Add(aggregator);
}
}
}
public void CounterAggregatorCanSmashAllBucketsTogether()
{
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 10));
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
this.VerifySample(superSamples.First(), 10);
Assert.AreEqual(1, superSamples.Count());
}
public void CounterAggregatorCanHandleRangesWhichDoNotOverlap()
{
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 5));
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now.AddDays(-1), 1, 1, 5));
Assert.AreEqual(10, aggregator.Samples.Count(item =>
{
this.VerifySample(item, 1);
return(true);
}));
}
public void CounterAggregatorCanHandleRangesWhichAlwaysOverlap()
{
var aggregator = new CounterAggregator(DefaultDimensions);
// 10 minute buckets, staggered by 2 minute increments. Should 100% overlap
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 10, 2, 10));
Assert.AreEqual(1, aggregator.Samples.Count(item =>
{
this.VerifySample(item, 10);
return(true);
}));
}
public void CounterAggregatorCanHandleSupersetRange()
{
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 9));
// super bucket
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now.AddHours(-1), 600, 10, 1));
Assert.AreEqual(1, aggregator.Samples.Count(item =>
{
this.VerifySample(item, 10);
return(true);
}));
}
public void CounterAggregatorCountsMachinesCorrectly()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = this.CreateResponse(DateTime.Now, 1, 1, 10);
const int MachineCount = 15;
for (int i = 0; i < MachineCount; i++)
{
aggregator.AddMachineResponse(response);
}
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
Assert.AreEqual(1, superSamples.Count());
Assert.AreEqual(MachineCount, (int)superSamples.First().MachineCount);
}
public CountersAutoDiscoverer(IObservable <LogEventArgs> observableEvents, Settings settings)
{
_observableEvents = observableEvents;
_settings = settings;
int cnt = _settings.Aggregators.Count;
for (int i = 0; i < cnt; i++)
{
CounterAggregator counterAggregator = _settings.Aggregators[i];
if (!_knownCounters.Contains(counterAggregator.CounterCategory + "\t" + counterAggregator.CounterName))
{
_knownCounters.Add(counterAggregator.CounterCategory + "\t" + counterAggregator.CounterName);
}
}
}
public void CounterAggregatorKeepsDifferentSamplesSeparate()
{
// this ensures split by works correctly
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 10, 5));
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
Assert.AreEqual(5, superSamples.Count());
superSamples.All(x =>
{
this.VerifySample(x, 10);
return(true);
});
}
public void CounterAggregatorCountsMachinesCorrectly()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = this.CreateResponse(DateTime.Now, 1, 1, 10);
const int MachineCount = 15;
for (int i = 0; i < MachineCount; i++)
{
aggregator.AddMachineResponse(response);
}
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
Assert.AreEqual(1, superSamples.Count());
Assert.AreEqual(MachineCount, (int)superSamples.First().MachineCount);
}
public void CounterAggregatorCalculatesPercentileAfterAggregation()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = new CounterQueryResponse
{
HttpResponseCode = 200,
Samples = new List <DataSample>
{
new DataSample
{
Name = "bob",
StartTime = DateTime.Now.ToMillisecondTimestamp(),
EndTime = DateTime.Now.ToMillisecondTimestamp(),
Dimensions =
new Dictionary <string, string> {
{ AnyDimensionName, "tacos" }
},
SampleType = DataSampleType.Histogram,
Histogram = new Dictionary <long, uint> {
{ 1, 1 }, { 2, 1 }, { 3, 1 }, { 4, 1 }, { 5, 1 }, { 6, 1 }, { 7, 1 }, { 8, 1 }, { 9, 1 }, { 10, 1 }
}
}
}
};
// by default, we do not apply percentile filtering
aggregator.AddMachineResponse(response);
var defaultValue = aggregator.Samples.First();
Assert.AreEqual(DataSampleType.Histogram, defaultValue.SampleType);
// now that the client asked for filtering, we calculate the 99.999% percentile (should be the max value from earlier)
aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "Percentile", "99.999" }
});
var aggregatedValue = aggregator.Samples.First();
Assert.AreEqual(DataSampleType.Percentile, aggregatedValue.SampleType);
Assert.AreEqual(10, aggregatedValue.PercentileValue);
}
/// <summary>
/// Merge the local Query request and the distributed tiered response into a single response
/// </summary>
private static CounterQueryResponse MergeResponses(IDictionary <string, string> queryParameters,
CounterQueryResponse localResponse, bool mergeTimeBuckets,
CounterQueryResponse distributedResponse)
{
// no distributed response, just use the local one
if (distributedResponse == null)
{
return(localResponse);
}
var sampleMerger = new CounterAggregator(queryParameters ?? new Dictionary <string, string>(0));
sampleMerger.AddMachineResponse(localResponse);
sampleMerger.AddMachineResponse(distributedResponse);
// response code from this server is:
// 200: *Anyone* has data
// 40X: *Everyone failed to get data, but everyone has the same response code
// 409: Mixed error codes ("Conflict" used liberally)
HttpStatusCode responseCode;
if ((localResponse.Samples != null && localResponse.Samples.Count > 0) ||
(distributedResponse.Samples != null && distributedResponse.Samples.Count > 0))
{
responseCode = HttpStatusCode.OK;
}
else
{
responseCode = distributedResponse.RequestDetails != null &&
distributedResponse.RequestDetails.Count > 0 &&
distributedResponse.RequestDetails.All(d => d.HttpResponseCode == (int)localResponse.HttpResponseCode)
? (HttpStatusCode)localResponse.HttpResponseCode
: HttpStatusCode.Conflict;
}
var mergedResponse = sampleMerger.GetResponse(mergeTimeBuckets);
mergedResponse.HttpResponseCode = (short)responseCode;
return(mergedResponse);
}
private void DiscoverBasedOnGenericCounters(LogEventArgs eventArgs)
{
int cnt = _settings.Aggregators.Count;
for (int i = 0; i < cnt; i++)
{
CounterAggregator counterAggregator = _settings.Aggregators[i];
if ((eventArgs.LogEvent.Category == counterAggregator.CounterCategory) &&
(eventArgs.LogEvent.Counter.StartsWith(counterAggregator.CounterName)) &&
(eventArgs.LogEvent.Counter.Length > counterAggregator.CounterName.Length))
{
CounterAggregator aggregator = new CounterAggregator(counterAggregator.CounterCategory,
eventArgs.LogEvent.Counter,
counterAggregator.AggregationType,
counterAggregator.InputType, "");
_settings.AddAggregator(aggregator);
File.AppendAllText(@"settings\counters.list",
"\r\n" + aggregator.CounterCategory + "\t" + aggregator.CounterName + "\t" +
aggregator.AggregationType.ToString() + "\t" + aggregator.InputType.FullName);
}
}
}
public void CounterAggregatorRollsUpMachineCountProperly()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = this.CreateResponse(DateTime.Now, 1, 1, 10);
const int ResponseCount = 15;
const int MachineCountPerResponse = 10;
// add multiple machines per response
response.Samples = response.Samples.Select(s =>
{
var originalValue = s;
return(new DataSample
{
Dimensions = originalValue.Dimensions,
EndTime = originalValue.EndTime,
HitCount = originalValue.HitCount,
MachineCount = MachineCountPerResponse,
Name = originalValue.Name,
SampleType = originalValue.SampleType,
StartTime = originalValue.StartTime
});
}).ToList();
for (int i = 0; i < ResponseCount; i++)
{
aggregator.AddMachineResponse(response);
}
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
Assert.AreEqual(1, superSamples.Count());
Assert.AreEqual(MachineCountPerResponse * ResponseCount, (int)superSamples.First().MachineCount);
}
public void CounterAggregatorKeepsDifferentSamplesSeparate()
{
// this ensures split by works correctly
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 10, 5));
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
Assert.AreEqual(5, superSamples.Count());
superSamples.All(x =>
{
this.VerifySample(x, 10);
return true;
});
}
public void CounterAggregatorCalculatesPercentileAfterAggregation()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = new CounterQueryResponse
{
HttpResponseCode = 200,
Samples = new List<DataSample>
{
new DataSample
{
Name = "bob",
StartTime = DateTime.Now.ToMillisecondTimestamp(),
EndTime = DateTime.Now.ToMillisecondTimestamp(),
Dimensions =
new Dictionary<string, string> {{AnyDimensionName, "tacos"}},
SampleType = DataSampleType.Histogram,
Histogram = new Dictionary<long, uint> {{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}}
}
}
};
// by default, we do not apply percentile filtering
aggregator.AddMachineResponse(response);
var defaultValue = aggregator.Samples.First();
Assert.AreEqual(DataSampleType.Histogram, defaultValue.SampleType);
// now that the client asked for filtering, we calculate the 99.999% percentile (should be the max value from earlier)
aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> {{"Percentile", "99.999"}});
var aggregatedValue = aggregator.Samples.First();
Assert.AreEqual(DataSampleType.Percentile, aggregatedValue.SampleType);
Assert.AreEqual(10, aggregatedValue.PercentileValue);
}
public void CounterAggregatorRejectsPerMachinePercentiles()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = new CounterQueryResponse
{
HttpResponseCode = 200,
Samples = new List<DataSample>
{
new DataSample
{
Name = "bob",
StartTime = DateTime.Now.ToMillisecondTimestamp(),
EndTime = DateTime.Now.ToMillisecondTimestamp(),
Dimensions =
new Dictionary<string, string> {{AnyDimensionName, "tacos"}},
SampleType = DataSampleType.Percentile,
Percentile = 40,
PercentileValue = 11,
}
}
};
Assert.Throws<ArgumentException>(() => aggregator.AddMachineResponse(response));
}
public void CounterAggregatorCanHandleRangesWhichAlwaysOverlap()
{
var aggregator = new CounterAggregator(DefaultDimensions);
// 10 minute buckets, staggered by 2 minute increments. Should 100% overlap
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 10, 2, 10));
Assert.AreEqual(1, aggregator.Samples.Count(item =>
{
this.VerifySample(item, 10);
return true;
}));
}
public void CounterAggregatorFiltersQueryParametersProperly()
{
var aggregator = new CounterAggregator(DefaultDimensions);
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(null));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string>()));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> {{"PERCENTILE", "50.0"}}));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "percentile", "50.0" } }));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "Percentile", "50.0" } }));
Assert.IsEmpty(aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "PErcenTILe", "50.0" } }));
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "percentile", "MAX" } }).Count());
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "percentile", "Average" } }).Count());
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "tacos", "delicious" } }).Count());
Assert.AreEqual(1, aggregator.ApplyPercentileCalculationAggregation(new Dictionary<string, string> { { "percentile", "i am a key tree!" } }).Count());
}
public void CounterAggregatorCanHandleSupersetRange()
{
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 9));
// super bucket
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now.AddHours(-1), 600, 10, 1));
Assert.AreEqual(1, aggregator.Samples.Count(item =>
{
this.VerifySample(item, 10);
return true;
}));
}
public void CounterAggregatorMergesCaseInsenstively()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var now = DateTime.Now;
aggregator.AddMachineResponse(this.CreateResponse(now, 1, 0, 1, 1, "taco"));
aggregator.AddMachineResponse(this.CreateResponse(now, 1, 0, 1, 1, "TACO"));
Assert.AreEqual(1, aggregator.Samples.Count());
}
public void CounterAggregatorDoesNothingWithoutData()
{
var aggregator = new CounterAggregator(DefaultDimensions);
Assert.IsTrue(!aggregator.Samples.Any());
Assert.IsTrue(!aggregator.TimeMergedSamples.Any());
}
public void CounterAggregatorRollsUpMachineCountProperly()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = this.CreateResponse(DateTime.Now, 1, 1, 10);
const int ResponseCount = 15;
const int MachineCountPerResponse = 10;
// add multiple machines per response
response.Samples = response.Samples.Select(s =>
{
var originalValue = s;
return new DataSample
{
Dimensions = originalValue.Dimensions,
EndTime = originalValue.EndTime,
HitCount = originalValue.HitCount,
MachineCount = MachineCountPerResponse,
Name = originalValue.Name,
SampleType = originalValue.SampleType,
StartTime = originalValue.StartTime
};
}).ToList();
for (int i = 0; i < ResponseCount; i++)
{
aggregator.AddMachineResponse(response);
}
var superSamples = aggregator.TimeMergedSamples;
Assert.IsNotNull(superSamples);
Assert.AreEqual(1, superSamples.Count());
Assert.AreEqual(MachineCountPerResponse * ResponseCount, (int)superSamples.First().MachineCount);
}
public void CounterAggregatorCanHandleRangesWhichDoNotOverlap()
{
var aggregator = new CounterAggregator(DefaultDimensions);
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now, 1, 1, 5));
aggregator.AddMachineResponse(this.CreateResponse(DateTime.Now.AddDays(-1), 1, 1, 5));
Assert.AreEqual(10, aggregator.Samples.Count(item =>
{
this.VerifySample(item, 1);
return true;
}));
}
