internal static async Task <HttpResponseMessage> CreateGoodHitCountResponse(HttpRequestMessage request,
DateTime startTime,
int numSampleBuckets)
{
var requestMessage = await UnpackRequest <TieredRequest>(request);
var responseContent = new CounterQueryResponse
{
Samples = new List <DataSample>(),
RequestDetails = requestMessage.Sources.Select(source =>
new RequestDetails
{
Server = source,
Status = RequestStatus.Success
}).ToList()
};
// add a response for the leader
responseContent.RequestDetails.Add(new RequestDetails
{
Server = ExtractServerInfo(request.RequestUri),
Status = RequestStatus.Success,
HttpResponseCode = 200
});
AddHitCountSamples(responseContent.Samples, startTime, numSampleBuckets,
(uint)requestMessage.Sources.Count + 1);
return(CreateResponse(responseContent));
}
private CounterQueryResponse CreateResponse(DateTime startTime, int bucketTimeInMinutes, int deltaBetweenBucketStarts, int numBuckets, int uniqueDimensions = 1, string dimensionValue = null)
{
var response = new CounterQueryResponse {
Samples = new List <DataSample>()
};
var bucketStart = startTime;
for (int dim = 0; dim < uniqueDimensions; dim++)
{
for (int i = 0; i < numBuckets; i++)
{
response.Samples.Add(
new DataSample
{
HitCount = 1,
Dimensions = new Dictionary <string, string> {
{ AnyDimensionName, dimensionValue ?? dim.ToString() }
},
SampleType = DataSampleType.HitCount,
StartTime = bucketStart.ToMillisecondTimestamp(),
EndTime =
bucketStart.AddMinutes(bucketTimeInMinutes)
.ToMillisecondTimestamp()
});
bucketStart = bucketStart.AddMinutes(deltaBetweenBucketStarts);
}
}
return(response);
}
public void CounterAggregatorRejectsPerMachinePercentiles()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = new CounterQueryResponse
{
HttpResponseCode = 200,
Samples = new List <DataSample>
{
new DataSample
{
Name = "bob",
StartTime = DateTime.Now.ToMillisecondTimestamp(),
EndTime = DateTime.Now.ToMillisecondTimestamp(),
Dimensions =
new Dictionary <string, string> {
{ AnyDimensionName, "tacos" }
},
SampleType = DataSampleType.Percentile,
Percentile = 40,
PercentileValue = 11,
}
}
};
Assert.Throws <ArgumentException>(() => aggregator.AddMachineResponse(response));
}
internal static HttpResponseMessage CreateGoodCounterResponse(DateTime startTime, int numBuckets)
{
var CounterQueryResponse = new CounterQueryResponse {
Samples = new List <DataSample>()
};
AddHitCountSamples(CounterQueryResponse.Samples, startTime, numBuckets, 1);
return(CreateResponse(CounterQueryResponse));
}
internal static HttpResponseMessage CreateFailedTieredResponse(string hostname, HttpStatusCode status)
{
var response = new CounterQueryResponse();
response.RequestDetails.Add(new RequestDetails
{
Server = new ServerInfo
{
Hostname = hostname,
Port = 42,
},
HttpResponseCode = (short)status,
Status = RequestStatus.ServerFailureResponse,
IsAggregator = false,
StatusDescription = "Not found"
});
return(CreateResponse(response, HttpStatusCode.NotFound));
}
public void CounterAggregatorCalculatesPercentileAfterAggregation()
{
var aggregator = new CounterAggregator(DefaultDimensions);
var response = new CounterQueryResponse
{
HttpResponseCode = 200,
Samples = new List <DataSample>
{
new DataSample
{
Name = "bob",
StartTime = DateTime.Now.ToMillisecondTimestamp(),
EndTime = DateTime.Now.ToMillisecondTimestamp(),
Dimensions =
new Dictionary <string, string> {
{ AnyDimensionName, "tacos" }
},
SampleType = DataSampleType.Histogram,
Histogram = new Dictionary <long, uint> {
{ 1, 1 }, { 2, 1 }, { 3, 1 }, { 4, 1 }, { 5, 1 }, { 6, 1 }, { 7, 1 }, { 8, 1 }, { 9, 1 }, { 10, 1 }
}
}
}
};
// by default, we do not apply percentile filtering
aggregator.AddMachineResponse(response);
var defaultValue = aggregator.Samples.First();
Assert.AreEqual(DataSampleType.Histogram, defaultValue.SampleType);
// now that the client asked for filtering, we calculate the 99.999% percentile (should be the max value from earlier)
aggregator.ApplyPercentileCalculationAggregation(new Dictionary <string, string> {
{ "Percentile", "99.999" }
});
var aggregatedValue = aggregator.Samples.First();
Assert.AreEqual(DataSampleType.Percentile, aggregatedValue.SampleType);
Assert.AreEqual(10, aggregatedValue.PercentileValue);
}
/// <summary>
/// Merge the local Query request and the distributed tiered response into a single response
/// </summary>
private static CounterQueryResponse MergeResponses(IDictionary <string, string> queryParameters,
CounterQueryResponse localResponse, bool mergeTimeBuckets,
CounterQueryResponse distributedResponse)
{
// no distributed response, just use the local one
if (distributedResponse == null)
{
return(localResponse);
}
var sampleMerger = new CounterAggregator(queryParameters ?? new Dictionary <string, string>(0));
sampleMerger.AddMachineResponse(localResponse);
sampleMerger.AddMachineResponse(distributedResponse);
// response code from this server is:
// 200: *Anyone* has data
// 40X: *Everyone failed to get data, but everyone has the same response code
// 409: Mixed error codes ("Conflict" used liberally)
HttpStatusCode responseCode;
if ((localResponse.Samples != null && localResponse.Samples.Count > 0) ||
(distributedResponse.Samples != null && distributedResponse.Samples.Count > 0))
{
responseCode = HttpStatusCode.OK;
}
else
{
responseCode = distributedResponse.RequestDetails != null &&
distributedResponse.RequestDetails.Count > 0 &&
distributedResponse.RequestDetails.All(d => d.HttpResponseCode == (int)localResponse.HttpResponseCode)
? (HttpStatusCode)localResponse.HttpResponseCode
: HttpStatusCode.Conflict;
}
var mergedResponse = sampleMerger.GetResponse(mergeTimeBuckets);
mergedResponse.HttpResponseCode = (short)responseCode;
return(mergedResponse);
}
private static async Task <HttpResponseMessage> GenerateFailureResponse(HttpRequestMessage request,
HttpStatusCode responseCode,
RequestStatus statusForDownstreamSources)
{
var requestMessage = await MockDataFactory.UnpackRequest <TieredRequest>(request);
var responseContent = new CounterQueryResponse();
responseContent.RequestDetails = requestMessage.Sources.Select(source =>
new RequestDetails
{
Server = source,
Status = statusForDownstreamSources
}).ToList();
// add one for the leader
responseContent.RequestDetails.Add(new RequestDetails
{
Server = MockDataFactory.ExtractServerInfo(request.RequestUri),
Status = RequestStatus.ServerFailureResponse,
HttpResponseCode = (short)responseCode
});
return(MockDataFactory.CreateResponse(responseContent, responseCode));
}
public async void BatchQueryAggregatesFanoutCorrectly()
{
const string tacoTruck = "/Tacos";
const string competingBurritoTruck = "/Burritos";
var dimensionSet = new DimensionSet(new HashSet<Dimension>());
var counter = await this.dataManager.CreateHitCounter(tacoTruck, dimensionSet);
// locally store dim1
counter.Increment(100, new DimensionSpecification());
this.dataManager.Flush();
var data = new BatchQueryRequest();
data.Queries.Add(new BatchCounterQuery { CounterName = tacoTruck, UserContext = tacoTruck});
data.Queries.Add(new BatchCounterQuery
{
CounterName = competingBurritoTruck,
UserContext = competingBurritoTruck
});
data.Sources.Add(new ServerInfo {Hostname = "a", Port = 42});
data.Sources.Add(new ServerInfo {Hostname = "b", Port = 42});
var sampleStart = DateTime.Now;
var sampleEnd = sampleStart.AddHours(1);
// remotely return 100 for dim2 only
DistributedQueryClient.RequesterFactory = new MockHttpRequesterFactory(message =>
{
var batchResponse = new BatchQueryResponse();
batchResponse.RequestDetails.Add(new RequestDetails
{
Server =
new ServerInfo
{
Hostname = "bob",
Port = 42
},
HttpResponseCode = 200
});
var counterResponse = new CounterQueryResponse
{
HttpResponseCode = 200,
UserContext = competingBurritoTruck,
Samples = new List<DataSample> {
new DataSample { HitCount = 100, Dimensions = new Dictionary<string, string>(), SampleType = DataSampleType.HitCount, StartTime = sampleStart.ToMillisecondTimestamp(), EndTime = sampleEnd.ToMillisecondTimestamp() }
}
};
batchResponse.Responses.Add(counterResponse);
return MockDataFactory.CreateResponse(batchResponse);
});
var response = await
this.httpClient.PostAsync(TestUtils.GetUri(this.server, RestCommands.BatchQueryCommand,
string.Empty),
GetRequestPayload(data));
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
var responseData = await MockDataFactory.ReadResponseData<BatchQueryResponse>(response);
Assert.IsNotNull(responseData);
Assert.AreEqual(3, responseData.RequestDetails.Count);
Assert.IsTrue(responseData.RequestDetails.All(x => x.HttpResponseCode == 200));
Assert.AreEqual(2, responseData.Responses.Count);
Assert.AreEqual(1, responseData.Responses.Count(x => x.UserContext.Equals(tacoTruck) && x.Samples[0].HitCount == 100));
Assert.AreEqual(1, responseData.Responses.Count(x => x.UserContext.Equals(competingBurritoTruck) && x.Samples[0].HitCount == 200));
}
private static async Task<HttpResponseMessage> GenerateFailureResponse(HttpRequestMessage request,
HttpStatusCode responseCode,
RequestStatus statusForDownstreamSources)
{
var requestMessage = await MockDataFactory.UnpackRequest<TieredRequest>(request);
var responseContent = new CounterQueryResponse();
responseContent.RequestDetails = requestMessage.Sources.Select(source =>
new RequestDetails
{
Server = source,
Status = statusForDownstreamSources
}).ToList();
// add one for the leader
responseContent.RequestDetails.Add(new RequestDetails
{
Server = MockDataFactory.ExtractServerInfo(request.RequestUri),
Status = RequestStatus.ServerFailureResponse,
HttpResponseCode = (short)responseCode
});
return MockDataFactory.CreateResponse(responseContent, responseCode);
}
public async void BatchQueryAggregatesFanoutCorrectly()
{
const string tacoTruck = "/Tacos";
const string competingBurritoTruck = "/Burritos";
var dimensionSet = new DimensionSet(new HashSet <Dimension>());
var counter = await this.dataManager.CreateHitCounter(tacoTruck, dimensionSet);
// locally store dim1
counter.Increment(100, new DimensionSpecification());
this.dataManager.Flush();
var data = new BatchQueryRequest();
data.Queries.Add(new BatchCounterQuery {
CounterName = tacoTruck, UserContext = tacoTruck
});
data.Queries.Add(new BatchCounterQuery
{
CounterName = competingBurritoTruck,
UserContext = competingBurritoTruck
});
data.Sources.Add(new ServerInfo {
Hostname = "a", Port = 42
});
data.Sources.Add(new ServerInfo {
Hostname = "b", Port = 42
});
var sampleStart = DateTime.Now;
var sampleEnd = sampleStart.AddHours(1);
// remotely return 100 for dim2 only
DistributedQueryClient.RequesterFactory = new MockHttpRequesterFactory(message =>
{
var batchResponse = new BatchQueryResponse();
batchResponse.RequestDetails.Add(new RequestDetails
{
Server =
new ServerInfo
{
Hostname = "bob",
Port = 42
},
HttpResponseCode = 200
});
var counterResponse = new CounterQueryResponse
{
HttpResponseCode = 200,
UserContext = competingBurritoTruck,
Samples = new List <DataSample> {
new DataSample {
HitCount = 100, Dimensions = new Dictionary <string, string>(), SampleType = DataSampleType.HitCount, StartTime = sampleStart.ToMillisecondTimestamp(), EndTime = sampleEnd.ToMillisecondTimestamp()
}
}
};
batchResponse.Responses.Add(counterResponse);
return(MockDataFactory.CreateResponse(batchResponse));
});
var response = await
this.httpClient.PostAsync(TestUtils.GetUri(this.server, RestCommands.BatchQueryCommand,
string.Empty),
GetRequestPayload(data));
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
var responseData = await MockDataFactory.ReadResponseData <BatchQueryResponse>(response);
Assert.IsNotNull(responseData);
Assert.AreEqual(3, responseData.RequestDetails.Count);
Assert.IsTrue(responseData.RequestDetails.All(x => x.HttpResponseCode == 200));
Assert.AreEqual(2, responseData.Responses.Count);
Assert.AreEqual(1, responseData.Responses.Count(x => x.UserContext.Equals(tacoTruck) && x.Samples[0].HitCount == 100));
Assert.AreEqual(1, responseData.Responses.Count(x => x.UserContext.Equals(competingBurritoTruck) && x.Samples[0].HitCount == 200));
}
/// <summary>
/// Merge the local Query request and the distributed tiered response into a single response
/// </summary>
private static CounterQueryResponse MergeResponses(IDictionary<string, string> queryParameters,
CounterQueryResponse localResponse, bool mergeTimeBuckets,
CounterQueryResponse distributedResponse)
{
// no distributed response, just use the local one
if (distributedResponse == null)
{
return localResponse;
}
var sampleMerger = new CounterAggregator(queryParameters ?? new Dictionary<string, string>(0));
sampleMerger.AddMachineResponse(localResponse);
sampleMerger.AddMachineResponse(distributedResponse);
// response code from this server is:
// 200: *Anyone* has data
// 40X: *Everyone failed to get data, but everyone has the same response code
// 409: Mixed error codes ("Conflict" used liberally)
HttpStatusCode responseCode;
if ((localResponse.Samples != null && localResponse.Samples.Count > 0) ||
(distributedResponse.Samples != null && distributedResponse.Samples.Count > 0))
{
responseCode = HttpStatusCode.OK;
}
else
{
responseCode = distributedResponse.RequestDetails != null &&
distributedResponse.RequestDetails.Count > 0 &&
distributedResponse.RequestDetails.All(d => d.HttpResponseCode == (int)localResponse.HttpResponseCode)
? (HttpStatusCode)localResponse.HttpResponseCode
: HttpStatusCode.Conflict;
}
var mergedResponse = sampleMerger.GetResponse(mergeTimeBuckets);
mergedResponse.HttpResponseCode = (short)responseCode;
return mergedResponse;
}
internal async Task<CounterQueryResponse> Query(string counterName, TieredRequest fanoutRequest,
DimensionSpecification queryParameters)
{
Task<CounterQueryResponse> distributedRequest;
// if fanout is required, start it here (don't block)
if (fanoutRequest != null && fanoutRequest.Sources != null && fanoutRequest.Sources.Count > 0)
{
distributedRequest = this.server.CreateQueryClient(fanoutRequest)
.CounterQuery(counterName, fanoutRequest, queryParameters,
DimensionSet.FromQueryParameters(queryParameters));
}
else
{
distributedRequest = Task.FromResult<CounterQueryResponse>(null);
}
// get the local data (in parallel to the async distributed request running, if necessary)
var localResponse = new CounterQueryResponse();
try
{
var counter = this.server.DataManager.GetCounter<Counter>(counterName);
if (counter == null)
{
localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound;
localResponse.ErrorMessage = "Counter not found.";
}
else
{
Events.Write.BeginProcessingQuery(RestCommands.CounterQueryCommand, counterName);
localResponse.Samples = counter.Query(queryParameters).ToList();
if (localResponse.Samples.Count > 0)
{
Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 200);
localResponse.HttpResponseCode = (int)HttpStatusCode.OK;
}
else
{
Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 404);
localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound;
localResponse.ErrorMessage = "No data matched query.";
}
}
}
catch (Exception ex)
{
if (ex is FormatException || ex is ArgumentOutOfRangeException || ex is ArgumentException
|| ex is KeyNotFoundException || ex is NotSupportedException)
{
localResponse.HttpResponseCode = (int)HttpStatusCode.BadRequest;
localResponse.ErrorMessage = ex.Message;
}
else
{
throw;
}
}
// wait for the distributed request to finish
var distributedResponse = await distributedRequest;
if (fanoutRequest != null && fanoutRequest.IncludeRequestDiagnostics && localResponse.RequestDetails != null)
{
localResponse.RequestDetails.Add(new RequestDetails
{
Server = new MetricSystem.ServerInfo
{
Hostname = this.server.Hostname,
Port = this.server.Port,
Datacenter = this.server.Datacenter,
MachineFunction = this.server.MachineFunction,
},
HttpResponseCode = localResponse.HttpResponseCode,
StatusDescription = localResponse.ErrorMessage ?? string.Empty,
Status = localResponse.HttpResponseCode == (short)HttpStatusCode.OK ? RequestStatus.Success : RequestStatus.ServerFailureResponse,
IsAggregator = distributedResponse != null
});
}
var response = MergeResponses(queryParameters, localResponse,
DistributedQueryClient.ShouldMergeTimeBuckets(queryParameters),
distributedResponse);
return response;
}
internal async Task <CounterQueryResponse> Query(string counterName, TieredRequest fanoutRequest,
DimensionSpecification queryParameters)
{
Task <CounterQueryResponse> distributedRequest;
// if fanout is required, start it here (don't block)
if (fanoutRequest != null && fanoutRequest.Sources != null && fanoutRequest.Sources.Count > 0)
{
distributedRequest = this.server.CreateQueryClient(fanoutRequest)
.CounterQuery(counterName, fanoutRequest, queryParameters,
DimensionSet.FromQueryParameters(queryParameters));
}
else
{
distributedRequest = Task.FromResult <CounterQueryResponse>(null);
}
// get the local data (in parallel to the async distributed request running, if necessary)
var localResponse = new CounterQueryResponse();
try
{
var counter = this.server.DataManager.GetCounter <Counter>(counterName);
if (counter == null)
{
localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound;
localResponse.ErrorMessage = "Counter not found.";
}
else
{
Events.Write.BeginProcessingQuery(RestCommands.CounterQueryCommand, counterName);
localResponse.Samples = counter.Query(queryParameters).ToList();
if (localResponse.Samples.Count > 0)
{
Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 200);
localResponse.HttpResponseCode = (int)HttpStatusCode.OK;
}
else
{
Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 404);
localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound;
localResponse.ErrorMessage = "No data matched query.";
}
}
}
catch (Exception ex)
{
if (ex is FormatException || ex is ArgumentOutOfRangeException || ex is ArgumentException ||
ex is KeyNotFoundException || ex is NotSupportedException)
{
localResponse.HttpResponseCode = (int)HttpStatusCode.BadRequest;
localResponse.ErrorMessage = ex.Message;
}
else
{
throw;
}
}
// wait for the distributed request to finish
var distributedResponse = await distributedRequest;
if (fanoutRequest != null && fanoutRequest.IncludeRequestDiagnostics && localResponse.RequestDetails != null)
{
localResponse.RequestDetails.Add(new RequestDetails
{
Server = new MetricSystem.ServerInfo
{
Hostname = this.server.Hostname,
Port = this.server.Port,
Datacenter = this.server.Datacenter,
MachineFunction = this.server.MachineFunction,
},
HttpResponseCode = localResponse.HttpResponseCode,
StatusDescription = localResponse.ErrorMessage ?? string.Empty,
Status = localResponse.HttpResponseCode == (short)HttpStatusCode.OK ? RequestStatus.Success : RequestStatus.ServerFailureResponse,
IsAggregator = distributedResponse != null
});
}
var response = MergeResponses(queryParameters, localResponse,
DistributedQueryClient.ShouldMergeTimeBuckets(queryParameters),
distributedResponse);
return(response);
}
/// <summary>
/// Add all samples from this query response. Combine overlapping time buckets as they are encountered.
/// This method IS threadsafe against itself.
/// </summary>
/// <param name="response"></param>
public void AddMachineResponse(CounterQueryResponse response)
{
if (response == null)
{
throw new ArgumentNullException("response");
}
if (response.RequestDetails != null)
{
lock (this.requestDetails)
{
this.requestDetails.AddRange(response.RequestDetails);
}
}
if (response.Samples == null)
{
return;
}
foreach (var sample in response.Samples)
{
var baseSample = sample;
var hashKey = this.dimensionSet.CreateKey(sample.Dimensions);
var sampleTimeRange = CreateTimeRange(baseSample);
var rangesToRemove = new List <TimeRange>();
SampleCombiner combiner = null;
SortedList <TimeRange, SampleCombiner> aggregatedBuckets;
// grab the appropriate bucket list
lock (this.dataDictionary)
{
if (!this.dataDictionary.TryGetValue(hashKey, out aggregatedBuckets))
{
aggregatedBuckets = new SortedList <TimeRange, SampleCombiner>();
this.dataDictionary.Add(hashKey, aggregatedBuckets);
}
}
lock (aggregatedBuckets)
{
// The buckets are ordered by start time - thus it is safe to merge and continue to
// walk forward as we cannot ever merge which requires a backwards reprocess
foreach (var bucket in aggregatedBuckets)
{
var existingRange = bucket.Key;
// did we get past the end of the range we are interested in?
if (existingRange.Start > sampleTimeRange.End)
{
break;
}
if (existingRange.IntersectsWith(sampleTimeRange))
{
sampleTimeRange = TimeRange.Merge(sampleTimeRange, existingRange);
rangesToRemove.Add(bucket.Key);
// if this is the first merge, just add this sample
if (combiner == null)
{
combiner = bucket.Value;
combiner.AddSample(sample);
combiner.MachineCount += SampleCombiner.ExtractMachineCount(sample);
}
else
{
// this is a N-merge (N > 1), thus sample is already accounted for in the combiner. Merge the values
combiner.Merge(bucket.Value);
}
}
}
// if there was no merge, then create a new bucket with this sample
if (combiner == null)
{
combiner = new SampleCombiner(sample)
{
MachineCount = SampleCombiner.ExtractMachineCount(sample)
};
}
// remove the merged items and add the new item
foreach (var range in rangesToRemove)
{
aggregatedBuckets.Remove(range);
}
aggregatedBuckets.Add(sampleTimeRange, combiner);
}
}
}