private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters)
{
var counterInfo = new CounterInfo
{
Name = counter.Name,
Type = counter.Type,
StartTime = counter.StartTime.ToMillisecondTimestamp(),
EndTime = counter.EndTime.ToMillisecondTimestamp(),
Dimensions = counter.Dimensions.ToList(),
DimensionValues = null, // null this out by default to avoid response bloat.
};
// Queries for dimension values will come with a 'dimension=pattern' query parameter.
// Dimension values can be further filtered with '<dimensionName>=pattern'
string dimensionPattern;
if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern))
{
counterInfo.DimensionValues = new Dictionary<string, ISet<string>>();
// We want to be able to filter dimension values by time (and only time)
var dimensionQuery = new DimensionSpecification();
string timeValue;
if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue))
{
timeValue = MinimumStartTime;
}
dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue;
if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue))
{
timeValue = MaximumEndTime;
}
dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue;
foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern)))
{
string filterPattern;
if (queryParameters.TryGetValue(dimensionPattern, out filterPattern))
{
counterInfo.AddDimensionValues(dim,
counter.GetDimensionValues(dim, dimensionQuery)
.Where(dimensionValue =>
dimensionValue.MatchGlob(filterPattern)));
}
else
{
counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery));
}
}
}
return counterInfo;
}
private static void RunOperation(Counter counter, CounterWriteOperation op)
{
var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow
? DateTime.UtcNow
: op.Timestamp.ToDateTime();
var dims = new DimensionSpecification(op.DimensionValues);
var hitCounter = counter as HitCounter;
var histogramCounter = counter as HistogramCounter;
if (hitCounter != null)
{
hitCounter.Increment(op.Value * op.Count, dims, timestamp);
}
else
{
// It would be nice to direct-inject multiple values at once, but the APIs don't currently
// support this. It's a reasonable amount of work to fix this and unknown whether folks will use this
// a lot at this time.
for (var i = 0; i < op.Count; ++i)
{
histogramCounter.AddValue(op.Value, dims, timestamp);
}
}
}
private async Task GetPendingCounterData(Counter counter)
{
this.UpdateCounterWithLatestTimestamps(counter);
DateTimeOffset lastPending = DateTimeOffset.MinValue;
PendingData pendingData = counter.DataSet.GetNextPendingData(lastPending);
if (pendingData == null)
{
return;
}
do
{
if (this.disposed)
{
return;
}
lastPending = pendingData.StartTime;
// TODO: Move below method to DateTimeOffset too.
using (var aggregator =
PersistedDataProtocol.CreateAggregatorForSampleType((MetricSystem.PersistedDataType)counter.DataSet.PersistedDataType,
counter.Name,
counter.DataSet.DimensionSet,
pendingData.Sources,
new DateTime(
pendingData.StartTime.DateTime.Ticks,
DateTimeKind.Utc),
new DateTime(pendingData.EndTime.DateTime.Ticks,
DateTimeKind.Utc),
this.dataManager.MemoryStreamManager))
{
var timeout = TimeoutTiers[ServerCountTiers.Length - 1];
for (var i = 0; i < ServerCountTiers.Length; ++i)
{
if (pendingData.Sources.Count <= ServerCountTiers[i])
{
timeout = TimeoutTiers[i];
break;
}
}
aggregator.MaxFanout = MaxFanout;
aggregator.Timeout = timeout;
Events.Write.BeginRetrieveCounterData(counter.Name, pendingData.StartTime, pendingData.EndTime,
timeout, pendingData.Sources);
var success = await aggregator.Run();
if (success)
{
counter.DataSet.UpdateFromAggregator(aggregator, pendingData.StartTime, pendingData.EndTime);
}
Events.Write.EndRetrieveCounterData(counter.Name, success);
}
this.UpdateCounterWithLatestTimestamps(counter);
} while ((pendingData = counter.DataSet.GetNextPendingData(lastPending)) != null);
lock (this.activeWorkers)
{
this.activeWorkers.Remove(counter.Name);
}
}
private void UpdateCounterWithLatestTimestamps(Counter counter)
{
var counterData = this.serverTimestamps[counter.Name];
var serversByTimestamp = new SortedList<DateTimeOffset, List<string>>(ReverseDateTimeOffsetComparer);
foreach (var kvp in counterData)
{
var serverName = kvp.Key;
var serverTimestamp = kvp.Value;
List<string> serverList;
if (!serversByTimestamp.TryGetValue(serverTimestamp, out serverList))
{
serverList = new List<string>();
serversByTimestamp.Add(serverTimestamp, serverList);
}
serverList.Add(serverName);
}
counter.DataSet.SetLatestTimeForDataSources(serversByTimestamp);
}
private async Task ScheduleWorker(Counter counter)
{
await this.taskRunner.RunAsync(() => this.GetPendingCounterData(counter));
}
private IEnumerable<DataSample> BucketedDataQuery(Counter counter, DimensionSpecification queryParams,
int expectedCount = 1)
{
if (queryParams != null)
{
queryParams[ReservedDimensions.AggregateSamplesDimension] = "false";
}
counter.DataSet.Flush();
var queryResult = counter.Query(queryParams);
if (queryResult == null)
{
return null;
}
var bucketedSamplesList = counter.Query(queryParams).ToList();
Assert.AreEqual(expectedCount, bucketedSamplesList.Count);
queryParams.Remove(ReservedDimensions.AggregateSamplesDimension);
return bucketedSamplesList;
}
private DataSample CombinedDataQuery(Counter counter, DimensionSpecification queryParams)
{
if (queryParams != null)
{
queryParams[ReservedDimensions.AggregateSamplesDimension] = "true";
}
counter.DataSet.Flush();
var queryResults = counter.Query(queryParams).ToList();
queryParams.Remove(ReservedDimensions.AggregateSamplesDimension);
return queryResults.Count > 0 ? queryResults[0] : null;
}
