public async Task GetDimensionValuesForAllDimensionInDataProvidesExpectedValues()
{
const int NumValuesPerDim = 100;
var dimensionValues = new DimensionSpecification();
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
for (var i = 0; i < NumValuesPerDim; ++i)
{
foreach (var d in AnyKeys.Dimensions)
{
dimensionValues[d.Name] = d.Name + i;
}
counter.Increment(dimensionValues);
}
counter.DataSet.Flush();
foreach (var d in AnyKeys.Dimensions)
{
var values = counter.GetDimensionValues(d.Name, new DimensionSpecification()).ToList();
Assert.AreEqual(NumValuesPerDim, values.Count);
for (var i = 0; i < NumValuesPerDim; ++i)
{
var expected = d.Name + i;
Assert.IsTrue(values.Contains(expected));
}
}
}
public void SetUp()
{
this.dimensionSet =
new DimensionSet(new HashSet<Dimension>
{
new Dimension("foo"),
new Dimension("bar"),
new Dimension("baz"),
});
dimensions = new DimensionSpecification();
this.storagePath = Path.Combine(Environment.CurrentDirectory, "msTemp");
if (Directory.Exists(this.storagePath))
{
Directory.Delete(this.storagePath, true);
}
Directory.CreateDirectory(this.storagePath);
foreach (var d in this.dimensionSet.Dimensions)
{
dimensions[d.Name] = DimValue;
}
this.properties = new MockSharedDataSetProperties();
}
private static void RunOperation(Counter counter, CounterWriteOperation op)
{
var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow
? DateTime.UtcNow
: op.Timestamp.ToDateTime();
var dims = new DimensionSpecification(op.DimensionValues);
var hitCounter = counter as HitCounter;
var histogramCounter = counter as HistogramCounter;
if (hitCounter != null)
{
hitCounter.Increment(op.Value * op.Count, dims, timestamp);
}
else
{
// It would be nice to direct-inject multiple values at once, but the APIs don't currently
// support this. It's a reasonable amount of work to fix this and unknown whether folks will use this
// a lot at this time.
for (var i = 0; i < op.Count; ++i)
{
histogramCounter.AddValue(op.Value, dims, timestamp);
}
}
}
public async Task HistogramPercentileValuesAreCorrectlyDistributedByBucket()
{
var bucket1Timestamp = DateTime.Now;
var bucket2Timestamp =
new DateTime(bucket1Timestamp.Ticks + this.defaultCompactionConfig.DefaultBucketTicks * 2);
var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);
for (int i = 0; i < 100; ++i)
{
counter.AddValue(i, AnyDimensions, bucket1Timestamp);
}
for (int i = 0; i < 100; ++i)
{
counter.AddValue(i * 10, AnyDimensions, bucket2Timestamp);
}
var dims = new DimensionSpecification(AnyDimensions);
for (int i = 1; i < 100; ++i)
{
dims[ReservedDimensions.PercentileDimension] = i.ToString();
var bucketedSamples =
this.BucketedDataQuery(counter, dims, 2).ToList();
var sample1 = bucketedSamples[1];
Assert.AreEqual(sample1.Percentile, i);
Assert.AreEqual(sample1.SampleCount, (ulong)100);
Assert.AreEqual(sample1.PercentileValue, i - 1);
var sample2 = bucketedSamples[0];
Assert.AreEqual(sample2.Percentile, i);
Assert.AreEqual(sample2.SampleCount, (ulong)100);
Assert.AreEqual(sample2.PercentileValue, (i - 1) * 10);
}
}
public override void Setup()
{
this.dataManager.CompactionConfiguration =
new DataCompactionConfiguration(new[]
{
new DataIntervalConfiguration(TimeSpan.FromMinutes(5), TimeSpan.MaxValue),
});
this.dataManager.MaximumDataAge = TimeSpan.Zero; // Don't want data getting deleted here.
// Make a counter and write some stuff.
this.dimensionSet = new DimensionSet(new HashSet <Dimension> {
new Dimension(AnyDimension),
});
var counter = this.dataManager.CreateHitCounter(AnyCounter, this.dimensionSet).Result;
var dims = new DimensionSpecification();
var timestamp = DateTime.Now;
dims[AnyDimension] = "a";
counter.Increment(dims, timestamp);
dims[AnyDimension] = "b";
counter.Increment(2, dims, timestamp);
dims[AnyDimension] = "c";
counter.Increment(dims, timestamp);
// We need to force a seal by setting some value waaaay in the future.
counter.Increment(dims, timestamp.AddYears(10));
this.memoryStreamManager = new RecyclableMemoryStreamManager(1 << 17, 1 << 20, 1 << 24);
}
public void MatchIsFalseIfFilterHasMoreOrLessValuesThanKey()
{
const string anyDim1 = "d1";
const string anyDim2 = "d2";
const string anyDim3 = "d3";
var dimensionSet = new DimensionSet(new HashSet <Dimension>
{
new Dimension(anyDim1),
new Dimension(anyDim2),
new Dimension(anyDim3),
});
var keyValueSet = new DimensionSpecification {
{ anyDim1, "val1" }, { anyDim2, "val2" }, { anyDim3, "val3" }
};
bool allDimensionsProvided;
Key key = dimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);
var smallerDimensionSet =
new DimensionSet(new HashSet <Dimension> {
new Dimension(anyDim1), new Dimension(anyDim2)
});
Key smallerFilter = smallerDimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);
Assert.IsFalse(smallerFilter.Matches(key));
var largerDimensionSet = new DimensionSet(new HashSet <Dimension>(dimensionSet.Dimensions)
{
new Dimension
("anotherDim")
});
Key largerFilter = largerDimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);
Assert.IsFalse(largerFilter.Matches(key));
}
/// <summary>
/// Safely increment a HitCounter by one using the given timestamp.
/// If the counter is null no operation will be performed.
/// </summary>
/// <param name="counter">Counter to increment.</param>
/// <param name="dims">Dimensions to use for incrementing.</param>
/// <param name="timestamp">Timestamp for written value.</param>
public static void SafeIncrement(this HitCounter counter, DimensionSpecification dims, DateTime timestamp)
{
if (counter != null)
{
counter.Increment(1, dims, timestamp);
}
}
public void SetUp()
{
this.dimensionSet =
new DimensionSet(new HashSet <Dimension>
{
new Dimension("foo"),
new Dimension("bar"),
new Dimension("baz"),
});
dimensions = new DimensionSpecification();
this.storagePath = Path.Combine(Environment.CurrentDirectory, "msTemp");
if (Directory.Exists(this.storagePath))
{
Directory.Delete(this.storagePath, true);
}
Directory.CreateDirectory(this.storagePath);
foreach (var d in this.dimensionSet.Dimensions)
{
dimensions[d.Name] = DimValue;
}
this.properties = new MockSharedDataSetProperties();
}
/// <summary>
/// Ask the counter aggregator to do all percentile computation as opposed to any metric system server. This will
/// strip out the 'percentile=xxx' parameter in the request and return the filtered parameter list. If a
/// percentile request was present, this will apply the percentile calculation in the final calculation of this object
/// </summary>
/// <param name="queryParameters">Parameters to filter</param>
/// <returns>Filtered parameter set</returns>
public IDictionary<string, string> ApplyPercentileCalculationAggregation(IDictionary<string, string> queryParameters)
{
if (queryParameters == null)
{
return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}
string percentileParameter;
double percentileRequested;
var dimensionSpec = new DimensionSpecification(queryParameters);
if (!dimensionSpec.TryGetValue(ReservedDimensions.PercentileDimension, out percentileParameter) ||
!double.TryParse(percentileParameter, out percentileRequested) ||
percentileRequested < 0 || percentileRequested > 100)
{
this.shouldFilterToFinalPercentile = false;
return queryParameters;
}
this.shouldFilterToFinalPercentile = true;
this.requestedPercentile = percentileRequested;
var retVal = new Dictionary<string, string>(dimensionSpec, StringComparer.OrdinalIgnoreCase);
retVal.Remove(ReservedDimensions.PercentileDimension);
return retVal;
}
public override void Setup()
{
this.dataManager.CompactionConfiguration =
new DataCompactionConfiguration(new[]
{
new DataIntervalConfiguration(TimeSpan.FromMinutes(5), TimeSpan.MaxValue),
});
this.dataManager.MaximumDataAge = TimeSpan.Zero; // Don't want data getting deleted here.
// Make a counter and write some stuff.
this.dimensionSet = new DimensionSet(new HashSet<Dimension> { new Dimension(AnyDimension), });
var counter = this.dataManager.CreateHitCounter(AnyCounter, this.dimensionSet).Result;
var dims = new DimensionSpecification();
var timestamp = DateTime.Now;
dims[AnyDimension] = "a";
counter.Increment(dims, timestamp);
dims[AnyDimension] = "b";
counter.Increment(2, dims, timestamp);
dims[AnyDimension] = "c";
counter.Increment(dims, timestamp);
// We need to force a seal by setting some value waaaay in the future.
counter.Increment(dims, timestamp.AddYears(10));
this.memoryStreamManager = new RecyclableMemoryStreamManager(1 << 17, 1 << 20, 1 << 24);
}
/// <summary>
/// Safely increment a HitCounter by the given amount using the current time.
/// If the counter is null no operation will be performed.
/// </summary>
/// <param name="counter">Counter to increment.</param>
/// <param name="amount">Amount to increment by.</param>
/// <param name="dims">Dimensions to use for incrementing.</param>
public static void SafeIncrement(this HitCounter counter, long amount, DimensionSpecification dims)
{
if (counter != null)
{
counter.Increment(amount, dims, DateTime.Now);
}
}
public async Task CanRetrieveHistogramPercentileData()
{
var anyTimestamp = DateTime.Now;
var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);
for (int i = 0; i < 100; ++i)
{
counter.AddValue(i, AnyDimensions, anyTimestamp);
}
var dims = new DimensionSpecification(AnyDimensions);
for (int i = 1; i < 100; ++i)
{
dims[ReservedDimensions.PercentileDimension] = i.ToString();
var sample =
this.CombinedDataQuery(counter, dims);
Assert.AreEqual(sample.Percentile, i);
Assert.AreEqual(sample.SampleCount, (ulong)100);
Assert.AreEqual(sample.PercentileValue, i - 1);
}
dims[ReservedDimensions.PercentileDimension] = "0";
var sample2 = this.CombinedDataQuery(counter, dims);
Assert.AreEqual(sample2.Percentile, 0);
Assert.AreEqual(sample2.SampleCount, (ulong)100);
Assert.AreEqual(sample2.PercentileValue, 0);
dims[ReservedDimensions.PercentileDimension] = "100";
sample2 = this.CombinedDataQuery(counter, dims);
Assert.AreEqual(sample2.Percentile, 100);
Assert.AreEqual(sample2.SampleCount, (ulong)100);
Assert.AreEqual(sample2.PercentileValue, 99);
}
/// <summary>
/// Ask the counter aggregator to do all percentile computation as opposed to any metric system server. This will
/// strip out the 'percentile=xxx' parameter in the request and return the filtered parameter list. If a
/// percentile request was present, this will apply the percentile calculation in the final calculation of this object
/// </summary>
/// <param name="queryParameters">Parameters to filter</param>
/// <returns>Filtered parameter set</returns>
public IDictionary <string, string> ApplyPercentileCalculationAggregation(IDictionary <string, string> queryParameters)
{
if (queryParameters == null)
{
return(new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase));
}
string percentileParameter;
double percentileRequested;
var dimensionSpec = new DimensionSpecification(queryParameters);
if (!dimensionSpec.TryGetValue(ReservedDimensions.PercentileDimension, out percentileParameter) ||
!double.TryParse(percentileParameter, out percentileRequested) ||
percentileRequested < 0 || percentileRequested > 100)
{
this.shouldFilterToFinalPercentile = false;
return(queryParameters);
}
this.shouldFilterToFinalPercentile = true;
this.requestedPercentile = percentileRequested;
var retVal = new Dictionary <string, string>(dimensionSpec, StringComparer.OrdinalIgnoreCase);
retVal.Remove(ReservedDimensions.PercentileDimension);
return(retVal);
}
private void GenerateRandomKey(DimensionSpecification dims)
{
dims.Clear();
foreach (var d in this.dimensionSet.Dimensions)
{
dims.Add(d.Name, GetRandom(MaxDimensionValue).ToString());
}
}
private void WriteComplexSampleData()
{
var rng = new Random();
histogramKeys = new int[BucketCount][][];
totalCountBySample = new Dictionary <long, uint>();
this.dimensionSet = new DimensionSet(new HashSet <Dimension>
{
new Dimension("one"),
new Dimension("two"),
});
this.sharedProperties.CompactionConfiguration =
new DataCompactionConfiguration(new[]
{
new DataIntervalConfiguration(TimeSpan.FromMinutes(1), TimeSpan.MaxValue),
});
this.histogramDataSet =
new DataSet <InternalHistogram>("/AggTest", null,
this.dimensionSet,
this.sharedProperties);
var dimSpec = new DimensionSpecification();
var startTime = new DateTime(2014, 7, 4, 0, 0, 0, DateTimeKind.Utc); // no DateTimeKind.MERKUH? Sigh.
for (var i = 0; i < BucketCount; ++i)
{
histogramKeys[i] = new int[DimensionOneCount][];
var ts = startTime +
TimeSpan.FromTicks(i * this.sharedProperties.CompactionConfiguration.DefaultBucketTicks);
for (var d1 = 0; d1 < DimensionOneCount; ++d1)
{
histogramKeys[i][d1] = new int[DimensionTwoCount];
dimSpec["one"] = d1.ToString();
for (var d2 = 0; d2 < DimensionTwoCount; ++d2)
{
dimSpec["two"] = d2.ToString();
var val = rng.Next(MaxValue);
this.histogramDataSet.AddValue(val, dimSpec, ts);
histogramKeys[i][d1][d2] = val;
if (!totalCountBySample.ContainsKey(val))
{
totalCountBySample[val] = 0;
}
totalCountBySample[val]++;
}
}
}
this.histogramDataSet.Flush();
}
public async Task GetDimensionValuesForUnknownFilterDimensionDoesNotThrow()
{
var filterDims = new DimensionSpecification {
{ "UnknownDim", "UnKnown" }
};
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
// if the following call throws test will fail
counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filterDims);
}
private void WriteRandomTestData(int maxValue, TestKeyedDataStore store)
{
Parallel.For(0, maxValue,
i =>
{
var dims = new DimensionSpecification();
this.GenerateRandomKey(dims);
store.AddValue(dims, i);
});
}
public void SetUp()
{
this.dimensionSet = new DimensionSet(new HashSet<Dimension>());
this.dimensionSpec = new DimensionSpecification();
this.sharedProperties = new MockSharedDataSetProperties();
this.bucketSpan = this.sharedProperties.CompactionConfiguration.Default.Interval;
this.firstBucketTimestamp = DateTime.UtcNow;
this.secondBucketTimestamp = this.firstBucketTimestamp + this.bucketSpan;
this.thirdBucketTimestamp = this.secondBucketTimestamp + this.bucketSpan;
}
public async Task GetDimensionValuesWithAllDimensionsFilledThrowsArgumentException()
{
var dimValues = new DimensionSpecification(AnyDimensions);
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
Assert.Throws <ArgumentException>(
() =>
counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, dimValues)
.ToList());
}
public void SetUp()
{
this.dimensionSet = new DimensionSet(new HashSet <Dimension>());
this.dimensionSpec = new DimensionSpecification();
this.sharedProperties = new MockSharedDataSetProperties();
this.bucketSpan = this.sharedProperties.CompactionConfiguration.Default.Interval;
this.firstBucketTimestamp = DateTime.UtcNow;
this.secondBucketTimestamp = this.firstBucketTimestamp + this.bucketSpan;
this.thirdBucketTimestamp = this.secondBucketTimestamp + this.bucketSpan;
}
private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters)
{
var counterInfo = new CounterInfo
{
Name = counter.Name,
Type = counter.Type,
StartTime = counter.StartTime.ToMillisecondTimestamp(),
EndTime = counter.EndTime.ToMillisecondTimestamp(),
Dimensions = counter.Dimensions.ToList(),
DimensionValues = null, // null this out by default to avoid response bloat.
};
// Queries for dimension values will come with a 'dimension=pattern' query parameter.
// Dimension values can be further filtered with '<dimensionName>=pattern'
string dimensionPattern;
if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern))
{
counterInfo.DimensionValues = new Dictionary <string, ISet <string> >();
// We want to be able to filter dimension values by time (and only time)
var dimensionQuery = new DimensionSpecification();
string timeValue;
if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue))
{
timeValue = MinimumStartTime;
}
dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue;
if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue))
{
timeValue = MaximumEndTime;
}
dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue;
foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern)))
{
string filterPattern;
if (queryParameters.TryGetValue(dimensionPattern, out filterPattern))
{
counterInfo.AddDimensionValues(dim,
counter.GetDimensionValues(dim, dimensionQuery)
.Where(dimensionValue =>
dimensionValue.MatchGlob(filterPattern)));
}
else
{
counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery));
}
}
}
return(counterInfo);
}
public async Task AddingDataWithEmptyKeyValuesIsValid()
{
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
var dimensions = new DimensionSpecification();
foreach (var d in AnyKeys.Dimensions)
{
dimensions[d.Name] = string.Empty;
}
counter.Increment(dimensions);
}
public override async Task <Response> ProcessRequest(Request request)
{
var fanoutRequest = request.HasInputBody ? await request.ReadInputBody <TieredRequest>() : null;
if (request.HasInputBody && fanoutRequest == null)
{
// This indicates failed deserialization
return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body"));
}
// aggregation servers should fanout automagically. we honor filters for machine function and datacenter.
if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation)
{
if (fanoutRequest == null)
{
fanoutRequest = new TieredRequest();
}
fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList();
}
var queryParameters = new DimensionSpecification(request.QueryParameters);
var endOfName = request.Path.LastIndexOf('/');
if (endOfName < 0)
{
return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided."));
}
var counterName = request.Path.Substring(0, endOfName);
if (string.IsNullOrEmpty(counterName))
{
return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided."));
}
var commandName = request.Path.Substring(endOfName + 1);
if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
{
return(await this.Info(request, fanoutRequest, counterName, queryParameters));
}
if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
{
var response = await this.Query(counterName, fanoutRequest, queryParameters);
return(Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response));
}
return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName));
}
public async Task GetBucketedDataCombinesDataAcrossDimensionValuesIfFilterValueIsNotProvided()
{
var anyTimestamp = DateTime.Now;
const string FirstDimension = "1st";
const string SecondDimension = "2nd";
const string ThirdDimension = "3rd";
var dimensions = new DimensionSet(new HashSet <Dimension>
{
new Dimension(FirstDimension),
new Dimension(SecondDimension),
new Dimension(ThirdDimension)
});
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, dimensions);
var dimValues = new DimensionSpecification();
for (int i = 0; i < 10; ++i)
{
dimValues[FirstDimension] = i.ToString();
for (int j = 0; j < 10; ++j)
{
dimValues[SecondDimension] = j.ToString();
for (int k = 0; k < 10; ++k)
{
dimValues[ThirdDimension] = k.ToString();
counter.Increment(dimValues, anyTimestamp);
}
}
}
var sample =
this.BucketedDataQuery(counter, new DimensionSpecification()).First();
Assert.AreEqual(DataSampleType.HitCount, sample.SampleType);
Assert.AreEqual((ulong)1000, sample.HitCount);
dimValues.Clear();
dimValues[FirstDimension] = "8"; // take one tenth of things by filter.
sample = this.BucketedDataQuery(counter, dimValues).First();
Assert.AreEqual((ulong)100, sample.HitCount);
dimValues[SecondDimension] = "6";
sample = this.BucketedDataQuery(counter, dimValues).First();
Assert.AreEqual((ulong)10, sample.HitCount);
dimValues[ThirdDimension] = "7";
sample = this.BucketedDataQuery(counter, dimValues).First();
Assert.AreEqual((ulong)1, sample.HitCount);
}
public void CombinedBucketsAreMergedCorrectly()
{
var ts1 = new DateTime(2014, 05, 10, 0, 0, 0);
var ts2 = new DateTime(2014, 05, 10, 0, 5, 0);
var sharedDimensions = new DimensionSpecification {
{ "one", "a1" }, { "two", "a2" }
};
var bucket1Dimensions = new DimensionSpecification {
{ "one", "b1" }, { "two", "b2" }
};
var bucket2Dimensions = new DimensionSpecification {
{ "one", "c1" }, { "two", "c2" }
};
var bucket1 =
new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket1.AddValue(sharedDimensions, 867);
bucket1.AddValue(bucket1Dimensions, 5309);
var bucket2 =
new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket2.AddValue(sharedDimensions, 867);
bucket2.AddValue(bucket2Dimensions, 42);
bucket1.Seal();
bucket2.Seal();
var bucket3 =
new DataBucket <InternalHitCount>(new[] { bucket1, bucket2 },
new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(10).Ticks, null,
this.properties.MemoryStreamManager);
var match = bucket3.GetMatches(bucket1Dimensions).First().Data;
Assert.AreEqual((ulong)5309, match.HitCount);
match = bucket3.GetMatches(bucket2Dimensions).First().Data;
Assert.AreEqual((ulong)42, match.HitCount);
match = bucket3.GetMatches(sharedDimensions).First().Data;
Assert.AreEqual((ulong)867 * 2, match.HitCount);
bucket1.Dispose();
bucket2.Dispose();
bucket3.Dispose();
}
public async Task InvalidNumericHistogramValueThrowsArgumentOutOfRangeException()
{
var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);
var dims = new DimensionSpecification(AnyDimensions);
dims[ReservedDimensions.PercentileDimension] = "-1";
Assert.Throws <ArgumentOutOfRangeException>(() => this.CombinedDataQuery(counter, dims));
Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, dims));
dims[ReservedDimensions.PercentileDimension] = "100.1";
Assert.Throws <ArgumentOutOfRangeException>(() => this.CombinedDataQuery(counter, dims));
Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, dims));
}
private void WriteSomeData()
{
var dimensionValues = new DimensionSpecification();
for (var i = 0; i < 10; ++i)
{
for (var j = 0; j < 10; ++j)
{
dimensionValues["one"] = i.ToString();
dimensionValues["two"] = j.ToString();
this.bucket.AddValue(dimensionValues, WrittenValue);
}
}
}
private DataSample CombinedDataQuery(Counter counter, DimensionSpecification queryParams)
{
if (queryParams != null)
{
queryParams[ReservedDimensions.AggregateSamplesDimension] = "true";
}
counter.DataSet.Flush();
var queryResults = counter.Query(queryParams).ToList();
queryParams.Remove(ReservedDimensions.AggregateSamplesDimension);
return(queryResults.Count > 0 ? queryResults[0] : null);
}
public override async Task<Response> ProcessRequest(Request request)
{
var fanoutRequest = request.HasInputBody ? await request.ReadInputBody<TieredRequest>() : null;
if (request.HasInputBody && fanoutRequest == null)
{
// This indicates failed deserialization
return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body");
}
// aggregation servers should fanout automagically. we honor filters for machine function and datacenter.
if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation)
{
if (fanoutRequest == null)
{
fanoutRequest = new TieredRequest();
}
fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList();
}
var queryParameters = new DimensionSpecification(request.QueryParameters);
var endOfName = request.Path.LastIndexOf('/');
if (endOfName < 0)
{
return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided.");
}
var counterName = request.Path.Substring(0, endOfName);
if (string.IsNullOrEmpty(counterName))
{
return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided.");
}
var commandName = request.Path.Substring(endOfName + 1);
if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
{
return await this.Info(request, fanoutRequest, counterName, queryParameters);
}
if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
{
var response = await this.Query(counterName, fanoutRequest, queryParameters);
return Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response);
}
return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName);
}
public async Task GetDimensionValuesWithTimeFilteredProvidesValuesOnlyInTimeRange()
{
var earlyTime = new DateTime(2013, 12, 11, 00, 01, 00); //bucket starttime = 2013, 12, 11,00,00,00, endtime = 2013, 12, 11,00,20,00
var laterTime = new DateTime(2013, 12, 11, 00, 22, 00); //bucket starttime = 2013, 12, 11,00,20,00, endtime = 2013, 12, 11,00,40,00
var dimensionValues = new DimensionSpecification(AnyDimensions);
//create the hit counters
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
dimensionValues[AnyKeys.Dimensions.First().Name] = "early";
counter.Increment(dimensionValues, earlyTime);
dimensionValues[AnyKeys.Dimensions.First().Name] = "later";
counter.Increment(dimensionValues, laterTime);
counter.DataSet.Flush();
//this should return early
var filter = new DimensionSpecification
{
{ ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 00, 00).ToString(Protocol.TimestampStringFormat) },
{ ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 20, 00).ToString(Protocol.TimestampStringFormat) },
};
var values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList();
Assert.AreEqual(1, values.Count);
Assert.AreEqual("early", values[0]);
//should only return later
filter = new DimensionSpecification
{
{ ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 20, 00).ToString(Protocol.TimestampStringFormat) },
{ ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 40, 00).ToString(Protocol.TimestampStringFormat) },
};
values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList();
Assert.AreEqual(1, values.Count);
Assert.AreEqual("later", values[0]);
//should return both later and early
filter = new DimensionSpecification
{
{ ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 00, 00).ToString(Protocol.TimestampStringFormat) },
{ ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 40, 00).ToString(Protocol.TimestampStringFormat) },
};
values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList();
Assert.AreEqual(2, values.Count);
Assert.IsTrue(values.Contains("early"));
Assert.IsTrue(values.Contains("later"));
}
public void CreateKeySetsAllDimensionsProvidedFalseIfSomeDimensionsAreNotProvided()
{
const string anyDim1 = "d1";
const string anyDim2 = "d2";
var dimensionSet = new DimensionSet(new HashSet <Dimension> {
new Dimension(anyDim1), new Dimension(anyDim2)
});
var partialDimensionValues = new DimensionSpecification {
{ anyDim1, "anyVal1" }
};
bool allDimensionsProvided;
dimensionSet.CreateKey(partialDimensionValues, out allDimensionsProvided);
Assert.AreEqual(false, allDimensionsProvided);
}
public async Task GetBucketedDataReturnsEmptyEnumerationIfNoSamplesMatchDimensionValues()
{
var anyTimestamp = DateTime.Now;
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
counter.Increment(AnyDimensions, anyTimestamp);
var dimValues = new DimensionSpecification();
foreach (var d in AnyKeys.Dimensions)
{
dimValues[d.Name] = "no match";
}
Assert.IsFalse(this.BucketedDataQuery(counter, dimValues, 0).Any());
}
public void CreateKeySetsAllDimensionsProvidedTrueIfAllDimensionsAreInDictionary()
{
const string anyDim1 = "d1";
const string anyDim2 = "d2";
var dimensionSet = new DimensionSet(new HashSet <Dimension> {
new Dimension(anyDim1), new Dimension(anyDim2)
});
var allDimensionValues = new DimensionSpecification {
{ anyDim1, "anyVal1" }, { anyDim2, "anyVal2" }
};
bool allDimensionsProvided;
dimensionSet.CreateKey(allDimensionValues, out allDimensionsProvided);
Assert.AreEqual(true, allDimensionsProvided);
}
public async Task GetDimensionValuesForAllDimensionInDataProvidesEmptyStringForEmptyDimensionValue()
{
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
counter.Increment(AnyDimensions);
var otherDimValues = new DimensionSpecification(AnyDimensions);
otherDimValues[AnyKeys.Dimensions.First().Name] = ReservedDimensions.EmptyDimensionValue;
counter.Increment(otherDimValues);
counter.DataSet.Flush();
var values =
counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, new DimensionSpecification()).ToList();
Assert.AreEqual(2, values.Count);
Assert.IsTrue(values.Contains(AnyDimValue));
Assert.IsTrue(values.Contains(string.Empty));
}
public async Task CanRetrieveHistogramMinimumData()
{
var anyTimestamp = DateTime.Now;
var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);
for (int i = 0; i < 100; ++i)
{
counter.AddValue(i, AnyDimensions, anyTimestamp);
}
var dims = new DimensionSpecification(AnyDimensions);
dims[ReservedDimensions.PercentileDimension] = ReservedDimensions.PercentileDimensionValueForMinimum;
var sample = this.CombinedDataQuery(counter, dims);
Assert.AreEqual(sample.MinValue, 0);
}
public void MatchIsFalseIfValuesDiffer()
{
const string anyDim1 = "d1";
const string anyDim2 = "d2";
const string anyDim3 = "d3";
var dimensionSet = new DimensionSet(new HashSet <Dimension>
{
new Dimension(anyDim1),
new Dimension(anyDim2),
new Dimension(anyDim3),
});
var keyValueSet = new DimensionSpecification {
{ anyDim1, "val1" }, { anyDim2, "val2" }, { anyDim3, "val3" }
};
bool allDimensionsProvided;
Key key = dimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);
// Longer values
foreach (var d in dimensionSet.Dimensions)
{
var filterValueSet = new DimensionSpecification(keyValueSet);
filterValueSet[d.Name] = "valAnyOther";
Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided);
Assert.IsFalse(filter.Matches(key));
}
// Same length
foreach (var d in dimensionSet.Dimensions)
{
var filterValueSet = new DimensionSpecification(keyValueSet);
filterValueSet[d.Name] = "valX";
Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided);
Assert.IsFalse(filter.Matches(key));
}
// Shorter length
foreach (var d in dimensionSet.Dimensions)
{
var filterValueSet = new DimensionSpecification(keyValueSet);
filterValueSet[d.Name] = "v";
Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided);
Assert.IsFalse(filter.Matches(key));
}
}
public async Task TimeFilterThrowsArgumentOutOfRangeExceptionIfStartTimeIsSameOrGreaterThanEndTime()
{
var anyTime = DateTime.Now;
var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);
var filter = new DimensionSpecification
{
{ ReservedDimensions.StartTimeDimension, anyTime.ToString() },
{ ReservedDimensions.EndTimeDimension, anyTime.ToString() },
};
Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, filter));
var anyLaterTime = anyTime.Add(new TimeSpan(0, 0, 1));
filter[ReservedDimensions.StartTimeDimension] = anyLaterTime.ToString();
Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, filter));
}
public async Task BatchQueryMultipleQueriesForSameCounterWithDifferentParametersWorksFine()
{
// Fill up the taco truck
var dimensionSet = new DimensionSet(new HashSet<Dimension> {new Dimension("Filling")});
var counter = await this.dataManager.CreateHitCounter("/Tacos", dimensionSet);
var chickenTacos = new DimensionSpecification {{"Filling", "Chicken"}};
var beefTacos = new DimensionSpecification {{"Filling", "Beef"}};
var veggieTacos = new DimensionSpecification {{"Filling", "TOFU"}};
var baconTacos = new DimensionSpecification {{"Filling", "bacon"}};
counter.Increment(100, chickenTacos);
counter.Increment(200, beefTacos);
counter.Increment(300, veggieTacos);
this.dataManager.Flush();
var data = new BatchQueryRequest();
data.Queries.Add(new BatchCounterQuery {CounterName = "/Tacos", UserContext = "TotalTacos"});
data.Queries.Add(new BatchCounterQuery
{
CounterName = "/Tacos",
UserContext = "CluckCluck",
QueryParameters = chickenTacos.Data
});
data.Queries.Add(new BatchCounterQuery
{
CounterName = "/Tacos",
UserContext = "BACON!",
QueryParameters = baconTacos.Data
});
var response = await
this.httpClient.PostAsync(
TestUtils.GetUri(this.server,
RestCommands.BatchQueryCommand,
string.Empty),
GetRequestPayload(data));
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
using (var readerStream = new ReaderStream(await response.Content.ReadAsStreamAsync()))
{
var reader = readerStream.CreateBondedCompactBinaryReader<BatchQueryResponse>();
var responseData = reader.Deserialize();
Assert.AreEqual(3, responseData.Responses.Count);
// unfiltered should have 100+200+300 hitcount
VerifyHitCounter(responseData, "TotalTacos", 600);
//only 100 chicken tacos
VerifyHitCounter(responseData, "CluckCluck", 100);
//sadly, there is no bacon...
VerifyHitCounter(responseData, "BACON!", -1);
}
}
public void SealAndReleaseAreThreadSafe()
{
var filterableDimension = new DimensionSet(new HashSet<Dimension> { new Dimension("thing") });
using (
var filterableBucket =
new DataBucket<InternalHitCount>(filterableDimension,
this.timestamp,
DefaultBucketTimeSpanTicks,
this.currentDirectory,
properties.MemoryStreamManager))
{
var allDims = new DimensionSpecification {{"thing", "thing"}};
Parallel.For(0, 10, (i) => filterableBucket.AddValue(allDims, i));
Parallel.For(0, 100, (i) =>
{
switch (i % 3)
{
case 0:
foreach (var item in filterableBucket.GetMatches(allDims))
{
Assert.IsNotNull(item);
}
break;
case 1:
filterableBucket.AddValue(allDims, 11);
break;
case 2:
filterableBucket.ReleaseData();
break;
}
});
}
}
private void ValidateData(ulong expectedPermutations = 100, ulong expectedData = WrittenValue)
{
this.bucket.Seal();
var uniqueKeys = new HashSet<string>();
var dimensionValues = new DimensionSpecification();
for (var i = 0; i < 10; ++i)
{
for (var j = 0; j < 10; ++j)
{
dimensionValues["one"] = i.ToString();
dimensionValues["two"] = j.ToString();
var keyString =
this.bucket.DimensionSet.KeyToString(this.bucket.DimensionSet.CreateKey(dimensionValues));
uniqueKeys.Add(keyString);
var matchData = this.bucket.GetMatches(dimensionValues).ToList()[0];
Assert.AreEqual(1, matchData.DataCount);
Assert.AreEqual(expectedData, matchData.Data.HitCount);
}
}
Assert.AreEqual(expectedPermutations, (ulong)uniqueKeys.Count);
}
/// <summary>
/// Increment the counter for the provided dimensions.
/// </summary>
/// <param name="amount">Amount to increment the counter by.</param>
/// <param name="dims">Full set of dimension values for the counter.</param>
/// <param name="timestamp">Timestamp to use for the data point.</param>
public void Increment(long amount, DimensionSpecification dims, DateTime timestamp)
{
this.hitCounter.AddValue(amount, dims, timestamp);
}
private void WriteRoundedSignificantDigitsValue(long value, DimensionSpecification dims, DateTime timestamp)
{
if (this.roundingFactor > 0 && value > this.minimumRoundingValue)
{
long factor = 0;
while (value > this.minimumRoundingValue)
{
factor += 1;
var lastDigit = value % 10;
if (lastDigit >= 5)
{
value += (10 - lastDigit);
}
value /= 10;
}
value *= (long)Math.Pow(10, factor);
}
this.histogram.AddValue(value, dims, timestamp);
}
private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters)
{
var counterInfo = new CounterInfo
{
Name = counter.Name,
Type = counter.Type,
StartTime = counter.StartTime.ToMillisecondTimestamp(),
EndTime = counter.EndTime.ToMillisecondTimestamp(),
Dimensions = counter.Dimensions.ToList(),
DimensionValues = null, // null this out by default to avoid response bloat.
};
// Queries for dimension values will come with a 'dimension=pattern' query parameter.
// Dimension values can be further filtered with '<dimensionName>=pattern'
string dimensionPattern;
if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern))
{
counterInfo.DimensionValues = new Dictionary<string, ISet<string>>();
// We want to be able to filter dimension values by time (and only time)
var dimensionQuery = new DimensionSpecification();
string timeValue;
if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue))
{
timeValue = MinimumStartTime;
}
dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue;
if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue))
{
timeValue = MaximumEndTime;
}
dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue;
foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern)))
{
string filterPattern;
if (queryParameters.TryGetValue(dimensionPattern, out filterPattern))
{
counterInfo.AddDimensionValues(dim,
counter.GetDimensionValues(dim, dimensionQuery)
.Where(dimensionValue =>
dimensionValue.MatchGlob(filterPattern)));
}
else
{
counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery));
}
}
}
return counterInfo;
}
private DimensionSpecification ProcessQueryParameters(DimensionSpecification queryParameters,
out QuerySpecification querySpec)
{
var filterDimensions = ExtractQuerySpec(queryParameters, out querySpec);
if (querySpec.QueryType == QueryType.Average && !this.DataSet.SupportsAverageQuery)
{
querySpec.QueryType = QueryType.Normal;
}
else if (querySpec.QueryType == QueryType.Percentile && !this.DataSet.SupportsPercentileQuery)
{
querySpec.QueryType = QueryType.Normal;
}
return filterDimensions;
}
public void AddValue(long value, DimensionSpecification dims, DateTime timestamp)
{
this.writeValue(value, dims, timestamp);
}
/// <summary>
/// Get all values for a particular dimension.
/// </summary>
/// <param name="dimensionName">Name of the dimension.</param>
/// <param name="filterDims">Filter dimensions</param>
/// <returns>An enumerator of values for the dimension (may be empty).</returns>
public IEnumerable<string> GetDimensionValues(string dimensionName, DimensionSpecification filterDims)
{
if (filterDims == null)
{
throw new ArgumentNullException("filterDims");
}
return this.DataSet.GetDimensionValues(dimensionName, filterDims);
}
/// <summary>
/// Execute a query against the counter data.
/// </summary>
/// <param name="queryParameters">Query parameters.</param>
/// <returns>An enumeration of <see cref="DataSample"/>s matching the query parameters.</returns>
public IEnumerable<DataSample> Query(DimensionSpecification queryParameters)
{
QuerySpecification querySpec;
if (queryParameters == null)
{
throw new ArgumentNullException("queryParameters");
}
var filterDims = this.ProcessQueryParameters(queryParameters, out querySpec);
if (filterDims == null)
{
return null;
}
return this.DataSet.QueryData(filterDims, querySpec);
}
public void CombinedBucketsAreMergedCorrectly()
{
var ts1 = new DateTime(2014, 05, 10, 0, 0, 0);
var ts2 = new DateTime(2014, 05, 10, 0, 5, 0);
var sharedDimensions = new DimensionSpecification { {"one", "a1"}, {"two", "a2"}};
var bucket1Dimensions = new DimensionSpecification { {"one", "b1"}, {"two", "b2"}};
var bucket2Dimensions = new DimensionSpecification { {"one", "c1"}, {"two", "c2"}};
var bucket1 =
new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket1.AddValue(sharedDimensions, 867);
bucket1.AddValue(bucket1Dimensions, 5309);
var bucket2 =
new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket2.AddValue(sharedDimensions, 867);
bucket2.AddValue(bucket2Dimensions, 42);
bucket1.Seal();
bucket2.Seal();
var bucket3 =
new DataBucket<InternalHitCount>(new[] {bucket1, bucket2},
new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(10).Ticks, null,
this.properties.MemoryStreamManager);
var match = bucket3.GetMatches(bucket1Dimensions).First().Data;
Assert.AreEqual((ulong)5309, match.HitCount);
match = bucket3.GetMatches(bucket2Dimensions).First().Data;
Assert.AreEqual((ulong)42, match.HitCount);
match = bucket3.GetMatches(sharedDimensions).First().Data;
Assert.AreEqual((ulong)867 * 2, match.HitCount);
bucket1.Dispose();
bucket2.Dispose();
bucket3.Dispose();
}
private void WriteRandomTestData(int maxValue, TestKeyedDataStore store)
{
Parallel.For(0, maxValue,
i =>
{
var dims = new DimensionSpecification();
this.GenerateRandomKey(dims);
store.AddValue(dims, i);
});
}
private void WriteSomeData()
{
var dimensionValues = new DimensionSpecification();
for (var i = 0; i < 10; ++i)
{
for (var j = 0; j < 10; ++j)
{
dimensionValues["one"] = i.ToString();
dimensionValues["two"] = j.ToString();
this.bucket.AddValue(dimensionValues, WrittenValue);
}
}
}
public void SplitByDimensionWithFiltersWorksProperly()
{
var filterableDimension = new DimensionSet(new HashSet<Dimension> {new Dimension("thing"), new Dimension("meat")});
using (
var filterableBucket =
new DataBucket<InternalHitCount>(filterableDimension, this.timestamp,
DefaultBucketTimeSpanTicks,
this.currentDirectory,
this.properties.MemoryStreamManager))
{
var queryDimensions = new DimensionSpecification();
queryDimensions["thing"] = "thingOne";
queryDimensions["meat"] = "bacon";
filterableBucket.AddValue(queryDimensions, 100);
queryDimensions["thing"] = "thingTwo";
queryDimensions["meat"] = "pepperoni";
filterableBucket.AddValue(queryDimensions, 200);
filterableBucket.Seal();
// thingOne and thingTwo will match with no filter
Assert.AreEqual(2,
filterableBucket.GetMatchesSplitByDimension(new DimensionSpecification(), "thing")
.Sum(match => match.DataCount));
// only thingOne matches bacon
var bestMatchFilter = new DimensionSpecification {{"meat", "bacon"}};
Assert.AreEqual(1,
filterableBucket.GetMatchesSplitByDimension(bestMatchFilter, "thing")
.Sum(match => match.DataCount));
}
}
internal static DimensionSpecification ExtractQuerySpec(DimensionSpecification queryParameters,
out QuerySpecification querySpec)
{
var filterDimensions = new DimensionSpecification();
querySpec = new QuerySpecification();
foreach (var param in queryParameters)
{
if (string.Equals(param.Key, ReservedDimensions.DimensionDimension, StringComparison.OrdinalIgnoreCase))
{
querySpec.CrossQueryDimension = param.Value;
// Ensure that split dimension is not also sent a filter dimension
if (queryParameters.ContainsKey(param.Value))
{
return null;
}
}
else if (string.Equals(param.Key, ReservedDimensions.AggregateSamplesDimension, StringComparison.OrdinalIgnoreCase))
{
querySpec.Combine = Convert.ToBoolean(param.Value);
}
else if (string.Equals(param.Key, ReservedDimensions.PercentileDimension, StringComparison.OrdinalIgnoreCase))
{
if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForAverage,
StringComparison.OrdinalIgnoreCase))
{
querySpec.QueryType = QueryType.Average;
}
else if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForMaximum,
StringComparison.OrdinalIgnoreCase))
{
querySpec.QueryType = QueryType.Maximum;
}
else if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForMinimum,
StringComparison.OrdinalIgnoreCase))
{
querySpec.QueryType = QueryType.Minimum;
}
else
{
querySpec.QueryType = QueryType.Percentile;
querySpec.Percentile = double.Parse(param.Value);
}
}
else
{
filterDimensions.Add(param.Key, param.Value);
}
}
return filterDimensions;
}
public void AddValue(long value, DimensionSpecification dims)
{
this.AddValue(value, dims, DateTime.Now);
}
/// <summary>
/// Increment the counter by one for the provided dimensions using the current time for the data point.
/// </summary>
/// <param name="dims">Full set of dimension values for the counter.</param>
public void Increment(DimensionSpecification dims)
{
this.Increment(dims, DateTime.Now);
}
private void WriteUnroundedValue(long value, DimensionSpecification dims, DateTime timestamp)
{
this.histogram.AddValue(value, dims, timestamp);
}
/// <summary>
/// Increment the counter by one for the provided dimensions.
/// </summary>
/// <param name="dims">Full set of dimension values for the counter.</param>
/// <param name="timestamp">Timestamp to use for the data point.</param>
public void Increment(DimensionSpecification dims, DateTime timestamp)
{
this.Increment(1, dims, timestamp);
}
private void WriteRoundedSizeValue(long value, DimensionSpecification dims, DateTime timestamp)
{
if (value < 0)
{
value = 0;
}
if (value > this.roundingFactor)
{
long remainder = value % this.roundingFactor;
if (remainder >= (this.roundingFactor / 2))
{
remainder = -remainder; // will roll up
}
value = value - remainder;
}
this.histogram.AddValue(value, dims, timestamp);
}
/// <summary>
/// Increment the counter for the provided dimensions using the current time for the data point.
/// </summary>
/// <param name="amount">Amount to increment the counter by.</param>
/// <param name="dims">Full set of dimension values for the counter.</param>
public void Increment(long amount, DimensionSpecification dims)
{
this.Increment(amount, dims, DateTime.Now);
}
private void GenerateRandomKey(DimensionSpecification dims)
{
dims.Clear();
foreach (var d in this.dimensionSet.Dimensions)
{
dims.Add(d.Name, GetRandom(MaxDimensionValue).ToString());
}
}
private static void RunOperation(Counter counter, CounterWriteOperation op)
{
var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow
? DateTime.UtcNow
: op.Timestamp.ToDateTime();
var dims = new DimensionSpecification(op.DimensionValues);
var hitCounter = counter as HitCounter;
var histogramCounter = counter as HistogramCounter;
if (hitCounter != null)
{
hitCounter.Increment(op.Value * op.Count, dims, timestamp);
}
else
{
// It would be nice to direct-inject multiple values at once, but the APIs don't currently
// support this. It's a reasonable amount of work to fix this and unknown whether folks will use this
// a lot at this time.
for (var i = 0; i < op.Count; ++i)
{
histogramCounter.AddValue(op.Value, dims, timestamp);
}
}
}
