public async Task GetDimensionValuesForAllDimensionInDataProvidesExpectedValues() { const int NumValuesPerDim = 100; var dimensionValues = new DimensionSpecification(); var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); for (var i = 0; i < NumValuesPerDim; ++i) { foreach (var d in AnyKeys.Dimensions) { dimensionValues[d.Name] = d.Name + i; } counter.Increment(dimensionValues); } counter.DataSet.Flush(); foreach (var d in AnyKeys.Dimensions) { var values = counter.GetDimensionValues(d.Name, new DimensionSpecification()).ToList(); Assert.AreEqual(NumValuesPerDim, values.Count); for (var i = 0; i < NumValuesPerDim; ++i) { var expected = d.Name + i; Assert.IsTrue(values.Contains(expected)); } } }
public void SetUp() { this.dimensionSet = new DimensionSet(new HashSet<Dimension> { new Dimension("foo"), new Dimension("bar"), new Dimension("baz"), }); dimensions = new DimensionSpecification(); this.storagePath = Path.Combine(Environment.CurrentDirectory, "msTemp"); if (Directory.Exists(this.storagePath)) { Directory.Delete(this.storagePath, true); } Directory.CreateDirectory(this.storagePath); foreach (var d in this.dimensionSet.Dimensions) { dimensions[d.Name] = DimValue; } this.properties = new MockSharedDataSetProperties(); }
private static void RunOperation(Counter counter, CounterWriteOperation op) { var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow ? DateTime.UtcNow : op.Timestamp.ToDateTime(); var dims = new DimensionSpecification(op.DimensionValues); var hitCounter = counter as HitCounter; var histogramCounter = counter as HistogramCounter; if (hitCounter != null) { hitCounter.Increment(op.Value * op.Count, dims, timestamp); } else { // It would be nice to direct-inject multiple values at once, but the APIs don't currently // support this. It's a reasonable amount of work to fix this and unknown whether folks will use this // a lot at this time. for (var i = 0; i < op.Count; ++i) { histogramCounter.AddValue(op.Value, dims, timestamp); } } }
public async Task HistogramPercentileValuesAreCorrectlyDistributedByBucket() { var bucket1Timestamp = DateTime.Now; var bucket2Timestamp = new DateTime(bucket1Timestamp.Ticks + this.defaultCompactionConfig.DefaultBucketTicks * 2); var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys); for (int i = 0; i < 100; ++i) { counter.AddValue(i, AnyDimensions, bucket1Timestamp); } for (int i = 0; i < 100; ++i) { counter.AddValue(i * 10, AnyDimensions, bucket2Timestamp); } var dims = new DimensionSpecification(AnyDimensions); for (int i = 1; i < 100; ++i) { dims[ReservedDimensions.PercentileDimension] = i.ToString(); var bucketedSamples = this.BucketedDataQuery(counter, dims, 2).ToList(); var sample1 = bucketedSamples[1]; Assert.AreEqual(sample1.Percentile, i); Assert.AreEqual(sample1.SampleCount, (ulong)100); Assert.AreEqual(sample1.PercentileValue, i - 1); var sample2 = bucketedSamples[0]; Assert.AreEqual(sample2.Percentile, i); Assert.AreEqual(sample2.SampleCount, (ulong)100); Assert.AreEqual(sample2.PercentileValue, (i - 1) * 10); } }
public override void Setup() { this.dataManager.CompactionConfiguration = new DataCompactionConfiguration(new[] { new DataIntervalConfiguration(TimeSpan.FromMinutes(5), TimeSpan.MaxValue), }); this.dataManager.MaximumDataAge = TimeSpan.Zero; // Don't want data getting deleted here. // Make a counter and write some stuff. this.dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension(AnyDimension), }); var counter = this.dataManager.CreateHitCounter(AnyCounter, this.dimensionSet).Result; var dims = new DimensionSpecification(); var timestamp = DateTime.Now; dims[AnyDimension] = "a"; counter.Increment(dims, timestamp); dims[AnyDimension] = "b"; counter.Increment(2, dims, timestamp); dims[AnyDimension] = "c"; counter.Increment(dims, timestamp); // We need to force a seal by setting some value waaaay in the future. counter.Increment(dims, timestamp.AddYears(10)); this.memoryStreamManager = new RecyclableMemoryStreamManager(1 << 17, 1 << 20, 1 << 24); }
public void MatchIsFalseIfFilterHasMoreOrLessValuesThanKey() { const string anyDim1 = "d1"; const string anyDim2 = "d2"; const string anyDim3 = "d3"; var dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension(anyDim1), new Dimension(anyDim2), new Dimension(anyDim3), }); var keyValueSet = new DimensionSpecification { { anyDim1, "val1" }, { anyDim2, "val2" }, { anyDim3, "val3" } }; bool allDimensionsProvided; Key key = dimensionSet.CreateKey(keyValueSet, out allDimensionsProvided); var smallerDimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension(anyDim1), new Dimension(anyDim2) }); Key smallerFilter = smallerDimensionSet.CreateKey(keyValueSet, out allDimensionsProvided); Assert.IsFalse(smallerFilter.Matches(key)); var largerDimensionSet = new DimensionSet(new HashSet <Dimension>(dimensionSet.Dimensions) { new Dimension ("anotherDim") }); Key largerFilter = largerDimensionSet.CreateKey(keyValueSet, out allDimensionsProvided); Assert.IsFalse(largerFilter.Matches(key)); }
/// <summary> /// Safely increment a HitCounter by one using the given timestamp. /// If the counter is null no operation will be performed. /// </summary> /// <param name="counter">Counter to increment.</param> /// <param name="dims">Dimensions to use for incrementing.</param> /// <param name="timestamp">Timestamp for written value.</param> public static void SafeIncrement(this HitCounter counter, DimensionSpecification dims, DateTime timestamp) { if (counter != null) { counter.Increment(1, dims, timestamp); } }
public void SetUp() { this.dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension("foo"), new Dimension("bar"), new Dimension("baz"), }); dimensions = new DimensionSpecification(); this.storagePath = Path.Combine(Environment.CurrentDirectory, "msTemp"); if (Directory.Exists(this.storagePath)) { Directory.Delete(this.storagePath, true); } Directory.CreateDirectory(this.storagePath); foreach (var d in this.dimensionSet.Dimensions) { dimensions[d.Name] = DimValue; } this.properties = new MockSharedDataSetProperties(); }
/// <summary> /// Ask the counter aggregator to do all percentile computation as opposed to any metric system server. This will /// strip out the 'percentile=xxx' parameter in the request and return the filtered parameter list. If a /// percentile request was present, this will apply the percentile calculation in the final calculation of this object /// </summary> /// <param name="queryParameters">Parameters to filter</param> /// <returns>Filtered parameter set</returns> public IDictionary<string, string> ApplyPercentileCalculationAggregation(IDictionary<string, string> queryParameters) { if (queryParameters == null) { return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); } string percentileParameter; double percentileRequested; var dimensionSpec = new DimensionSpecification(queryParameters); if (!dimensionSpec.TryGetValue(ReservedDimensions.PercentileDimension, out percentileParameter) || !double.TryParse(percentileParameter, out percentileRequested) || percentileRequested < 0 || percentileRequested > 100) { this.shouldFilterToFinalPercentile = false; return queryParameters; } this.shouldFilterToFinalPercentile = true; this.requestedPercentile = percentileRequested; var retVal = new Dictionary<string, string>(dimensionSpec, StringComparer.OrdinalIgnoreCase); retVal.Remove(ReservedDimensions.PercentileDimension); return retVal; }
public override void Setup() { this.dataManager.CompactionConfiguration = new DataCompactionConfiguration(new[] { new DataIntervalConfiguration(TimeSpan.FromMinutes(5), TimeSpan.MaxValue), }); this.dataManager.MaximumDataAge = TimeSpan.Zero; // Don't want data getting deleted here. // Make a counter and write some stuff. this.dimensionSet = new DimensionSet(new HashSet<Dimension> { new Dimension(AnyDimension), }); var counter = this.dataManager.CreateHitCounter(AnyCounter, this.dimensionSet).Result; var dims = new DimensionSpecification(); var timestamp = DateTime.Now; dims[AnyDimension] = "a"; counter.Increment(dims, timestamp); dims[AnyDimension] = "b"; counter.Increment(2, dims, timestamp); dims[AnyDimension] = "c"; counter.Increment(dims, timestamp); // We need to force a seal by setting some value waaaay in the future. counter.Increment(dims, timestamp.AddYears(10)); this.memoryStreamManager = new RecyclableMemoryStreamManager(1 << 17, 1 << 20, 1 << 24); }
/// <summary> /// Safely increment a HitCounter by the given amount using the current time. /// If the counter is null no operation will be performed. /// </summary> /// <param name="counter">Counter to increment.</param> /// <param name="amount">Amount to increment by.</param> /// <param name="dims">Dimensions to use for incrementing.</param> public static void SafeIncrement(this HitCounter counter, long amount, DimensionSpecification dims) { if (counter != null) { counter.Increment(amount, dims, DateTime.Now); } }
public async Task CanRetrieveHistogramPercentileData() { var anyTimestamp = DateTime.Now; var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys); for (int i = 0; i < 100; ++i) { counter.AddValue(i, AnyDimensions, anyTimestamp); } var dims = new DimensionSpecification(AnyDimensions); for (int i = 1; i < 100; ++i) { dims[ReservedDimensions.PercentileDimension] = i.ToString(); var sample = this.CombinedDataQuery(counter, dims); Assert.AreEqual(sample.Percentile, i); Assert.AreEqual(sample.SampleCount, (ulong)100); Assert.AreEqual(sample.PercentileValue, i - 1); } dims[ReservedDimensions.PercentileDimension] = "0"; var sample2 = this.CombinedDataQuery(counter, dims); Assert.AreEqual(sample2.Percentile, 0); Assert.AreEqual(sample2.SampleCount, (ulong)100); Assert.AreEqual(sample2.PercentileValue, 0); dims[ReservedDimensions.PercentileDimension] = "100"; sample2 = this.CombinedDataQuery(counter, dims); Assert.AreEqual(sample2.Percentile, 100); Assert.AreEqual(sample2.SampleCount, (ulong)100); Assert.AreEqual(sample2.PercentileValue, 99); }
/// <summary> /// Ask the counter aggregator to do all percentile computation as opposed to any metric system server. This will /// strip out the 'percentile=xxx' parameter in the request and return the filtered parameter list. If a /// percentile request was present, this will apply the percentile calculation in the final calculation of this object /// </summary> /// <param name="queryParameters">Parameters to filter</param> /// <returns>Filtered parameter set</returns> public IDictionary <string, string> ApplyPercentileCalculationAggregation(IDictionary <string, string> queryParameters) { if (queryParameters == null) { return(new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase)); } string percentileParameter; double percentileRequested; var dimensionSpec = new DimensionSpecification(queryParameters); if (!dimensionSpec.TryGetValue(ReservedDimensions.PercentileDimension, out percentileParameter) || !double.TryParse(percentileParameter, out percentileRequested) || percentileRequested < 0 || percentileRequested > 100) { this.shouldFilterToFinalPercentile = false; return(queryParameters); } this.shouldFilterToFinalPercentile = true; this.requestedPercentile = percentileRequested; var retVal = new Dictionary <string, string>(dimensionSpec, StringComparer.OrdinalIgnoreCase); retVal.Remove(ReservedDimensions.PercentileDimension); return(retVal); }
private void GenerateRandomKey(DimensionSpecification dims) { dims.Clear(); foreach (var d in this.dimensionSet.Dimensions) { dims.Add(d.Name, GetRandom(MaxDimensionValue).ToString()); } }
private void WriteComplexSampleData() { var rng = new Random(); histogramKeys = new int[BucketCount][][]; totalCountBySample = new Dictionary <long, uint>(); this.dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension("one"), new Dimension("two"), }); this.sharedProperties.CompactionConfiguration = new DataCompactionConfiguration(new[] { new DataIntervalConfiguration(TimeSpan.FromMinutes(1), TimeSpan.MaxValue), }); this.histogramDataSet = new DataSet <InternalHistogram>("/AggTest", null, this.dimensionSet, this.sharedProperties); var dimSpec = new DimensionSpecification(); var startTime = new DateTime(2014, 7, 4, 0, 0, 0, DateTimeKind.Utc); // no DateTimeKind.MERKUH? Sigh. for (var i = 0; i < BucketCount; ++i) { histogramKeys[i] = new int[DimensionOneCount][]; var ts = startTime + TimeSpan.FromTicks(i * this.sharedProperties.CompactionConfiguration.DefaultBucketTicks); for (var d1 = 0; d1 < DimensionOneCount; ++d1) { histogramKeys[i][d1] = new int[DimensionTwoCount]; dimSpec["one"] = d1.ToString(); for (var d2 = 0; d2 < DimensionTwoCount; ++d2) { dimSpec["two"] = d2.ToString(); var val = rng.Next(MaxValue); this.histogramDataSet.AddValue(val, dimSpec, ts); histogramKeys[i][d1][d2] = val; if (!totalCountBySample.ContainsKey(val)) { totalCountBySample[val] = 0; } totalCountBySample[val]++; } } } this.histogramDataSet.Flush(); }
public async Task GetDimensionValuesForUnknownFilterDimensionDoesNotThrow() { var filterDims = new DimensionSpecification { { "UnknownDim", "UnKnown" } }; var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); // if the following call throws test will fail counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filterDims); }
private void WriteRandomTestData(int maxValue, TestKeyedDataStore store) { Parallel.For(0, maxValue, i => { var dims = new DimensionSpecification(); this.GenerateRandomKey(dims); store.AddValue(dims, i); }); }
public void SetUp() { this.dimensionSet = new DimensionSet(new HashSet<Dimension>()); this.dimensionSpec = new DimensionSpecification(); this.sharedProperties = new MockSharedDataSetProperties(); this.bucketSpan = this.sharedProperties.CompactionConfiguration.Default.Interval; this.firstBucketTimestamp = DateTime.UtcNow; this.secondBucketTimestamp = this.firstBucketTimestamp + this.bucketSpan; this.thirdBucketTimestamp = this.secondBucketTimestamp + this.bucketSpan; }
public async Task GetDimensionValuesWithAllDimensionsFilledThrowsArgumentException() { var dimValues = new DimensionSpecification(AnyDimensions); var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); Assert.Throws <ArgumentException>( () => counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, dimValues) .ToList()); }
public void SetUp() { this.dimensionSet = new DimensionSet(new HashSet <Dimension>()); this.dimensionSpec = new DimensionSpecification(); this.sharedProperties = new MockSharedDataSetProperties(); this.bucketSpan = this.sharedProperties.CompactionConfiguration.Default.Interval; this.firstBucketTimestamp = DateTime.UtcNow; this.secondBucketTimestamp = this.firstBucketTimestamp + this.bucketSpan; this.thirdBucketTimestamp = this.secondBucketTimestamp + this.bucketSpan; }
private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters) { var counterInfo = new CounterInfo { Name = counter.Name, Type = counter.Type, StartTime = counter.StartTime.ToMillisecondTimestamp(), EndTime = counter.EndTime.ToMillisecondTimestamp(), Dimensions = counter.Dimensions.ToList(), DimensionValues = null, // null this out by default to avoid response bloat. }; // Queries for dimension values will come with a 'dimension=pattern' query parameter. // Dimension values can be further filtered with '<dimensionName>=pattern' string dimensionPattern; if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern)) { counterInfo.DimensionValues = new Dictionary <string, ISet <string> >(); // We want to be able to filter dimension values by time (and only time) var dimensionQuery = new DimensionSpecification(); string timeValue; if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue)) { timeValue = MinimumStartTime; } dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue; if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue)) { timeValue = MaximumEndTime; } dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue; foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern))) { string filterPattern; if (queryParameters.TryGetValue(dimensionPattern, out filterPattern)) { counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery) .Where(dimensionValue => dimensionValue.MatchGlob(filterPattern))); } else { counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery)); } } } return(counterInfo); }
public async Task AddingDataWithEmptyKeyValuesIsValid() { var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); var dimensions = new DimensionSpecification(); foreach (var d in AnyKeys.Dimensions) { dimensions[d.Name] = string.Empty; } counter.Increment(dimensions); }
public override async Task <Response> ProcessRequest(Request request) { var fanoutRequest = request.HasInputBody ? await request.ReadInputBody <TieredRequest>() : null; if (request.HasInputBody && fanoutRequest == null) { // This indicates failed deserialization return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body")); } // aggregation servers should fanout automagically. we honor filters for machine function and datacenter. if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation) { if (fanoutRequest == null) { fanoutRequest = new TieredRequest(); } fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList(); } var queryParameters = new DimensionSpecification(request.QueryParameters); var endOfName = request.Path.LastIndexOf('/'); if (endOfName < 0) { return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided.")); } var counterName = request.Path.Substring(0, endOfName); if (string.IsNullOrEmpty(counterName)) { return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided.")); } var commandName = request.Path.Substring(endOfName + 1); if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { return(await this.Info(request, fanoutRequest, counterName, queryParameters)); } if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { var response = await this.Query(counterName, fanoutRequest, queryParameters); return(Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response)); } return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName)); }
public async Task GetBucketedDataCombinesDataAcrossDimensionValuesIfFilterValueIsNotProvided() { var anyTimestamp = DateTime.Now; const string FirstDimension = "1st"; const string SecondDimension = "2nd"; const string ThirdDimension = "3rd"; var dimensions = new DimensionSet(new HashSet <Dimension> { new Dimension(FirstDimension), new Dimension(SecondDimension), new Dimension(ThirdDimension) }); var counter = await this.dataManager.CreateHitCounter(AnyCounterName, dimensions); var dimValues = new DimensionSpecification(); for (int i = 0; i < 10; ++i) { dimValues[FirstDimension] = i.ToString(); for (int j = 0; j < 10; ++j) { dimValues[SecondDimension] = j.ToString(); for (int k = 0; k < 10; ++k) { dimValues[ThirdDimension] = k.ToString(); counter.Increment(dimValues, anyTimestamp); } } } var sample = this.BucketedDataQuery(counter, new DimensionSpecification()).First(); Assert.AreEqual(DataSampleType.HitCount, sample.SampleType); Assert.AreEqual((ulong)1000, sample.HitCount); dimValues.Clear(); dimValues[FirstDimension] = "8"; // take one tenth of things by filter. sample = this.BucketedDataQuery(counter, dimValues).First(); Assert.AreEqual((ulong)100, sample.HitCount); dimValues[SecondDimension] = "6"; sample = this.BucketedDataQuery(counter, dimValues).First(); Assert.AreEqual((ulong)10, sample.HitCount); dimValues[ThirdDimension] = "7"; sample = this.BucketedDataQuery(counter, dimValues).First(); Assert.AreEqual((ulong)1, sample.HitCount); }
public void CombinedBucketsAreMergedCorrectly() { var ts1 = new DateTime(2014, 05, 10, 0, 0, 0); var ts2 = new DateTime(2014, 05, 10, 0, 5, 0); var sharedDimensions = new DimensionSpecification { { "one", "a1" }, { "two", "a2" } }; var bucket1Dimensions = new DimensionSpecification { { "one", "b1" }, { "two", "b2" } }; var bucket2Dimensions = new DimensionSpecification { { "one", "c1" }, { "two", "c2" } }; var bucket1 = new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket1.AddValue(sharedDimensions, 867); bucket1.AddValue(bucket1Dimensions, 5309); var bucket2 = new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket2.AddValue(sharedDimensions, 867); bucket2.AddValue(bucket2Dimensions, 42); bucket1.Seal(); bucket2.Seal(); var bucket3 = new DataBucket <InternalHitCount>(new[] { bucket1, bucket2 }, new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(10).Ticks, null, this.properties.MemoryStreamManager); var match = bucket3.GetMatches(bucket1Dimensions).First().Data; Assert.AreEqual((ulong)5309, match.HitCount); match = bucket3.GetMatches(bucket2Dimensions).First().Data; Assert.AreEqual((ulong)42, match.HitCount); match = bucket3.GetMatches(sharedDimensions).First().Data; Assert.AreEqual((ulong)867 * 2, match.HitCount); bucket1.Dispose(); bucket2.Dispose(); bucket3.Dispose(); }
public async Task InvalidNumericHistogramValueThrowsArgumentOutOfRangeException() { var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys); var dims = new DimensionSpecification(AnyDimensions); dims[ReservedDimensions.PercentileDimension] = "-1"; Assert.Throws <ArgumentOutOfRangeException>(() => this.CombinedDataQuery(counter, dims)); Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, dims)); dims[ReservedDimensions.PercentileDimension] = "100.1"; Assert.Throws <ArgumentOutOfRangeException>(() => this.CombinedDataQuery(counter, dims)); Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, dims)); }
private void WriteSomeData() { var dimensionValues = new DimensionSpecification(); for (var i = 0; i < 10; ++i) { for (var j = 0; j < 10; ++j) { dimensionValues["one"] = i.ToString(); dimensionValues["two"] = j.ToString(); this.bucket.AddValue(dimensionValues, WrittenValue); } } }
private DataSample CombinedDataQuery(Counter counter, DimensionSpecification queryParams) { if (queryParams != null) { queryParams[ReservedDimensions.AggregateSamplesDimension] = "true"; } counter.DataSet.Flush(); var queryResults = counter.Query(queryParams).ToList(); queryParams.Remove(ReservedDimensions.AggregateSamplesDimension); return(queryResults.Count > 0 ? queryResults[0] : null); }
public override async Task<Response> ProcessRequest(Request request) { var fanoutRequest = request.HasInputBody ? await request.ReadInputBody<TieredRequest>() : null; if (request.HasInputBody && fanoutRequest == null) { // This indicates failed deserialization return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body"); } // aggregation servers should fanout automagically. we honor filters for machine function and datacenter. if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation) { if (fanoutRequest == null) { fanoutRequest = new TieredRequest(); } fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList(); } var queryParameters = new DimensionSpecification(request.QueryParameters); var endOfName = request.Path.LastIndexOf('/'); if (endOfName < 0) { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided."); } var counterName = request.Path.Substring(0, endOfName); if (string.IsNullOrEmpty(counterName)) { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided."); } var commandName = request.Path.Substring(endOfName + 1); if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { return await this.Info(request, fanoutRequest, counterName, queryParameters); } if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { var response = await this.Query(counterName, fanoutRequest, queryParameters); return Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response); } return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName); }
public async Task GetDimensionValuesWithTimeFilteredProvidesValuesOnlyInTimeRange() { var earlyTime = new DateTime(2013, 12, 11, 00, 01, 00); //bucket starttime = 2013, 12, 11,00,00,00, endtime = 2013, 12, 11,00,20,00 var laterTime = new DateTime(2013, 12, 11, 00, 22, 00); //bucket starttime = 2013, 12, 11,00,20,00, endtime = 2013, 12, 11,00,40,00 var dimensionValues = new DimensionSpecification(AnyDimensions); //create the hit counters var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); dimensionValues[AnyKeys.Dimensions.First().Name] = "early"; counter.Increment(dimensionValues, earlyTime); dimensionValues[AnyKeys.Dimensions.First().Name] = "later"; counter.Increment(dimensionValues, laterTime); counter.DataSet.Flush(); //this should return early var filter = new DimensionSpecification { { ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 00, 00).ToString(Protocol.TimestampStringFormat) }, { ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 20, 00).ToString(Protocol.TimestampStringFormat) }, }; var values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList(); Assert.AreEqual(1, values.Count); Assert.AreEqual("early", values[0]); //should only return later filter = new DimensionSpecification { { ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 20, 00).ToString(Protocol.TimestampStringFormat) }, { ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 40, 00).ToString(Protocol.TimestampStringFormat) }, }; values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList(); Assert.AreEqual(1, values.Count); Assert.AreEqual("later", values[0]); //should return both later and early filter = new DimensionSpecification { { ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 00, 00).ToString(Protocol.TimestampStringFormat) }, { ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 40, 00).ToString(Protocol.TimestampStringFormat) }, }; values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList(); Assert.AreEqual(2, values.Count); Assert.IsTrue(values.Contains("early")); Assert.IsTrue(values.Contains("later")); }
public void CreateKeySetsAllDimensionsProvidedFalseIfSomeDimensionsAreNotProvided() { const string anyDim1 = "d1"; const string anyDim2 = "d2"; var dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension(anyDim1), new Dimension(anyDim2) }); var partialDimensionValues = new DimensionSpecification { { anyDim1, "anyVal1" } }; bool allDimensionsProvided; dimensionSet.CreateKey(partialDimensionValues, out allDimensionsProvided); Assert.AreEqual(false, allDimensionsProvided); }
public async Task GetBucketedDataReturnsEmptyEnumerationIfNoSamplesMatchDimensionValues() { var anyTimestamp = DateTime.Now; var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); counter.Increment(AnyDimensions, anyTimestamp); var dimValues = new DimensionSpecification(); foreach (var d in AnyKeys.Dimensions) { dimValues[d.Name] = "no match"; } Assert.IsFalse(this.BucketedDataQuery(counter, dimValues, 0).Any()); }
public void CreateKeySetsAllDimensionsProvidedTrueIfAllDimensionsAreInDictionary() { const string anyDim1 = "d1"; const string anyDim2 = "d2"; var dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension(anyDim1), new Dimension(anyDim2) }); var allDimensionValues = new DimensionSpecification { { anyDim1, "anyVal1" }, { anyDim2, "anyVal2" } }; bool allDimensionsProvided; dimensionSet.CreateKey(allDimensionValues, out allDimensionsProvided); Assert.AreEqual(true, allDimensionsProvided); }
public async Task GetDimensionValuesForAllDimensionInDataProvidesEmptyStringForEmptyDimensionValue() { var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); counter.Increment(AnyDimensions); var otherDimValues = new DimensionSpecification(AnyDimensions); otherDimValues[AnyKeys.Dimensions.First().Name] = ReservedDimensions.EmptyDimensionValue; counter.Increment(otherDimValues); counter.DataSet.Flush(); var values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, new DimensionSpecification()).ToList(); Assert.AreEqual(2, values.Count); Assert.IsTrue(values.Contains(AnyDimValue)); Assert.IsTrue(values.Contains(string.Empty)); }
public async Task CanRetrieveHistogramMinimumData() { var anyTimestamp = DateTime.Now; var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys); for (int i = 0; i < 100; ++i) { counter.AddValue(i, AnyDimensions, anyTimestamp); } var dims = new DimensionSpecification(AnyDimensions); dims[ReservedDimensions.PercentileDimension] = ReservedDimensions.PercentileDimensionValueForMinimum; var sample = this.CombinedDataQuery(counter, dims); Assert.AreEqual(sample.MinValue, 0); }
public void MatchIsFalseIfValuesDiffer() { const string anyDim1 = "d1"; const string anyDim2 = "d2"; const string anyDim3 = "d3"; var dimensionSet = new DimensionSet(new HashSet <Dimension> { new Dimension(anyDim1), new Dimension(anyDim2), new Dimension(anyDim3), }); var keyValueSet = new DimensionSpecification { { anyDim1, "val1" }, { anyDim2, "val2" }, { anyDim3, "val3" } }; bool allDimensionsProvided; Key key = dimensionSet.CreateKey(keyValueSet, out allDimensionsProvided); // Longer values foreach (var d in dimensionSet.Dimensions) { var filterValueSet = new DimensionSpecification(keyValueSet); filterValueSet[d.Name] = "valAnyOther"; Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided); Assert.IsFalse(filter.Matches(key)); } // Same length foreach (var d in dimensionSet.Dimensions) { var filterValueSet = new DimensionSpecification(keyValueSet); filterValueSet[d.Name] = "valX"; Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided); Assert.IsFalse(filter.Matches(key)); } // Shorter length foreach (var d in dimensionSet.Dimensions) { var filterValueSet = new DimensionSpecification(keyValueSet); filterValueSet[d.Name] = "v"; Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided); Assert.IsFalse(filter.Matches(key)); } }
public async Task TimeFilterThrowsArgumentOutOfRangeExceptionIfStartTimeIsSameOrGreaterThanEndTime() { var anyTime = DateTime.Now; var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys); var filter = new DimensionSpecification { { ReservedDimensions.StartTimeDimension, anyTime.ToString() }, { ReservedDimensions.EndTimeDimension, anyTime.ToString() }, }; Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, filter)); var anyLaterTime = anyTime.Add(new TimeSpan(0, 0, 1)); filter[ReservedDimensions.StartTimeDimension] = anyLaterTime.ToString(); Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, filter)); }
public async Task BatchQueryMultipleQueriesForSameCounterWithDifferentParametersWorksFine() { // Fill up the taco truck var dimensionSet = new DimensionSet(new HashSet<Dimension> {new Dimension("Filling")}); var counter = await this.dataManager.CreateHitCounter("/Tacos", dimensionSet); var chickenTacos = new DimensionSpecification {{"Filling", "Chicken"}}; var beefTacos = new DimensionSpecification {{"Filling", "Beef"}}; var veggieTacos = new DimensionSpecification {{"Filling", "TOFU"}}; var baconTacos = new DimensionSpecification {{"Filling", "bacon"}}; counter.Increment(100, chickenTacos); counter.Increment(200, beefTacos); counter.Increment(300, veggieTacos); this.dataManager.Flush(); var data = new BatchQueryRequest(); data.Queries.Add(new BatchCounterQuery {CounterName = "/Tacos", UserContext = "TotalTacos"}); data.Queries.Add(new BatchCounterQuery { CounterName = "/Tacos", UserContext = "CluckCluck", QueryParameters = chickenTacos.Data }); data.Queries.Add(new BatchCounterQuery { CounterName = "/Tacos", UserContext = "BACON!", QueryParameters = baconTacos.Data }); var response = await this.httpClient.PostAsync( TestUtils.GetUri(this.server, RestCommands.BatchQueryCommand, string.Empty), GetRequestPayload(data)); Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); using (var readerStream = new ReaderStream(await response.Content.ReadAsStreamAsync())) { var reader = readerStream.CreateBondedCompactBinaryReader<BatchQueryResponse>(); var responseData = reader.Deserialize(); Assert.AreEqual(3, responseData.Responses.Count); // unfiltered should have 100+200+300 hitcount VerifyHitCounter(responseData, "TotalTacos", 600); //only 100 chicken tacos VerifyHitCounter(responseData, "CluckCluck", 100); //sadly, there is no bacon... VerifyHitCounter(responseData, "BACON!", -1); } }
public void SealAndReleaseAreThreadSafe() { var filterableDimension = new DimensionSet(new HashSet<Dimension> { new Dimension("thing") }); using ( var filterableBucket = new DataBucket<InternalHitCount>(filterableDimension, this.timestamp, DefaultBucketTimeSpanTicks, this.currentDirectory, properties.MemoryStreamManager)) { var allDims = new DimensionSpecification {{"thing", "thing"}}; Parallel.For(0, 10, (i) => filterableBucket.AddValue(allDims, i)); Parallel.For(0, 100, (i) => { switch (i % 3) { case 0: foreach (var item in filterableBucket.GetMatches(allDims)) { Assert.IsNotNull(item); } break; case 1: filterableBucket.AddValue(allDims, 11); break; case 2: filterableBucket.ReleaseData(); break; } }); } }
private void ValidateData(ulong expectedPermutations = 100, ulong expectedData = WrittenValue) { this.bucket.Seal(); var uniqueKeys = new HashSet<string>(); var dimensionValues = new DimensionSpecification(); for (var i = 0; i < 10; ++i) { for (var j = 0; j < 10; ++j) { dimensionValues["one"] = i.ToString(); dimensionValues["two"] = j.ToString(); var keyString = this.bucket.DimensionSet.KeyToString(this.bucket.DimensionSet.CreateKey(dimensionValues)); uniqueKeys.Add(keyString); var matchData = this.bucket.GetMatches(dimensionValues).ToList()[0]; Assert.AreEqual(1, matchData.DataCount); Assert.AreEqual(expectedData, matchData.Data.HitCount); } } Assert.AreEqual(expectedPermutations, (ulong)uniqueKeys.Count); }
/// <summary> /// Increment the counter for the provided dimensions. /// </summary> /// <param name="amount">Amount to increment the counter by.</param> /// <param name="dims">Full set of dimension values for the counter.</param> /// <param name="timestamp">Timestamp to use for the data point.</param> public void Increment(long amount, DimensionSpecification dims, DateTime timestamp) { this.hitCounter.AddValue(amount, dims, timestamp); }
private void WriteRoundedSignificantDigitsValue(long value, DimensionSpecification dims, DateTime timestamp) { if (this.roundingFactor > 0 && value > this.minimumRoundingValue) { long factor = 0; while (value > this.minimumRoundingValue) { factor += 1; var lastDigit = value % 10; if (lastDigit >= 5) { value += (10 - lastDigit); } value /= 10; } value *= (long)Math.Pow(10, factor); } this.histogram.AddValue(value, dims, timestamp); }
private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters) { var counterInfo = new CounterInfo { Name = counter.Name, Type = counter.Type, StartTime = counter.StartTime.ToMillisecondTimestamp(), EndTime = counter.EndTime.ToMillisecondTimestamp(), Dimensions = counter.Dimensions.ToList(), DimensionValues = null, // null this out by default to avoid response bloat. }; // Queries for dimension values will come with a 'dimension=pattern' query parameter. // Dimension values can be further filtered with '<dimensionName>=pattern' string dimensionPattern; if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern)) { counterInfo.DimensionValues = new Dictionary<string, ISet<string>>(); // We want to be able to filter dimension values by time (and only time) var dimensionQuery = new DimensionSpecification(); string timeValue; if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue)) { timeValue = MinimumStartTime; } dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue; if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue)) { timeValue = MaximumEndTime; } dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue; foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern))) { string filterPattern; if (queryParameters.TryGetValue(dimensionPattern, out filterPattern)) { counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery) .Where(dimensionValue => dimensionValue.MatchGlob(filterPattern))); } else { counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery)); } } } return counterInfo; }
private DimensionSpecification ProcessQueryParameters(DimensionSpecification queryParameters, out QuerySpecification querySpec) { var filterDimensions = ExtractQuerySpec(queryParameters, out querySpec); if (querySpec.QueryType == QueryType.Average && !this.DataSet.SupportsAverageQuery) { querySpec.QueryType = QueryType.Normal; } else if (querySpec.QueryType == QueryType.Percentile && !this.DataSet.SupportsPercentileQuery) { querySpec.QueryType = QueryType.Normal; } return filterDimensions; }
public void AddValue(long value, DimensionSpecification dims, DateTime timestamp) { this.writeValue(value, dims, timestamp); }
/// <summary> /// Get all values for a particular dimension. /// </summary> /// <param name="dimensionName">Name of the dimension.</param> /// <param name="filterDims">Filter dimensions</param> /// <returns>An enumerator of values for the dimension (may be empty).</returns> public IEnumerable<string> GetDimensionValues(string dimensionName, DimensionSpecification filterDims) { if (filterDims == null) { throw new ArgumentNullException("filterDims"); } return this.DataSet.GetDimensionValues(dimensionName, filterDims); }
/// <summary> /// Execute a query against the counter data. /// </summary> /// <param name="queryParameters">Query parameters.</param> /// <returns>An enumeration of <see cref="DataSample"/>s matching the query parameters.</returns> public IEnumerable<DataSample> Query(DimensionSpecification queryParameters) { QuerySpecification querySpec; if (queryParameters == null) { throw new ArgumentNullException("queryParameters"); } var filterDims = this.ProcessQueryParameters(queryParameters, out querySpec); if (filterDims == null) { return null; } return this.DataSet.QueryData(filterDims, querySpec); }
public void CombinedBucketsAreMergedCorrectly() { var ts1 = new DateTime(2014, 05, 10, 0, 0, 0); var ts2 = new DateTime(2014, 05, 10, 0, 5, 0); var sharedDimensions = new DimensionSpecification { {"one", "a1"}, {"two", "a2"}}; var bucket1Dimensions = new DimensionSpecification { {"one", "b1"}, {"two", "b2"}}; var bucket2Dimensions = new DimensionSpecification { {"one", "c1"}, {"two", "c2"}}; var bucket1 = new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket1.AddValue(sharedDimensions, 867); bucket1.AddValue(bucket1Dimensions, 5309); var bucket2 = new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket2.AddValue(sharedDimensions, 867); bucket2.AddValue(bucket2Dimensions, 42); bucket1.Seal(); bucket2.Seal(); var bucket3 = new DataBucket<InternalHitCount>(new[] {bucket1, bucket2}, new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(10).Ticks, null, this.properties.MemoryStreamManager); var match = bucket3.GetMatches(bucket1Dimensions).First().Data; Assert.AreEqual((ulong)5309, match.HitCount); match = bucket3.GetMatches(bucket2Dimensions).First().Data; Assert.AreEqual((ulong)42, match.HitCount); match = bucket3.GetMatches(sharedDimensions).First().Data; Assert.AreEqual((ulong)867 * 2, match.HitCount); bucket1.Dispose(); bucket2.Dispose(); bucket3.Dispose(); }
public void SplitByDimensionWithFiltersWorksProperly() { var filterableDimension = new DimensionSet(new HashSet<Dimension> {new Dimension("thing"), new Dimension("meat")}); using ( var filterableBucket = new DataBucket<InternalHitCount>(filterableDimension, this.timestamp, DefaultBucketTimeSpanTicks, this.currentDirectory, this.properties.MemoryStreamManager)) { var queryDimensions = new DimensionSpecification(); queryDimensions["thing"] = "thingOne"; queryDimensions["meat"] = "bacon"; filterableBucket.AddValue(queryDimensions, 100); queryDimensions["thing"] = "thingTwo"; queryDimensions["meat"] = "pepperoni"; filterableBucket.AddValue(queryDimensions, 200); filterableBucket.Seal(); // thingOne and thingTwo will match with no filter Assert.AreEqual(2, filterableBucket.GetMatchesSplitByDimension(new DimensionSpecification(), "thing") .Sum(match => match.DataCount)); // only thingOne matches bacon var bestMatchFilter = new DimensionSpecification {{"meat", "bacon"}}; Assert.AreEqual(1, filterableBucket.GetMatchesSplitByDimension(bestMatchFilter, "thing") .Sum(match => match.DataCount)); } }
internal static DimensionSpecification ExtractQuerySpec(DimensionSpecification queryParameters, out QuerySpecification querySpec) { var filterDimensions = new DimensionSpecification(); querySpec = new QuerySpecification(); foreach (var param in queryParameters) { if (string.Equals(param.Key, ReservedDimensions.DimensionDimension, StringComparison.OrdinalIgnoreCase)) { querySpec.CrossQueryDimension = param.Value; // Ensure that split dimension is not also sent a filter dimension if (queryParameters.ContainsKey(param.Value)) { return null; } } else if (string.Equals(param.Key, ReservedDimensions.AggregateSamplesDimension, StringComparison.OrdinalIgnoreCase)) { querySpec.Combine = Convert.ToBoolean(param.Value); } else if (string.Equals(param.Key, ReservedDimensions.PercentileDimension, StringComparison.OrdinalIgnoreCase)) { if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForAverage, StringComparison.OrdinalIgnoreCase)) { querySpec.QueryType = QueryType.Average; } else if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForMaximum, StringComparison.OrdinalIgnoreCase)) { querySpec.QueryType = QueryType.Maximum; } else if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForMinimum, StringComparison.OrdinalIgnoreCase)) { querySpec.QueryType = QueryType.Minimum; } else { querySpec.QueryType = QueryType.Percentile; querySpec.Percentile = double.Parse(param.Value); } } else { filterDimensions.Add(param.Key, param.Value); } } return filterDimensions; }
public void AddValue(long value, DimensionSpecification dims) { this.AddValue(value, dims, DateTime.Now); }
/// <summary> /// Increment the counter by one for the provided dimensions using the current time for the data point. /// </summary> /// <param name="dims">Full set of dimension values for the counter.</param> public void Increment(DimensionSpecification dims) { this.Increment(dims, DateTime.Now); }
private void WriteUnroundedValue(long value, DimensionSpecification dims, DateTime timestamp) { this.histogram.AddValue(value, dims, timestamp); }
/// <summary> /// Increment the counter by one for the provided dimensions. /// </summary> /// <param name="dims">Full set of dimension values for the counter.</param> /// <param name="timestamp">Timestamp to use for the data point.</param> public void Increment(DimensionSpecification dims, DateTime timestamp) { this.Increment(1, dims, timestamp); }
private void WriteRoundedSizeValue(long value, DimensionSpecification dims, DateTime timestamp) { if (value < 0) { value = 0; } if (value > this.roundingFactor) { long remainder = value % this.roundingFactor; if (remainder >= (this.roundingFactor / 2)) { remainder = -remainder; // will roll up } value = value - remainder; } this.histogram.AddValue(value, dims, timestamp); }
/// <summary> /// Increment the counter for the provided dimensions using the current time for the data point. /// </summary> /// <param name="amount">Amount to increment the counter by.</param> /// <param name="dims">Full set of dimension values for the counter.</param> public void Increment(long amount, DimensionSpecification dims) { this.Increment(amount, dims, DateTime.Now); }