public async Task GetDimensionValuesForAllDimensionInDataProvidesExpectedValues()
        {
            const int NumValuesPerDim = 100;
            var       dimensionValues = new DimensionSpecification();

            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            for (var i = 0; i < NumValuesPerDim; ++i)
            {
                foreach (var d in AnyKeys.Dimensions)
                {
                    dimensionValues[d.Name] = d.Name + i;
                }
                counter.Increment(dimensionValues);
            }
            counter.DataSet.Flush();

            foreach (var d in AnyKeys.Dimensions)
            {
                var values = counter.GetDimensionValues(d.Name, new DimensionSpecification()).ToList();
                Assert.AreEqual(NumValuesPerDim, values.Count);
                for (var i = 0; i < NumValuesPerDim; ++i)
                {
                    var expected = d.Name + i;
                    Assert.IsTrue(values.Contains(expected));
                }
            }
        }
        public void SetUp()
        {
            this.dimensionSet =
                new DimensionSet(new HashSet<Dimension>
                                 {
                                     new Dimension("foo"),
                                     new Dimension("bar"),
                                     new Dimension("baz"),
                                 });

            dimensions = new DimensionSpecification();
            this.storagePath = Path.Combine(Environment.CurrentDirectory, "msTemp");
            if (Directory.Exists(this.storagePath))
            {
                Directory.Delete(this.storagePath, true);
            }

            Directory.CreateDirectory(this.storagePath);
            foreach (var d in this.dimensionSet.Dimensions)
            {
                dimensions[d.Name] = DimValue;
            }

            this.properties = new MockSharedDataSetProperties();
        }
Beispiel #3
0
        private static void RunOperation(Counter counter, CounterWriteOperation op)
        {
            var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow
                                ? DateTime.UtcNow
                                : op.Timestamp.ToDateTime();

            var dims = new DimensionSpecification(op.DimensionValues);

            var hitCounter       = counter as HitCounter;
            var histogramCounter = counter as HistogramCounter;

            if (hitCounter != null)
            {
                hitCounter.Increment(op.Value * op.Count, dims, timestamp);
            }
            else
            {
                // It would be nice to direct-inject multiple values at once, but the APIs don't currently
                // support this. It's a reasonable amount of work to fix this and unknown whether folks will use this
                // a lot at this time.
                for (var i = 0; i < op.Count; ++i)
                {
                    histogramCounter.AddValue(op.Value, dims, timestamp);
                }
            }
        }
        public async Task HistogramPercentileValuesAreCorrectlyDistributedByBucket()
        {
            var bucket1Timestamp = DateTime.Now;
            var bucket2Timestamp =
                new DateTime(bucket1Timestamp.Ticks + this.defaultCompactionConfig.DefaultBucketTicks * 2);

            var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);

            for (int i = 0; i < 100; ++i)
            {
                counter.AddValue(i, AnyDimensions, bucket1Timestamp);
            }
            for (int i = 0; i < 100; ++i)
            {
                counter.AddValue(i * 10, AnyDimensions, bucket2Timestamp);
            }

            var dims = new DimensionSpecification(AnyDimensions);

            for (int i = 1; i < 100; ++i)
            {
                dims[ReservedDimensions.PercentileDimension] = i.ToString();
                var bucketedSamples =
                    this.BucketedDataQuery(counter, dims, 2).ToList();
                var sample1 = bucketedSamples[1];
                Assert.AreEqual(sample1.Percentile, i);
                Assert.AreEqual(sample1.SampleCount, (ulong)100);
                Assert.AreEqual(sample1.PercentileValue, i - 1);

                var sample2 = bucketedSamples[0];
                Assert.AreEqual(sample2.Percentile, i);
                Assert.AreEqual(sample2.SampleCount, (ulong)100);
                Assert.AreEqual(sample2.PercentileValue, (i - 1) * 10);
            }
        }
        public override void Setup()
        {
            this.dataManager.CompactionConfiguration =
                new DataCompactionConfiguration(new[]
            {
                new DataIntervalConfiguration(TimeSpan.FromMinutes(5), TimeSpan.MaxValue),
            });
            this.dataManager.MaximumDataAge = TimeSpan.Zero; // Don't want data getting deleted here.

            // Make a counter and write some stuff.
            this.dimensionSet = new DimensionSet(new HashSet <Dimension> {
                new Dimension(AnyDimension),
            });
            var counter = this.dataManager.CreateHitCounter(AnyCounter, this.dimensionSet).Result;

            var dims      = new DimensionSpecification();
            var timestamp = DateTime.Now;

            dims[AnyDimension] = "a";
            counter.Increment(dims, timestamp);
            dims[AnyDimension] = "b";
            counter.Increment(2, dims, timestamp);
            dims[AnyDimension] = "c";
            counter.Increment(dims, timestamp);

            // We need to force a seal by setting some value waaaay in the future.
            counter.Increment(dims, timestamp.AddYears(10));

            this.memoryStreamManager = new RecyclableMemoryStreamManager(1 << 17, 1 << 20, 1 << 24);
        }
        public void MatchIsFalseIfFilterHasMoreOrLessValuesThanKey()
        {
            const string anyDim1      = "d1";
            const string anyDim2      = "d2";
            const string anyDim3      = "d3";
            var          dimensionSet = new DimensionSet(new HashSet <Dimension>
            {
                new Dimension(anyDim1),
                new Dimension(anyDim2),
                new Dimension(anyDim3),
            });
            var keyValueSet = new DimensionSpecification {
                { anyDim1, "val1" }, { anyDim2, "val2" }, { anyDim3, "val3" }
            };
            bool allDimensionsProvided;
            Key  key = dimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);

            var smallerDimensionSet =
                new DimensionSet(new HashSet <Dimension> {
                new Dimension(anyDim1), new Dimension(anyDim2)
            });
            Key smallerFilter = smallerDimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);

            Assert.IsFalse(smallerFilter.Matches(key));

            var largerDimensionSet = new DimensionSet(new HashSet <Dimension>(dimensionSet.Dimensions)
            {
                new Dimension
                    ("anotherDim")
            });
            Key largerFilter = largerDimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);

            Assert.IsFalse(largerFilter.Matches(key));
        }
 /// <summary>
 /// Safely increment a HitCounter by one using the given timestamp.
 /// If the counter is null no operation will be performed.
 /// </summary>
 /// <param name="counter">Counter to increment.</param>
 /// <param name="dims">Dimensions to use for incrementing.</param>
 /// <param name="timestamp">Timestamp for written value.</param>
 public static void SafeIncrement(this HitCounter counter, DimensionSpecification dims, DateTime timestamp)
 {
     if (counter != null)
     {
         counter.Increment(1, dims, timestamp);
     }
 }
Beispiel #8
0
        public void SetUp()
        {
            this.dimensionSet =
                new DimensionSet(new HashSet <Dimension>
            {
                new Dimension("foo"),
                new Dimension("bar"),
                new Dimension("baz"),
            });

            dimensions       = new DimensionSpecification();
            this.storagePath = Path.Combine(Environment.CurrentDirectory, "msTemp");
            if (Directory.Exists(this.storagePath))
            {
                Directory.Delete(this.storagePath, true);
            }

            Directory.CreateDirectory(this.storagePath);
            foreach (var d in this.dimensionSet.Dimensions)
            {
                dimensions[d.Name] = DimValue;
            }

            this.properties = new MockSharedDataSetProperties();
        }
        /// <summary>
        /// Ask the counter aggregator to do all percentile computation as opposed to any metric system server. This will 
        /// strip out the 'percentile=xxx' parameter in the request and return the filtered parameter list. If a 
        /// percentile request was present, this will apply the percentile calculation in the final calculation of this object 
        /// </summary>
        /// <param name="queryParameters">Parameters to filter</param>
        /// <returns>Filtered parameter set</returns>
        public IDictionary<string, string> ApplyPercentileCalculationAggregation(IDictionary<string, string> queryParameters)
        {
            if (queryParameters == null)
            {
                return new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
            }

            string percentileParameter;
            double percentileRequested;

            var dimensionSpec = new DimensionSpecification(queryParameters);

            if (!dimensionSpec.TryGetValue(ReservedDimensions.PercentileDimension, out percentileParameter) ||
                !double.TryParse(percentileParameter, out percentileRequested) ||
                percentileRequested < 0 || percentileRequested > 100)
            {
                this.shouldFilterToFinalPercentile = false;
                return queryParameters;
            }

            this.shouldFilterToFinalPercentile = true;
            this.requestedPercentile = percentileRequested;

            var retVal = new Dictionary<string, string>(dimensionSpec, StringComparer.OrdinalIgnoreCase);
            retVal.Remove(ReservedDimensions.PercentileDimension);

            return retVal;
        }
Beispiel #10
0
        public override void Setup()
        {
            this.dataManager.CompactionConfiguration =
                new DataCompactionConfiguration(new[]
                                         {
                                             new DataIntervalConfiguration(TimeSpan.FromMinutes(5), TimeSpan.MaxValue), 
                                         });
            this.dataManager.MaximumDataAge = TimeSpan.Zero; // Don't want data getting deleted here.

            // Make a counter and write some stuff.
            this.dimensionSet = new DimensionSet(new HashSet<Dimension> { new Dimension(AnyDimension), });
            var counter = this.dataManager.CreateHitCounter(AnyCounter, this.dimensionSet).Result; 

            var dims = new DimensionSpecification();
            var timestamp = DateTime.Now;
            dims[AnyDimension] = "a";
            counter.Increment(dims, timestamp);
            dims[AnyDimension] = "b";
            counter.Increment(2, dims, timestamp);
            dims[AnyDimension] = "c";
            counter.Increment(dims, timestamp);

            // We need to force a seal by setting some value waaaay in the future.
            counter.Increment(dims, timestamp.AddYears(10));

            this.memoryStreamManager = new RecyclableMemoryStreamManager(1 << 17, 1 << 20, 1 << 24);
        }
 /// <summary>
 /// Safely increment a HitCounter by the given amount using the current time.
 /// If the counter is null no operation will be performed.
 /// </summary>
 /// <param name="counter">Counter to increment.</param>
 /// <param name="amount">Amount to increment by.</param>
 /// <param name="dims">Dimensions to use for incrementing.</param>
 public static void SafeIncrement(this HitCounter counter, long amount, DimensionSpecification dims)
 {
     if (counter != null)
     {
         counter.Increment(amount, dims, DateTime.Now);
     }
 }
        public async Task CanRetrieveHistogramPercentileData()
        {
            var anyTimestamp = DateTime.Now;
            var counter      = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);

            for (int i = 0; i < 100; ++i)
            {
                counter.AddValue(i, AnyDimensions, anyTimestamp);
            }

            var dims = new DimensionSpecification(AnyDimensions);

            for (int i = 1; i < 100; ++i)
            {
                dims[ReservedDimensions.PercentileDimension] = i.ToString();
                var sample =
                    this.CombinedDataQuery(counter, dims);
                Assert.AreEqual(sample.Percentile, i);
                Assert.AreEqual(sample.SampleCount, (ulong)100);
                Assert.AreEqual(sample.PercentileValue, i - 1);
            }

            dims[ReservedDimensions.PercentileDimension] = "0";
            var sample2 = this.CombinedDataQuery(counter, dims);

            Assert.AreEqual(sample2.Percentile, 0);
            Assert.AreEqual(sample2.SampleCount, (ulong)100);
            Assert.AreEqual(sample2.PercentileValue, 0);

            dims[ReservedDimensions.PercentileDimension] = "100";
            sample2 = this.CombinedDataQuery(counter, dims);
            Assert.AreEqual(sample2.Percentile, 100);
            Assert.AreEqual(sample2.SampleCount, (ulong)100);
            Assert.AreEqual(sample2.PercentileValue, 99);
        }
        /// <summary>
        /// Ask the counter aggregator to do all percentile computation as opposed to any metric system server. This will
        /// strip out the 'percentile=xxx' parameter in the request and return the filtered parameter list. If a
        /// percentile request was present, this will apply the percentile calculation in the final calculation of this object
        /// </summary>
        /// <param name="queryParameters">Parameters to filter</param>
        /// <returns>Filtered parameter set</returns>
        public IDictionary <string, string> ApplyPercentileCalculationAggregation(IDictionary <string, string> queryParameters)
        {
            if (queryParameters == null)
            {
                return(new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase));
            }

            string percentileParameter;
            double percentileRequested;

            var dimensionSpec = new DimensionSpecification(queryParameters);

            if (!dimensionSpec.TryGetValue(ReservedDimensions.PercentileDimension, out percentileParameter) ||
                !double.TryParse(percentileParameter, out percentileRequested) ||
                percentileRequested < 0 || percentileRequested > 100)
            {
                this.shouldFilterToFinalPercentile = false;
                return(queryParameters);
            }

            this.shouldFilterToFinalPercentile = true;
            this.requestedPercentile           = percentileRequested;

            var retVal = new Dictionary <string, string>(dimensionSpec, StringComparer.OrdinalIgnoreCase);

            retVal.Remove(ReservedDimensions.PercentileDimension);

            return(retVal);
        }
Beispiel #14
0
 private void GenerateRandomKey(DimensionSpecification dims)
 {
     dims.Clear();
     foreach (var d in this.dimensionSet.Dimensions)
     {
         dims.Add(d.Name, GetRandom(MaxDimensionValue).ToString());
     }
 }
Beispiel #15
0
        private void WriteComplexSampleData()
        {
            var rng = new Random();

            histogramKeys      = new int[BucketCount][][];
            totalCountBySample = new Dictionary <long, uint>();

            this.dimensionSet = new DimensionSet(new HashSet <Dimension>
            {
                new Dimension("one"),
                new Dimension("two"),
            });

            this.sharedProperties.CompactionConfiguration =
                new DataCompactionConfiguration(new[]
            {
                new DataIntervalConfiguration(TimeSpan.FromMinutes(1), TimeSpan.MaxValue),
            });
            this.histogramDataSet =
                new DataSet <InternalHistogram>("/AggTest", null,
                                                this.dimensionSet,
                                                this.sharedProperties);

            var dimSpec   = new DimensionSpecification();
            var startTime = new DateTime(2014, 7, 4, 0, 0, 0, DateTimeKind.Utc); // no DateTimeKind.MERKUH? Sigh.

            for (var i = 0; i < BucketCount; ++i)
            {
                histogramKeys[i] = new int[DimensionOneCount][];

                var ts = startTime +
                         TimeSpan.FromTicks(i * this.sharedProperties.CompactionConfiguration.DefaultBucketTicks);
                for (var d1 = 0; d1 < DimensionOneCount; ++d1)
                {
                    histogramKeys[i][d1] = new int[DimensionTwoCount];

                    dimSpec["one"] = d1.ToString();
                    for (var d2 = 0; d2 < DimensionTwoCount; ++d2)
                    {
                        dimSpec["two"] = d2.ToString();

                        var val = rng.Next(MaxValue);
                        this.histogramDataSet.AddValue(val, dimSpec, ts);

                        histogramKeys[i][d1][d2] = val;
                        if (!totalCountBySample.ContainsKey(val))
                        {
                            totalCountBySample[val] = 0;
                        }
                        totalCountBySample[val]++;
                    }
                }
            }

            this.histogramDataSet.Flush();
        }
        public async Task GetDimensionValuesForUnknownFilterDimensionDoesNotThrow()
        {
            var filterDims = new DimensionSpecification {
                { "UnknownDim", "UnKnown" }
            };
            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            // if the following call throws test will fail
            counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filterDims);
        }
Beispiel #17
0
 private void WriteRandomTestData(int maxValue, TestKeyedDataStore store)
 {
     Parallel.For(0, maxValue,
                  i =>
     {
         var dims = new DimensionSpecification();
         this.GenerateRandomKey(dims);
         store.AddValue(dims, i);
     });
 }
Beispiel #18
0
        public void SetUp()
        {
            this.dimensionSet = new DimensionSet(new HashSet<Dimension>());
            this.dimensionSpec = new DimensionSpecification();
            this.sharedProperties = new MockSharedDataSetProperties();
            this.bucketSpan = this.sharedProperties.CompactionConfiguration.Default.Interval;

            this.firstBucketTimestamp = DateTime.UtcNow;
            this.secondBucketTimestamp = this.firstBucketTimestamp + this.bucketSpan;
            this.thirdBucketTimestamp = this.secondBucketTimestamp + this.bucketSpan;
        }
        public async Task GetDimensionValuesWithAllDimensionsFilledThrowsArgumentException()
        {
            var dimValues = new DimensionSpecification(AnyDimensions);

            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            Assert.Throws <ArgumentException>(
                () =>
                counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, dimValues)
                .ToList());
        }
Beispiel #20
0
        public void SetUp()
        {
            this.dimensionSet     = new DimensionSet(new HashSet <Dimension>());
            this.dimensionSpec    = new DimensionSpecification();
            this.sharedProperties = new MockSharedDataSetProperties();
            this.bucketSpan       = this.sharedProperties.CompactionConfiguration.Default.Interval;

            this.firstBucketTimestamp  = DateTime.UtcNow;
            this.secondBucketTimestamp = this.firstBucketTimestamp + this.bucketSpan;
            this.thirdBucketTimestamp  = this.secondBucketTimestamp + this.bucketSpan;
        }
        private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters)
        {
            var counterInfo = new CounterInfo
            {
                Name            = counter.Name,
                Type            = counter.Type,
                StartTime       = counter.StartTime.ToMillisecondTimestamp(),
                EndTime         = counter.EndTime.ToMillisecondTimestamp(),
                Dimensions      = counter.Dimensions.ToList(),
                DimensionValues = null,                   // null this out by default to avoid response bloat.
            };

            // Queries for dimension values will come with a 'dimension=pattern' query parameter.
            // Dimension values can be further filtered with '<dimensionName>=pattern'
            string dimensionPattern;

            if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern))
            {
                counterInfo.DimensionValues = new Dictionary <string, ISet <string> >();

                // We want to be able to filter dimension values by time (and only time)
                var    dimensionQuery = new DimensionSpecification();
                string timeValue;
                if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue))
                {
                    timeValue = MinimumStartTime;
                }
                dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue;
                if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue))
                {
                    timeValue = MaximumEndTime;
                }
                dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue;

                foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern)))
                {
                    string filterPattern;
                    if (queryParameters.TryGetValue(dimensionPattern, out filterPattern))
                    {
                        counterInfo.AddDimensionValues(dim,
                                                       counter.GetDimensionValues(dim, dimensionQuery)
                                                       .Where(dimensionValue =>
                                                              dimensionValue.MatchGlob(filterPattern)));
                    }
                    else
                    {
                        counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery));
                    }
                }
            }

            return(counterInfo);
        }
        public async Task AddingDataWithEmptyKeyValuesIsValid()
        {
            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            var dimensions = new DimensionSpecification();

            foreach (var d in AnyKeys.Dimensions)
            {
                dimensions[d.Name] = string.Empty;
            }
            counter.Increment(dimensions);
        }
        public override async Task <Response> ProcessRequest(Request request)
        {
            var fanoutRequest = request.HasInputBody ? await request.ReadInputBody <TieredRequest>() : null;

            if (request.HasInputBody && fanoutRequest == null)
            {
                // This indicates failed deserialization
                return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body"));
            }

            // aggregation servers should fanout automagically. we honor filters for machine function and datacenter.
            if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation)
            {
                if (fanoutRequest == null)
                {
                    fanoutRequest = new TieredRequest();
                }

                fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList();
            }

            var queryParameters = new DimensionSpecification(request.QueryParameters);
            var endOfName       = request.Path.LastIndexOf('/');

            if (endOfName < 0)
            {
                return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided."));
            }

            var counterName = request.Path.Substring(0, endOfName);

            if (string.IsNullOrEmpty(counterName))
            {
                return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided."));
            }

            var commandName = request.Path.Substring(endOfName + 1);

            if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
            {
                return(await this.Info(request, fanoutRequest, counterName, queryParameters));
            }

            if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
            {
                var response = await this.Query(counterName, fanoutRequest, queryParameters);

                return(Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response));
            }

            return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName));
        }
        public async Task GetBucketedDataCombinesDataAcrossDimensionValuesIfFilterValueIsNotProvided()
        {
            var          anyTimestamp    = DateTime.Now;
            const string FirstDimension  = "1st";
            const string SecondDimension = "2nd";
            const string ThirdDimension  = "3rd";

            var dimensions = new DimensionSet(new HashSet <Dimension>
            {
                new Dimension(FirstDimension),
                new Dimension(SecondDimension),
                new Dimension(ThirdDimension)
            });
            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, dimensions);


            var dimValues = new DimensionSpecification();

            for (int i = 0; i < 10; ++i)
            {
                dimValues[FirstDimension] = i.ToString();
                for (int j = 0; j < 10; ++j)
                {
                    dimValues[SecondDimension] = j.ToString();
                    for (int k = 0; k < 10; ++k)
                    {
                        dimValues[ThirdDimension] = k.ToString();
                        counter.Increment(dimValues, anyTimestamp);
                    }
                }
            }

            var sample =
                this.BucketedDataQuery(counter, new DimensionSpecification()).First();

            Assert.AreEqual(DataSampleType.HitCount, sample.SampleType);
            Assert.AreEqual((ulong)1000, sample.HitCount);

            dimValues.Clear();
            dimValues[FirstDimension] = "8"; // take one tenth of things by filter.
            sample = this.BucketedDataQuery(counter, dimValues).First();
            Assert.AreEqual((ulong)100, sample.HitCount);

            dimValues[SecondDimension] = "6";
            sample = this.BucketedDataQuery(counter, dimValues).First();
            Assert.AreEqual((ulong)10, sample.HitCount);

            dimValues[ThirdDimension] = "7";
            sample = this.BucketedDataQuery(counter, dimValues).First();
            Assert.AreEqual((ulong)1, sample.HitCount);
        }
Beispiel #25
0
        public void CombinedBucketsAreMergedCorrectly()
        {
            var ts1 = new DateTime(2014, 05, 10, 0, 0, 0);
            var ts2 = new DateTime(2014, 05, 10, 0, 5, 0);
            var sharedDimensions = new DimensionSpecification {
                { "one", "a1" }, { "two", "a2" }
            };
            var bucket1Dimensions = new DimensionSpecification {
                { "one", "b1" }, { "two", "b2" }
            };
            var bucket2Dimensions = new DimensionSpecification {
                { "one", "c1" }, { "two", "c2" }
            };

            var bucket1 =
                new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1,
                                                  TimeSpan.FromMinutes(5).Ticks, null,
                                                  this.properties.MemoryStreamManager);

            bucket1.AddValue(sharedDimensions, 867);
            bucket1.AddValue(bucket1Dimensions, 5309);

            var bucket2 =
                new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2,
                                                  TimeSpan.FromMinutes(5).Ticks, null,
                                                  this.properties.MemoryStreamManager);

            bucket2.AddValue(sharedDimensions, 867);
            bucket2.AddValue(bucket2Dimensions, 42);

            bucket1.Seal();
            bucket2.Seal();
            var bucket3 =
                new DataBucket <InternalHitCount>(new[] { bucket1, bucket2 },
                                                  new DimensionSet(this.twoDimensionSet), ts1,
                                                  TimeSpan.FromMinutes(10).Ticks, null,
                                                  this.properties.MemoryStreamManager);

            var match = bucket3.GetMatches(bucket1Dimensions).First().Data;

            Assert.AreEqual((ulong)5309, match.HitCount);
            match = bucket3.GetMatches(bucket2Dimensions).First().Data;
            Assert.AreEqual((ulong)42, match.HitCount);

            match = bucket3.GetMatches(sharedDimensions).First().Data;
            Assert.AreEqual((ulong)867 * 2, match.HitCount);

            bucket1.Dispose();
            bucket2.Dispose();
            bucket3.Dispose();
        }
        public async Task InvalidNumericHistogramValueThrowsArgumentOutOfRangeException()
        {
            var counter = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);

            var dims = new DimensionSpecification(AnyDimensions);

            dims[ReservedDimensions.PercentileDimension] = "-1";
            Assert.Throws <ArgumentOutOfRangeException>(() => this.CombinedDataQuery(counter, dims));
            Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, dims));

            dims[ReservedDimensions.PercentileDimension] = "100.1";
            Assert.Throws <ArgumentOutOfRangeException>(() => this.CombinedDataQuery(counter, dims));
            Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, dims));
        }
Beispiel #27
0
        private void WriteSomeData()
        {
            var dimensionValues = new DimensionSpecification();

            for (var i = 0; i < 10; ++i)
            {
                for (var j = 0; j < 10; ++j)
                {
                    dimensionValues["one"] = i.ToString();
                    dimensionValues["two"] = j.ToString();
                    this.bucket.AddValue(dimensionValues, WrittenValue);
                }
            }
        }
        private DataSample CombinedDataQuery(Counter counter, DimensionSpecification queryParams)
        {
            if (queryParams != null)
            {
                queryParams[ReservedDimensions.AggregateSamplesDimension] = "true";
            }

            counter.DataSet.Flush();
            var queryResults = counter.Query(queryParams).ToList();

            queryParams.Remove(ReservedDimensions.AggregateSamplesDimension);

            return(queryResults.Count > 0 ? queryResults[0] : null);
        }
Beispiel #29
0
        public override async Task<Response> ProcessRequest(Request request)
        {
            var fanoutRequest = request.HasInputBody ? await request.ReadInputBody<TieredRequest>() : null;
            if (request.HasInputBody && fanoutRequest == null)
            {
                // This indicates failed deserialization
                return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body");
            }

            // aggregation servers should fanout automagically. we honor filters for machine function and datacenter.
            if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation)
            {
                if (fanoutRequest == null)
                {
                    fanoutRequest = new TieredRequest();
                }

                fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList();
            }

            var queryParameters = new DimensionSpecification(request.QueryParameters);
            var endOfName = request.Path.LastIndexOf('/');
            if (endOfName < 0)
            {
                return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided.");
            }

            var counterName = request.Path.Substring(0, endOfName);
            if (string.IsNullOrEmpty(counterName))
            {
                return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided.");
            }

            var commandName = request.Path.Substring(endOfName + 1);

            if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
            {
                return await this.Info(request, fanoutRequest, counterName, queryParameters);
            }

            if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase))
            {
                var response = await this.Query(counterName, fanoutRequest, queryParameters);
                return Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response);
            }

            return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName);
        }
        public async Task GetDimensionValuesWithTimeFilteredProvidesValuesOnlyInTimeRange()
        {
            var earlyTime = new DateTime(2013, 12, 11, 00, 01, 00); //bucket starttime  = 2013, 12, 11,00,00,00, endtime = 2013, 12, 11,00,20,00
            var laterTime = new DateTime(2013, 12, 11, 00, 22, 00); //bucket starttime  = 2013, 12, 11,00,20,00, endtime = 2013, 12, 11,00,40,00

            var dimensionValues = new DimensionSpecification(AnyDimensions);

            //create the hit counters
            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            dimensionValues[AnyKeys.Dimensions.First().Name] = "early";
            counter.Increment(dimensionValues, earlyTime);
            dimensionValues[AnyKeys.Dimensions.First().Name] = "later";
            counter.Increment(dimensionValues, laterTime);
            counter.DataSet.Flush();

            //this should return early
            var filter = new DimensionSpecification
            {
                { ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 00, 00).ToString(Protocol.TimestampStringFormat) },
                { ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 20, 00).ToString(Protocol.TimestampStringFormat) },
            };
            var values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList();

            Assert.AreEqual(1, values.Count);
            Assert.AreEqual("early", values[0]);

            //should only return later
            filter = new DimensionSpecification
            {
                { ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 20, 00).ToString(Protocol.TimestampStringFormat) },
                { ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 40, 00).ToString(Protocol.TimestampStringFormat) },
            };
            values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList();
            Assert.AreEqual(1, values.Count);
            Assert.AreEqual("later", values[0]);

            //should return both later and early
            filter = new DimensionSpecification
            {
                { ReservedDimensions.StartTimeDimension, new DateTime(2013, 12, 11, 00, 00, 00).ToString(Protocol.TimestampStringFormat) },
                { ReservedDimensions.EndTimeDimension, new DateTime(2013, 12, 11, 00, 40, 00).ToString(Protocol.TimestampStringFormat) },
            };
            values = counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, filter).ToList();
            Assert.AreEqual(2, values.Count);
            Assert.IsTrue(values.Contains("early"));
            Assert.IsTrue(values.Contains("later"));
        }
        public void CreateKeySetsAllDimensionsProvidedFalseIfSomeDimensionsAreNotProvided()
        {
            const string anyDim1      = "d1";
            const string anyDim2      = "d2";
            var          dimensionSet = new DimensionSet(new HashSet <Dimension> {
                new Dimension(anyDim1), new Dimension(anyDim2)
            });
            var partialDimensionValues = new DimensionSpecification {
                { anyDim1, "anyVal1" }
            };

            bool allDimensionsProvided;

            dimensionSet.CreateKey(partialDimensionValues, out allDimensionsProvided);
            Assert.AreEqual(false, allDimensionsProvided);
        }
        public async Task GetBucketedDataReturnsEmptyEnumerationIfNoSamplesMatchDimensionValues()
        {
            var anyTimestamp = DateTime.Now;

            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            counter.Increment(AnyDimensions, anyTimestamp);

            var dimValues = new DimensionSpecification();

            foreach (var d in AnyKeys.Dimensions)
            {
                dimValues[d.Name] = "no match";
            }
            Assert.IsFalse(this.BucketedDataQuery(counter, dimValues, 0).Any());
        }
        public void CreateKeySetsAllDimensionsProvidedTrueIfAllDimensionsAreInDictionary()
        {
            const string anyDim1      = "d1";
            const string anyDim2      = "d2";
            var          dimensionSet = new DimensionSet(new HashSet <Dimension> {
                new Dimension(anyDim1), new Dimension(anyDim2)
            });
            var allDimensionValues = new DimensionSpecification {
                { anyDim1, "anyVal1" }, { anyDim2, "anyVal2" }
            };

            bool allDimensionsProvided;

            dimensionSet.CreateKey(allDimensionValues, out allDimensionsProvided);
            Assert.AreEqual(true, allDimensionsProvided);
        }
        public async Task GetDimensionValuesForAllDimensionInDataProvidesEmptyStringForEmptyDimensionValue()
        {
            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            counter.Increment(AnyDimensions);
            var otherDimValues = new DimensionSpecification(AnyDimensions);

            otherDimValues[AnyKeys.Dimensions.First().Name] = ReservedDimensions.EmptyDimensionValue;
            counter.Increment(otherDimValues);
            counter.DataSet.Flush();

            var values =
                counter.GetDimensionValues(AnyKeys.Dimensions.First().Name, new DimensionSpecification()).ToList();

            Assert.AreEqual(2, values.Count);
            Assert.IsTrue(values.Contains(AnyDimValue));
            Assert.IsTrue(values.Contains(string.Empty));
        }
        public async Task CanRetrieveHistogramMinimumData()
        {
            var anyTimestamp = DateTime.Now;
            var counter      = await this.dataManager.CreateHistogramCounter(AnyCounterName, AnyKeys);

            for (int i = 0; i < 100; ++i)
            {
                counter.AddValue(i, AnyDimensions, anyTimestamp);
            }

            var dims = new DimensionSpecification(AnyDimensions);

            dims[ReservedDimensions.PercentileDimension] = ReservedDimensions.PercentileDimensionValueForMinimum;

            var sample = this.CombinedDataQuery(counter, dims);

            Assert.AreEqual(sample.MinValue, 0);
        }
        public void MatchIsFalseIfValuesDiffer()
        {
            const string anyDim1      = "d1";
            const string anyDim2      = "d2";
            const string anyDim3      = "d3";
            var          dimensionSet = new DimensionSet(new HashSet <Dimension>
            {
                new Dimension(anyDim1),
                new Dimension(anyDim2),
                new Dimension(anyDim3),
            });
            var keyValueSet = new DimensionSpecification {
                { anyDim1, "val1" }, { anyDim2, "val2" }, { anyDim3, "val3" }
            };
            bool allDimensionsProvided;
            Key  key = dimensionSet.CreateKey(keyValueSet, out allDimensionsProvided);

            // Longer values
            foreach (var d in dimensionSet.Dimensions)
            {
                var filterValueSet = new DimensionSpecification(keyValueSet);
                filterValueSet[d.Name] = "valAnyOther";
                Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided);
                Assert.IsFalse(filter.Matches(key));
            }

            // Same length
            foreach (var d in dimensionSet.Dimensions)
            {
                var filterValueSet = new DimensionSpecification(keyValueSet);
                filterValueSet[d.Name] = "valX";
                Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided);
                Assert.IsFalse(filter.Matches(key));
            }

            // Shorter length
            foreach (var d in dimensionSet.Dimensions)
            {
                var filterValueSet = new DimensionSpecification(keyValueSet);
                filterValueSet[d.Name] = "v";
                Key filter = dimensionSet.CreateKey(filterValueSet, out allDimensionsProvided);
                Assert.IsFalse(filter.Matches(key));
            }
        }
        public async Task TimeFilterThrowsArgumentOutOfRangeExceptionIfStartTimeIsSameOrGreaterThanEndTime()
        {
            var anyTime = DateTime.Now;

            var counter = await this.dataManager.CreateHitCounter(AnyCounterName, AnyKeys);

            var filter = new DimensionSpecification
            {
                { ReservedDimensions.StartTimeDimension, anyTime.ToString() },
                { ReservedDimensions.EndTimeDimension, anyTime.ToString() },
            };

            Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, filter));

            var anyLaterTime = anyTime.Add(new TimeSpan(0, 0, 1));

            filter[ReservedDimensions.StartTimeDimension] = anyLaterTime.ToString();
            Assert.Throws <ArgumentOutOfRangeException>(() => this.BucketedDataQuery(counter, filter));
        }
        public async Task BatchQueryMultipleQueriesForSameCounterWithDifferentParametersWorksFine()
        {
            // Fill up the taco truck
            var dimensionSet = new DimensionSet(new HashSet<Dimension> {new Dimension("Filling")});
            var counter = await this.dataManager.CreateHitCounter("/Tacos", dimensionSet);

            var chickenTacos = new DimensionSpecification {{"Filling", "Chicken"}};
            var beefTacos = new DimensionSpecification {{"Filling", "Beef"}};
            var veggieTacos = new DimensionSpecification {{"Filling", "TOFU"}};
            var baconTacos = new DimensionSpecification {{"Filling", "bacon"}};

            counter.Increment(100, chickenTacos);
            counter.Increment(200, beefTacos);
            counter.Increment(300, veggieTacos);
            this.dataManager.Flush();

            var data = new BatchQueryRequest();
            data.Queries.Add(new BatchCounterQuery {CounterName = "/Tacos", UserContext = "TotalTacos"});
            data.Queries.Add(new BatchCounterQuery
                             {
                                 CounterName = "/Tacos",
                                 UserContext = "CluckCluck",
                                 QueryParameters = chickenTacos.Data
                             });
            data.Queries.Add(new BatchCounterQuery
                             {
                                 CounterName = "/Tacos",
                                 UserContext = "BACON!",
                                 QueryParameters = baconTacos.Data
                             });

            var response = await
                           this.httpClient.PostAsync(
                                                     TestUtils.GetUri(this.server,
                                                                      RestCommands.BatchQueryCommand,
                                                                      string.Empty),
                                                     GetRequestPayload(data));
            Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);

            using (var readerStream = new ReaderStream(await response.Content.ReadAsStreamAsync()))
            {
                var reader = readerStream.CreateBondedCompactBinaryReader<BatchQueryResponse>();
                var responseData = reader.Deserialize();
                Assert.AreEqual(3, responseData.Responses.Count);

                // unfiltered should have 100+200+300 hitcount
                VerifyHitCounter(responseData, "TotalTacos", 600);

                //only 100 chicken tacos
                VerifyHitCounter(responseData, "CluckCluck", 100);

                //sadly, there is no bacon...
                VerifyHitCounter(responseData, "BACON!", -1);
            }
        }
Beispiel #39
0
        public void SealAndReleaseAreThreadSafe()
        {
            var filterableDimension = new DimensionSet(new HashSet<Dimension> { new Dimension("thing") });
            using (
                var filterableBucket =
                    new DataBucket<InternalHitCount>(filterableDimension,
                                                                                this.timestamp,
                                                                                DefaultBucketTimeSpanTicks,
                                                                                this.currentDirectory,
                                                                                properties.MemoryStreamManager))
            {


                var allDims = new DimensionSpecification {{"thing", "thing"}};
                Parallel.For(0, 10, (i) => filterableBucket.AddValue(allDims, i));

                Parallel.For(0, 100, (i) =>
                                     {
                                         switch (i % 3)
                                         {
                                         case 0:
                                             foreach (var item in filterableBucket.GetMatches(allDims))
                                             {
                                                 Assert.IsNotNull(item);
                                             }
                                             break;

                                         case 1:
                                             filterableBucket.AddValue(allDims, 11);
                                             break;

                                         case 2:
                                             filterableBucket.ReleaseData();
                                             break;
                                         }
                                     });
            }
        }
Beispiel #40
0
        private void ValidateData(ulong expectedPermutations = 100, ulong expectedData = WrittenValue)
        {
            this.bucket.Seal();

            var uniqueKeys = new HashSet<string>();
            var dimensionValues = new DimensionSpecification();
            for (var i = 0; i < 10; ++i)
            {
                for (var j = 0; j < 10; ++j)
                {
                    dimensionValues["one"] = i.ToString();
                    dimensionValues["two"] = j.ToString();
                    var keyString =
                        this.bucket.DimensionSet.KeyToString(this.bucket.DimensionSet.CreateKey(dimensionValues));
                    uniqueKeys.Add(keyString);

                    var matchData = this.bucket.GetMatches(dimensionValues).ToList()[0];
                    Assert.AreEqual(1, matchData.DataCount);
                    Assert.AreEqual(expectedData, matchData.Data.HitCount);
                }
            }
            Assert.AreEqual(expectedPermutations, (ulong)uniqueKeys.Count);
        }
Beispiel #41
0
 /// <summary>
 /// Increment the counter for the provided dimensions.
 /// </summary>
 /// <param name="amount">Amount to increment the counter by.</param>
 /// <param name="dims">Full set of dimension values for the counter.</param>
 /// <param name="timestamp">Timestamp to use for the data point.</param>
 public void Increment(long amount, DimensionSpecification dims, DateTime timestamp)
 {
     this.hitCounter.AddValue(amount, dims, timestamp);
 }
        private void WriteRoundedSignificantDigitsValue(long value, DimensionSpecification dims, DateTime timestamp)
        {
            if (this.roundingFactor > 0 && value > this.minimumRoundingValue)
            {
                long factor = 0;
                while (value > this.minimumRoundingValue)
                {
                    factor += 1;
                    var lastDigit = value % 10;
                    if (lastDigit >= 5)
                    {
                        value += (10 - lastDigit);
                    }
                    value /= 10;
                }

                value *= (long)Math.Pow(10, factor);
            }

            this.histogram.AddValue(value, dims, timestamp);
        }
Beispiel #43
0
        private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters)
        {
            var counterInfo = new CounterInfo
                              {
                                  Name = counter.Name,
                                  Type = counter.Type,
                                  StartTime = counter.StartTime.ToMillisecondTimestamp(),
                                  EndTime = counter.EndTime.ToMillisecondTimestamp(),
                                  Dimensions = counter.Dimensions.ToList(),
                                  DimensionValues = null, // null this out by default to avoid response bloat.
                              };

            // Queries for dimension values will come with a 'dimension=pattern' query parameter.
            // Dimension values can be further filtered with '<dimensionName>=pattern'
            string dimensionPattern;
            if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern))
            {
                counterInfo.DimensionValues = new Dictionary<string, ISet<string>>();

                // We want to be able to filter dimension values by time (and only time)
                var dimensionQuery = new DimensionSpecification();
                string timeValue;
                if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue))
                {
                    timeValue = MinimumStartTime;
                }
                dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue;
                if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue))
                {
                    timeValue = MaximumEndTime;
                }
                dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue;

                foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern)))
                {
                    string filterPattern;
                    if (queryParameters.TryGetValue(dimensionPattern, out filterPattern))
                    {
                        
                        counterInfo.AddDimensionValues(dim,
                                                       counter.GetDimensionValues(dim, dimensionQuery)
                                                              .Where(dimensionValue =>
                                                                     dimensionValue.MatchGlob(filterPattern)));
                    }
                    else
                    {
                        counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery));
                    }
                }
            }

            return counterInfo;
        }
Beispiel #44
0
 private DimensionSpecification ProcessQueryParameters(DimensionSpecification queryParameters,
                                                       out QuerySpecification querySpec)
 {
     var filterDimensions = ExtractQuerySpec(queryParameters, out querySpec);
     if (querySpec.QueryType == QueryType.Average && !this.DataSet.SupportsAverageQuery)
     {
         querySpec.QueryType = QueryType.Normal;
     }
     else if (querySpec.QueryType == QueryType.Percentile && !this.DataSet.SupportsPercentileQuery)
     {
         querySpec.QueryType = QueryType.Normal;
     }
     return filterDimensions;
 }
 public void AddValue(long value, DimensionSpecification dims, DateTime timestamp)
 {
     this.writeValue(value, dims, timestamp);
 }
Beispiel #46
0
 /// <summary>
 /// Get all values for a particular dimension.
 /// </summary>
 /// <param name="dimensionName">Name of the dimension.</param>
 /// <param name="filterDims">Filter dimensions</param>
 /// <returns>An enumerator of values for the dimension (may be empty).</returns>
 public IEnumerable<string> GetDimensionValues(string dimensionName, DimensionSpecification filterDims)
 {
     if (filterDims == null)
     {
         throw new ArgumentNullException("filterDims");
     }
     return this.DataSet.GetDimensionValues(dimensionName, filterDims);
 }
Beispiel #47
0
        /// <summary>
        /// Execute a query against the counter data.
        /// </summary>
        /// <param name="queryParameters">Query parameters.</param>
        /// <returns>An enumeration of <see cref="DataSample"/>s matching the query parameters.</returns>
        public IEnumerable<DataSample> Query(DimensionSpecification queryParameters)
        {
            QuerySpecification querySpec;
            if (queryParameters == null)
            {
                throw new ArgumentNullException("queryParameters");
            }

            var filterDims = this.ProcessQueryParameters(queryParameters, out querySpec);
            if (filterDims == null)
            {
                return null;
            }

            return this.DataSet.QueryData(filterDims, querySpec);
        }
Beispiel #48
0
        public void CombinedBucketsAreMergedCorrectly()
        {
            var ts1 = new DateTime(2014, 05, 10, 0, 0, 0);
            var ts2 = new DateTime(2014, 05, 10, 0, 5, 0);
            var sharedDimensions = new DimensionSpecification { {"one", "a1"}, {"two", "a2"}};
            var bucket1Dimensions = new DimensionSpecification { {"one", "b1"}, {"two", "b2"}};
            var bucket2Dimensions = new DimensionSpecification { {"one", "c1"}, {"two", "c2"}};

            var bucket1 =
                new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1,
                                                                            TimeSpan.FromMinutes(5).Ticks, null,
                                                                            this.properties.MemoryStreamManager);
            bucket1.AddValue(sharedDimensions, 867);
            bucket1.AddValue(bucket1Dimensions, 5309);

            var bucket2 =
                new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2,
                                                                            TimeSpan.FromMinutes(5).Ticks, null,
                                                                            this.properties.MemoryStreamManager);
            bucket2.AddValue(sharedDimensions, 867);
            bucket2.AddValue(bucket2Dimensions, 42);

            bucket1.Seal();
            bucket2.Seal();
            var bucket3 =
                new DataBucket<InternalHitCount>(new[] {bucket1, bucket2},
                                                                            new DimensionSet(this.twoDimensionSet), ts1,
                                                                            TimeSpan.FromMinutes(10).Ticks, null,
                                                                            this.properties.MemoryStreamManager);

            var match = bucket3.GetMatches(bucket1Dimensions).First().Data;
            Assert.AreEqual((ulong)5309, match.HitCount);
            match = bucket3.GetMatches(bucket2Dimensions).First().Data;
            Assert.AreEqual((ulong)42, match.HitCount);

            match = bucket3.GetMatches(sharedDimensions).First().Data;
            Assert.AreEqual((ulong)867 * 2, match.HitCount);

            bucket1.Dispose();
            bucket2.Dispose();
            bucket3.Dispose();
        }
 private void WriteRandomTestData(int maxValue, TestKeyedDataStore store)
 {
     Parallel.For(0, maxValue,
                  i =>
                  {
                      var dims = new DimensionSpecification();
                      this.GenerateRandomKey(dims);
                      store.AddValue(dims, i);
                  });
 }
Beispiel #50
0
 private void WriteSomeData()
 {
     var dimensionValues = new DimensionSpecification();
     for (var i = 0; i < 10; ++i)
     {
         for (var j = 0; j < 10; ++j)
         {
             dimensionValues["one"] = i.ToString();
             dimensionValues["two"] = j.ToString();
             this.bucket.AddValue(dimensionValues, WrittenValue);
         }
     }
 }
Beispiel #51
0
        public void SplitByDimensionWithFiltersWorksProperly()
        {
            var filterableDimension = new DimensionSet(new HashSet<Dimension> {new Dimension("thing"), new Dimension("meat")});
            using (
                var filterableBucket =
                    new DataBucket<InternalHitCount>(filterableDimension, this.timestamp,
                                                                                DefaultBucketTimeSpanTicks,
                                                                                this.currentDirectory,
                                                                                this.properties.MemoryStreamManager))
            {
                var queryDimensions = new DimensionSpecification();
                queryDimensions["thing"] = "thingOne";
                queryDimensions["meat"] = "bacon";
                filterableBucket.AddValue(queryDimensions, 100);

                queryDimensions["thing"] = "thingTwo";
                queryDimensions["meat"] = "pepperoni";
                filterableBucket.AddValue(queryDimensions, 200);
                filterableBucket.Seal();

                // thingOne and thingTwo will match with no filter
                Assert.AreEqual(2,
                                filterableBucket.GetMatchesSplitByDimension(new DimensionSpecification(), "thing")
                                                .Sum(match => match.DataCount));

                // only thingOne matches bacon
                var bestMatchFilter = new DimensionSpecification {{"meat", "bacon"}};
                Assert.AreEqual(1,
                                filterableBucket.GetMatchesSplitByDimension(bestMatchFilter, "thing")
                                                .Sum(match => match.DataCount));
            }
        }
Beispiel #52
0
        internal static DimensionSpecification ExtractQuerySpec(DimensionSpecification queryParameters,
                                                                out QuerySpecification querySpec)
        {
            var filterDimensions = new DimensionSpecification();
            querySpec = new QuerySpecification();

            foreach (var param in queryParameters)
            {
                if (string.Equals(param.Key, ReservedDimensions.DimensionDimension, StringComparison.OrdinalIgnoreCase))
                {
                    querySpec.CrossQueryDimension = param.Value;

                    // Ensure that split dimension is not also sent a filter dimension
                    if (queryParameters.ContainsKey(param.Value))
                    {
                        return null;
                    }
                }
                else if (string.Equals(param.Key, ReservedDimensions.AggregateSamplesDimension, StringComparison.OrdinalIgnoreCase))
                {
                    querySpec.Combine = Convert.ToBoolean(param.Value);
                }
                else if (string.Equals(param.Key, ReservedDimensions.PercentileDimension, StringComparison.OrdinalIgnoreCase))
                {
                    if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForAverage,
                                      StringComparison.OrdinalIgnoreCase))
                    {
                        querySpec.QueryType = QueryType.Average;
                    }
                    else if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForMaximum,
                                           StringComparison.OrdinalIgnoreCase))
                    {
                        querySpec.QueryType = QueryType.Maximum;
                    }
                    else if (string.Equals(param.Value, ReservedDimensions.PercentileDimensionValueForMinimum,
                                           StringComparison.OrdinalIgnoreCase))
                    {
                        querySpec.QueryType = QueryType.Minimum;
                    }
                    else
                    {
                        querySpec.QueryType = QueryType.Percentile;
                        querySpec.Percentile = double.Parse(param.Value);
                    }
                }
                else
                {
                    filterDimensions.Add(param.Key, param.Value);
                }
            }

            return filterDimensions;
        }
 public void AddValue(long value, DimensionSpecification dims)
 {
     this.AddValue(value, dims, DateTime.Now);
 }
Beispiel #54
0
 /// <summary>
 /// Increment the counter by one for the provided dimensions using the current time for the data point.
 /// </summary>
 /// <param name="dims">Full set of dimension values for the counter.</param>
 public void Increment(DimensionSpecification dims)
 {
     this.Increment(dims, DateTime.Now);
 }
 private void WriteUnroundedValue(long value, DimensionSpecification dims, DateTime timestamp)
 {
     this.histogram.AddValue(value, dims, timestamp);
 }
Beispiel #56
0
 /// <summary>
 /// Increment the counter by one for the provided dimensions.
 /// </summary>
 /// <param name="dims">Full set of dimension values for the counter.</param>
 /// <param name="timestamp">Timestamp to use for the data point.</param>
 public void Increment(DimensionSpecification dims, DateTime timestamp)
 {
     this.Increment(1, dims, timestamp);
 }
        private void WriteRoundedSizeValue(long value, DimensionSpecification dims, DateTime timestamp)
        {
            if (value < 0)
            {
                value = 0;
            }

            if (value > this.roundingFactor)
            {
                long remainder = value % this.roundingFactor;
                if (remainder >= (this.roundingFactor / 2))
                {
                    remainder = -remainder; // will roll up
                }
                value = value - remainder;
            }

            this.histogram.AddValue(value, dims, timestamp);
        }
Beispiel #58
0
 /// <summary>
 /// Increment the counter for the provided dimensions using the current time for the data point.
 /// </summary>
 /// <param name="amount">Amount to increment the counter by.</param>
 /// <param name="dims">Full set of dimension values for the counter.</param>
 public void Increment(long amount, DimensionSpecification dims)
 {
     this.Increment(amount, dims, DateTime.Now);
 }
 private void GenerateRandomKey(DimensionSpecification dims)
 {
     dims.Clear();
     foreach (var d in this.dimensionSet.Dimensions)
     {
         dims.Add(d.Name, GetRandom(MaxDimensionValue).ToString());
     }
 }
Beispiel #60
0
        private static void RunOperation(Counter counter, CounterWriteOperation op)
        {
            var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow
                                ? DateTime.UtcNow
                                : op.Timestamp.ToDateTime();

            var dims = new DimensionSpecification(op.DimensionValues);

            var hitCounter = counter as HitCounter;
            var histogramCounter = counter as HistogramCounter;
            if (hitCounter != null)
            {
                hitCounter.Increment(op.Value * op.Count, dims, timestamp);
            }
            else
            {
                // It would be nice to direct-inject multiple values at once, but the APIs don't currently
                // support this. It's a reasonable amount of work to fix this and unknown whether folks will use this
                // a lot at this time.
                for (var i = 0; i < op.Count; ++i)
                {
                    histogramCounter.AddValue(op.Value, dims, timestamp);
                }
            }
        }