public void CombinedBucketsAreMergedCorrectly() { var ts1 = new DateTime(2014, 05, 10, 0, 0, 0); var ts2 = new DateTime(2014, 05, 10, 0, 5, 0); var sharedDimensions = new DimensionSpecification { { "one", "a1" }, { "two", "a2" } }; var bucket1Dimensions = new DimensionSpecification { { "one", "b1" }, { "two", "b2" } }; var bucket2Dimensions = new DimensionSpecification { { "one", "c1" }, { "two", "c2" } }; var bucket1 = new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket1.AddValue(sharedDimensions, 867); bucket1.AddValue(bucket1Dimensions, 5309); var bucket2 = new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket2.AddValue(sharedDimensions, 867); bucket2.AddValue(bucket2Dimensions, 42); bucket1.Seal(); bucket2.Seal(); var bucket3 = new DataBucket <InternalHitCount>(new[] { bucket1, bucket2 }, new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(10).Ticks, null, this.properties.MemoryStreamManager); var match = bucket3.GetMatches(bucket1Dimensions).First().Data; Assert.AreEqual((ulong)5309, match.HitCount); match = bucket3.GetMatches(bucket2Dimensions).First().Data; Assert.AreEqual((ulong)42, match.HitCount); match = bucket3.GetMatches(sharedDimensions).First().Data; Assert.AreEqual((ulong)867 * 2, match.HitCount); bucket1.Dispose(); bucket2.Dispose(); bucket3.Dispose(); }
public void HistogramCompactionIsCorrect() { const int numBuckets = 5; const int samplesPerBucket = 100; var buckets = new DataBucket <InternalHistogram> [numBuckets]; for (int i = 0; i < numBuckets; i++) { var newBucket = new DataBucket <InternalHistogram>(this.dimensionSet, DateTime.Now, TimeSpan.FromMinutes(1).Ticks, this.storagePath, this.properties.MemoryStreamManager); buckets[i] = newBucket; for (int sample = 0; sample < samplesPerBucket; sample++) { newBucket.AddValue(this.dimensions, sample); } newBucket.Seal(); } var compactedBucket = new DataBucket <InternalHistogram>(buckets, this.dimensionSet, DateTime.Now, TimeSpan.FromMinutes(60).Ticks, this.storagePath, this.properties.MemoryStreamManager); var data = compactedBucket.GetMatches(this.dimensions); var singleVal = data.First().Data; Assert.IsNotNull(singleVal); Assert.AreEqual(numBuckets * samplesPerBucket, (int)singleVal.SampleCount); compactedBucket.Dispose(); }
public void ExistingDataFilesAreCompactedOnCompactCall() { // Create one bucket at present time so that remaining buckets // being created are out of compaction var timeStamp = DateTime.UtcNow; var bucket = new DataBucket <InternalHitCount>(this.dimensionSet, timeStamp, TimeSpan.TicksPerMinute, this.storagePath, this.properties.MemoryStreamManager); bucket.AddValue(this.dimensions, 5); bucket.Seal(); bucket.Persist(); // Create buckets manually at 1 min and 5 min quantum // but they actually belong at a 10 min once compacted var expectedNewBucketQuantum = customCompaction.Intervals.ElementAt(2).Interval.Ticks; var timeStamp2 = RoundTimeStamp(timeStamp.Subtract(new TimeSpan(2, 20, 0)), expectedNewBucketQuantum); // The buckets will have files written out but not be // part of the dataset until it is created var bucket1 = new DataBucket <InternalHitCount>(this.dimensionSet, timeStamp2, TimeSpan.TicksPerMinute, this.storagePath, this.properties.MemoryStreamManager); bucket1.AddValue(this.dimensions, 5); bucket1.Seal(); bucket1.Persist(); var timeStamp3 = timeStamp2.Add(new TimeSpan(0, 5, 0)); var bucket2 = new DataBucket <InternalHitCount>(this.dimensionSet, timeStamp3, TimeSpan.TicksPerMinute * 5, this.storagePath, this.properties.MemoryStreamManager); bucket2.AddValue(this.dimensions, 2); bucket2.Seal(); bucket2.Persist(); bucket.Dispose(); bucket1.Dispose(); bucket2.Dispose(); this.InitializeDataSet(null); this.dataSet.Compact(); // Verify that a 10 minute compacted bucket was created var newBucket = this.dataSet.GetDataBucket(timeStamp2); Assert.AreEqual(timeStamp2.Ticks, newBucket.StartTicks); Assert.AreEqual(expectedNewBucketQuantum, newBucket.EndTicks - newBucket.StartTicks); var matches = newBucket.GetMatches(this.dimensions).ToList(); Assert.AreEqual(1, matches.Count); var result = matches[0].Data; Assert.AreEqual((ulong)7, result.HitCount); }
public void SealAndReleaseAreThreadSafe() { var filterableDimension = new DimensionSet(new HashSet <Dimension> { new Dimension("thing") }); using ( var filterableBucket = new DataBucket <InternalHitCount>(filterableDimension, this.timestamp, DefaultBucketTimeSpanTicks, this.currentDirectory, properties.MemoryStreamManager)) { var allDims = new DimensionSpecification { { "thing", "thing" } }; Parallel.For(0, 10, (i) => filterableBucket.AddValue(allDims, i)); Parallel.For(0, 100, (i) => { switch (i % 3) { case 0: foreach (var item in filterableBucket.GetMatches(allDims)) { Assert.IsNotNull(item); } break; case 1: filterableBucket.AddValue(allDims, 11); break; case 2: filterableBucket.ReleaseData(); break; } }); } }
public void SplitByDimensionWithFiltersWorksProperly() { var filterableDimension = new DimensionSet(new HashSet <Dimension> { new Dimension("thing"), new Dimension("meat") }); using ( var filterableBucket = new DataBucket <InternalHitCount>(filterableDimension, this.timestamp, DefaultBucketTimeSpanTicks, this.currentDirectory, this.properties.MemoryStreamManager)) { var queryDimensions = new DimensionSpecification(); queryDimensions["thing"] = "thingOne"; queryDimensions["meat"] = "bacon"; filterableBucket.AddValue(queryDimensions, 100); queryDimensions["thing"] = "thingTwo"; queryDimensions["meat"] = "pepperoni"; filterableBucket.AddValue(queryDimensions, 200); filterableBucket.Seal(); // thingOne and thingTwo will match with no filter Assert.AreEqual(2, filterableBucket.GetMatchesSplitByDimension(new DimensionSpecification(), "thing") .Sum(match => match.DataCount)); // only thingOne matches bacon var bestMatchFilter = new DimensionSpecification { { "meat", "bacon" } }; Assert.AreEqual(1, filterableBucket.GetMatchesSplitByDimension(bestMatchFilter, "thing") .Sum(match => match.DataCount)); } }
public void HistogramCompactionIsCorrect() { const int numBuckets = 5; const int samplesPerBucket = 100; var buckets = new DataBucket<InternalHistogram>[numBuckets]; for (int i = 0; i < numBuckets; i++) { var newBucket = new DataBucket<InternalHistogram>(this.dimensionSet, DateTime.Now, TimeSpan.FromMinutes(1).Ticks, this.storagePath, this.properties.MemoryStreamManager); buckets[i] = newBucket; for (int sample = 0; sample < samplesPerBucket; sample++) { newBucket.AddValue(this.dimensions, sample); } newBucket.Seal(); } var compactedBucket = new DataBucket<InternalHistogram>(buckets, this.dimensionSet, DateTime.Now, TimeSpan.FromMinutes(60).Ticks, this.storagePath, this.properties.MemoryStreamManager); var data = compactedBucket.GetMatches(this.dimensions); var singleVal = data.First().Data; Assert.IsNotNull(singleVal); Assert.AreEqual(numBuckets * samplesPerBucket, (int)singleVal.SampleCount); compactedBucket.Dispose(); }
public void ExistingDataFilesAreCompactedOnCompactCall() { // Create one bucket at present time so that remaining buckets // being created are out of compaction var timeStamp = DateTime.UtcNow; var bucket = new DataBucket<InternalHitCount>(this.dimensionSet, timeStamp, TimeSpan.TicksPerMinute, this.storagePath, this.properties.MemoryStreamManager); bucket.AddValue(this.dimensions, 5); bucket.Seal(); bucket.Persist(); // Create buckets manually at 1 min and 5 min quantum // but they actually belong at a 10 min once compacted var expectedNewBucketQuantum = customCompaction.Intervals.ElementAt(2).Interval.Ticks; var timeStamp2 = RoundTimeStamp(timeStamp.Subtract(new TimeSpan(2, 20, 0)), expectedNewBucketQuantum); // The buckets will have files written out but not be // part of the dataset until it is created var bucket1 = new DataBucket<InternalHitCount>(this.dimensionSet, timeStamp2, TimeSpan.TicksPerMinute, this.storagePath, this.properties.MemoryStreamManager); bucket1.AddValue(this.dimensions, 5); bucket1.Seal(); bucket1.Persist(); var timeStamp3 = timeStamp2.Add(new TimeSpan(0, 5, 0)); var bucket2 = new DataBucket<InternalHitCount>(this.dimensionSet, timeStamp3, TimeSpan.TicksPerMinute * 5, this.storagePath, this.properties.MemoryStreamManager); bucket2.AddValue(this.dimensions, 2); bucket2.Seal(); bucket2.Persist(); bucket.Dispose(); bucket1.Dispose(); bucket2.Dispose(); this.InitializeDataSet(null); this.dataSet.Compact(); // Verify that a 10 minute compacted bucket was created var newBucket = this.dataSet.GetDataBucket(timeStamp2); Assert.AreEqual(timeStamp2.Ticks, newBucket.StartTicks); Assert.AreEqual(expectedNewBucketQuantum, newBucket.EndTicks - newBucket.StartTicks); var matches = newBucket.GetMatches(this.dimensions).ToList(); Assert.AreEqual(1, matches.Count); var result = matches[0].Data; Assert.AreEqual((ulong)7, result.HitCount); }
public void CombinedBucketsAreMergedCorrectly() { var ts1 = new DateTime(2014, 05, 10, 0, 0, 0); var ts2 = new DateTime(2014, 05, 10, 0, 5, 0); var sharedDimensions = new DimensionSpecification { {"one", "a1"}, {"two", "a2"}}; var bucket1Dimensions = new DimensionSpecification { {"one", "b1"}, {"two", "b2"}}; var bucket2Dimensions = new DimensionSpecification { {"one", "c1"}, {"two", "c2"}}; var bucket1 = new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket1.AddValue(sharedDimensions, 867); bucket1.AddValue(bucket1Dimensions, 5309); var bucket2 = new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2, TimeSpan.FromMinutes(5).Ticks, null, this.properties.MemoryStreamManager); bucket2.AddValue(sharedDimensions, 867); bucket2.AddValue(bucket2Dimensions, 42); bucket1.Seal(); bucket2.Seal(); var bucket3 = new DataBucket<InternalHitCount>(new[] {bucket1, bucket2}, new DimensionSet(this.twoDimensionSet), ts1, TimeSpan.FromMinutes(10).Ticks, null, this.properties.MemoryStreamManager); var match = bucket3.GetMatches(bucket1Dimensions).First().Data; Assert.AreEqual((ulong)5309, match.HitCount); match = bucket3.GetMatches(bucket2Dimensions).First().Data; Assert.AreEqual((ulong)42, match.HitCount); match = bucket3.GetMatches(sharedDimensions).First().Data; Assert.AreEqual((ulong)867 * 2, match.HitCount); bucket1.Dispose(); bucket2.Dispose(); bucket3.Dispose(); }
public void SealAndReleaseAreThreadSafe() { var filterableDimension = new DimensionSet(new HashSet<Dimension> { new Dimension("thing") }); using ( var filterableBucket = new DataBucket<InternalHitCount>(filterableDimension, this.timestamp, DefaultBucketTimeSpanTicks, this.currentDirectory, properties.MemoryStreamManager)) { var allDims = new DimensionSpecification {{"thing", "thing"}}; Parallel.For(0, 10, (i) => filterableBucket.AddValue(allDims, i)); Parallel.For(0, 100, (i) => { switch (i % 3) { case 0: foreach (var item in filterableBucket.GetMatches(allDims)) { Assert.IsNotNull(item); } break; case 1: filterableBucket.AddValue(allDims, 11); break; case 2: filterableBucket.ReleaseData(); break; } }); } }
public void SplitByDimensionWithFiltersWorksProperly() { var filterableDimension = new DimensionSet(new HashSet<Dimension> {new Dimension("thing"), new Dimension("meat")}); using ( var filterableBucket = new DataBucket<InternalHitCount>(filterableDimension, this.timestamp, DefaultBucketTimeSpanTicks, this.currentDirectory, this.properties.MemoryStreamManager)) { var queryDimensions = new DimensionSpecification(); queryDimensions["thing"] = "thingOne"; queryDimensions["meat"] = "bacon"; filterableBucket.AddValue(queryDimensions, 100); queryDimensions["thing"] = "thingTwo"; queryDimensions["meat"] = "pepperoni"; filterableBucket.AddValue(queryDimensions, 200); filterableBucket.Seal(); // thingOne and thingTwo will match with no filter Assert.AreEqual(2, filterableBucket.GetMatchesSplitByDimension(new DimensionSpecification(), "thing") .Sum(match => match.DataCount)); // only thingOne matches bacon var bestMatchFilter = new DimensionSpecification {{"meat", "bacon"}}; Assert.AreEqual(1, filterableBucket.GetMatchesSplitByDimension(bestMatchFilter, "thing") .Sum(match => match.DataCount)); } }