public void HistogramCompactionIsCorrect()
{
const int numBuckets = 5;
const int samplesPerBucket = 100;
var buckets = new DataBucket <InternalHistogram> [numBuckets];
for (int i = 0; i < numBuckets; i++)
{
var newBucket =
new DataBucket <InternalHistogram>(this.dimensionSet, DateTime.Now,
TimeSpan.FromMinutes(1).Ticks,
this.storagePath,
this.properties.MemoryStreamManager);
buckets[i] = newBucket;
for (int sample = 0; sample < samplesPerBucket; sample++)
{
newBucket.AddValue(this.dimensions, sample);
}
newBucket.Seal();
}
var compactedBucket =
new DataBucket <InternalHistogram>(buckets, this.dimensionSet, DateTime.Now,
TimeSpan.FromMinutes(60).Ticks,
this.storagePath,
this.properties.MemoryStreamManager);
var data = compactedBucket.GetMatches(this.dimensions);
var singleVal = data.First().Data;
Assert.IsNotNull(singleVal);
Assert.AreEqual(numBuckets * samplesPerBucket, (int)singleVal.SampleCount);
compactedBucket.Dispose();
}
public void ExistingDataFilesAreCompactedOnCompactCall()
{
// Create one bucket at present time so that remaining buckets
// being created are out of compaction
var timeStamp = DateTime.UtcNow;
var bucket =
new DataBucket <InternalHitCount>(this.dimensionSet, timeStamp, TimeSpan.TicksPerMinute,
this.storagePath,
this.properties.MemoryStreamManager);
bucket.AddValue(this.dimensions, 5);
bucket.Seal();
bucket.Persist();
// Create buckets manually at 1 min and 5 min quantum
// but they actually belong at a 10 min once compacted
var expectedNewBucketQuantum = customCompaction.Intervals.ElementAt(2).Interval.Ticks;
var timeStamp2 = RoundTimeStamp(timeStamp.Subtract(new TimeSpan(2, 20, 0)), expectedNewBucketQuantum);
// The buckets will have files written out but not be
// part of the dataset until it is created
var bucket1 =
new DataBucket <InternalHitCount>(this.dimensionSet, timeStamp2, TimeSpan.TicksPerMinute,
this.storagePath,
this.properties.MemoryStreamManager);
bucket1.AddValue(this.dimensions, 5);
bucket1.Seal();
bucket1.Persist();
var timeStamp3 = timeStamp2.Add(new TimeSpan(0, 5, 0));
var bucket2 =
new DataBucket <InternalHitCount>(this.dimensionSet, timeStamp3,
TimeSpan.TicksPerMinute * 5,
this.storagePath,
this.properties.MemoryStreamManager);
bucket2.AddValue(this.dimensions, 2);
bucket2.Seal();
bucket2.Persist();
bucket.Dispose();
bucket1.Dispose();
bucket2.Dispose();
this.InitializeDataSet(null);
this.dataSet.Compact();
// Verify that a 10 minute compacted bucket was created
var newBucket = this.dataSet.GetDataBucket(timeStamp2);
Assert.AreEqual(timeStamp2.Ticks, newBucket.StartTicks);
Assert.AreEqual(expectedNewBucketQuantum, newBucket.EndTicks - newBucket.StartTicks);
var matches = newBucket.GetMatches(this.dimensions).ToList();
Assert.AreEqual(1, matches.Count);
var result = matches[0].Data;
Assert.AreEqual((ulong)7, result.HitCount);
}
public void CombinedBucketsAreMergedCorrectly()
{
var ts1 = new DateTime(2014, 05, 10, 0, 0, 0);
var ts2 = new DateTime(2014, 05, 10, 0, 5, 0);
var sharedDimensions = new DimensionSpecification {
{ "one", "a1" }, { "two", "a2" }
};
var bucket1Dimensions = new DimensionSpecification {
{ "one", "b1" }, { "two", "b2" }
};
var bucket2Dimensions = new DimensionSpecification {
{ "one", "c1" }, { "two", "c2" }
};
var bucket1 =
new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket1.AddValue(sharedDimensions, 867);
bucket1.AddValue(bucket1Dimensions, 5309);
var bucket2 =
new DataBucket <InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket2.AddValue(sharedDimensions, 867);
bucket2.AddValue(bucket2Dimensions, 42);
bucket1.Seal();
bucket2.Seal();
var bucket3 =
new DataBucket <InternalHitCount>(new[] { bucket1, bucket2 },
new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(10).Ticks, null,
this.properties.MemoryStreamManager);
var match = bucket3.GetMatches(bucket1Dimensions).First().Data;
Assert.AreEqual((ulong)5309, match.HitCount);
match = bucket3.GetMatches(bucket2Dimensions).First().Data;
Assert.AreEqual((ulong)42, match.HitCount);
match = bucket3.GetMatches(sharedDimensions).First().Data;
Assert.AreEqual((ulong)867 * 2, match.HitCount);
bucket1.Dispose();
bucket2.Dispose();
bucket3.Dispose();
}
public void HistogramCompactionIsCorrect()
{
const int numBuckets = 5;
const int samplesPerBucket = 100;
var buckets = new DataBucket<InternalHistogram>[numBuckets];
for (int i = 0; i < numBuckets; i++)
{
var newBucket =
new DataBucket<InternalHistogram>(this.dimensionSet, DateTime.Now,
TimeSpan.FromMinutes(1).Ticks,
this.storagePath,
this.properties.MemoryStreamManager);
buckets[i] = newBucket;
for (int sample = 0; sample < samplesPerBucket; sample++)
{
newBucket.AddValue(this.dimensions, sample);
}
newBucket.Seal();
}
var compactedBucket =
new DataBucket<InternalHistogram>(buckets, this.dimensionSet, DateTime.Now,
TimeSpan.FromMinutes(60).Ticks,
this.storagePath,
this.properties.MemoryStreamManager);
var data = compactedBucket.GetMatches(this.dimensions);
var singleVal = data.First().Data;
Assert.IsNotNull(singleVal);
Assert.AreEqual(numBuckets * samplesPerBucket, (int)singleVal.SampleCount);
compactedBucket.Dispose();
}
public void ExistingDataFilesAreCompactedOnCompactCall()
{
// Create one bucket at present time so that remaining buckets
// being created are out of compaction
var timeStamp = DateTime.UtcNow;
var bucket =
new DataBucket<InternalHitCount>(this.dimensionSet, timeStamp, TimeSpan.TicksPerMinute,
this.storagePath,
this.properties.MemoryStreamManager);
bucket.AddValue(this.dimensions, 5);
bucket.Seal();
bucket.Persist();
// Create buckets manually at 1 min and 5 min quantum
// but they actually belong at a 10 min once compacted
var expectedNewBucketQuantum = customCompaction.Intervals.ElementAt(2).Interval.Ticks;
var timeStamp2 = RoundTimeStamp(timeStamp.Subtract(new TimeSpan(2, 20, 0)), expectedNewBucketQuantum);
// The buckets will have files written out but not be
// part of the dataset until it is created
var bucket1 =
new DataBucket<InternalHitCount>(this.dimensionSet, timeStamp2, TimeSpan.TicksPerMinute,
this.storagePath,
this.properties.MemoryStreamManager);
bucket1.AddValue(this.dimensions, 5);
bucket1.Seal();
bucket1.Persist();
var timeStamp3 = timeStamp2.Add(new TimeSpan(0, 5, 0));
var bucket2 =
new DataBucket<InternalHitCount>(this.dimensionSet, timeStamp3,
TimeSpan.TicksPerMinute * 5,
this.storagePath,
this.properties.MemoryStreamManager);
bucket2.AddValue(this.dimensions, 2);
bucket2.Seal();
bucket2.Persist();
bucket.Dispose();
bucket1.Dispose();
bucket2.Dispose();
this.InitializeDataSet(null);
this.dataSet.Compact();
// Verify that a 10 minute compacted bucket was created
var newBucket = this.dataSet.GetDataBucket(timeStamp2);
Assert.AreEqual(timeStamp2.Ticks, newBucket.StartTicks);
Assert.AreEqual(expectedNewBucketQuantum, newBucket.EndTicks - newBucket.StartTicks);
var matches = newBucket.GetMatches(this.dimensions).ToList();
Assert.AreEqual(1, matches.Count);
var result = matches[0].Data;
Assert.AreEqual((ulong)7, result.HitCount);
}
public void CombinedBucketsAreMergedCorrectly()
{
var ts1 = new DateTime(2014, 05, 10, 0, 0, 0);
var ts2 = new DateTime(2014, 05, 10, 0, 5, 0);
var sharedDimensions = new DimensionSpecification { {"one", "a1"}, {"two", "a2"}};
var bucket1Dimensions = new DimensionSpecification { {"one", "b1"}, {"two", "b2"}};
var bucket2Dimensions = new DimensionSpecification { {"one", "c1"}, {"two", "c2"}};
var bucket1 =
new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket1.AddValue(sharedDimensions, 867);
bucket1.AddValue(bucket1Dimensions, 5309);
var bucket2 =
new DataBucket<InternalHitCount>(new DimensionSet(this.twoDimensionSet), ts2,
TimeSpan.FromMinutes(5).Ticks, null,
this.properties.MemoryStreamManager);
bucket2.AddValue(sharedDimensions, 867);
bucket2.AddValue(bucket2Dimensions, 42);
bucket1.Seal();
bucket2.Seal();
var bucket3 =
new DataBucket<InternalHitCount>(new[] {bucket1, bucket2},
new DimensionSet(this.twoDimensionSet), ts1,
TimeSpan.FromMinutes(10).Ticks, null,
this.properties.MemoryStreamManager);
var match = bucket3.GetMatches(bucket1Dimensions).First().Data;
Assert.AreEqual((ulong)5309, match.HitCount);
match = bucket3.GetMatches(bucket2Dimensions).First().Data;
Assert.AreEqual((ulong)42, match.HitCount);
match = bucket3.GetMatches(sharedDimensions).First().Data;
Assert.AreEqual((ulong)867 * 2, match.HitCount);
bucket1.Dispose();
bucket2.Dispose();
bucket3.Dispose();
}
