public void PivotData_Merge()
{
var pvtData = new PivotData(new string[] { "a", "year", "month" },
new AverageAggregatorFactory("i"));
var pvtData1 = new PivotData(new string[] { "a", "year", "month" },
new AverageAggregatorFactory("i"));
var pvtData2 = new PivotData(new string[] { "a", "year", "month" },
new AverageAggregatorFactory("i"));
var pvtData3 = new PivotData(new string[] { "a", "b" }, new CountAggregatorFactory());
Assert.Throws <ArgumentException>(() => {
pvtData1.Merge(pvtData3);
});
pvtData.ProcessData(SampleGenerator(20000), GetRecordValue);
pvtData1.ProcessData(SampleGenerator(10000), GetRecordValue);
pvtData2.ProcessData(SampleGenerator(10000, 10000), GetRecordValue);
pvtData1.Merge(pvtData2);
foreach (var v in pvtData.AllValues)
{
var aggr = pvtData[v.Key];
var aggrMerged = pvtData1[v.Key];
Assert.Equal(aggr.Count, aggrMerged.Count);
Assert.Equal(aggr.Value, aggrMerged.Value);
}
Assert.Equal(pvtData.GetDimensionKeys()[0].Length, pvtData1.GetDimensionKeys()[0].Length);
Assert.Equal(pvtData.GetDimensionKeys()[1].Length, pvtData1.GetDimensionKeys()[1].Length);
Assert.Equal(pvtData.GetDimensionKeys()[2].Length, pvtData1.GetDimensionKeys()[2].Length);
}
static void Main(string[] args)
{
var fieldToIdx = new Dictionary <string, int>()
{
{ "name", 0 },
{ "age", 1 },
{ "company", 2 },
{ "date", 3 },
{ "hours", 4 },
};
Func <object, string, object> getValue = (r, f) => {
return(((object[])r)[fieldToIdx[f]]);
};
var dataChunks = GetDataChunks();
Console.WriteLine("Generating 50 data chunks (each 100,000 records) on the fly = 5 mln in total");
var dimensions = new string[] {
"company", "name", "age", "date"
};
var aggrFactory = new SumAggregatorFactory("hours");
var allPvtData = new PivotData(dimensions, aggrFactory, true);
Parallel.ForEach(dataChunks, (t) => {
var chunkPvtData = new PivotData(dimensions, aggrFactory, true);
chunkPvtData.ProcessData(t, getValue);
Console.WriteLine("Calculated pivot data chunk, aggregated values: {0}", chunkPvtData.Count);
lock (allPvtData) {
allPvtData.Merge(chunkPvtData);
}
});
Console.WriteLine("Parallel calculation of cube finished.\nTotal dimensions: {0}\nTotal aggregated values: {1}\nTotal hours: {2}",
dimensions.Length, allPvtData.Count,
allPvtData[Key.Empty, Key.Empty, Key.Empty, Key.Empty].Value);
Console.ReadKey();
}
