public unsafe void CanMergeManySources()
{
const int writeCount = 100000;
const int maxKeyValue = 20; // we want a lot of collisions.
var data = new BufferedKeyedData <int> [20];
var dimensionSet = DimensionSetTests.CreateDimensionSet(1);
for (var i = 0; i < data.Length; ++i)
{
data[i] =
new BufferedKeyedData <int>(
new byte[
BufferedKeyedData <int> .GetBufferSizeForKeyCount(writeCount / data.Length, dimensionSet)], 0,
0, new DimensionSet(dimensionSet), true);
}
var rng = new Random();
var expected = new Dictionary <uint, List <int> >();
for (int i = 0; i < writeCount; ++i)
{
var key = (uint)rng.Next(maxKeyValue);
Assert.IsTrue(data[i % data.Length].TryWrite(&key, i));
List <int> expectedValuesForKey;
if (!expected.TryGetValue(key, out expectedValuesForKey))
{
expectedValuesForKey = new List <int>();
expected.Add(key, expectedValuesForKey);
}
expectedValuesForKey.Add(i);
}
var dataToMerge = new List <IEnumerable <KeyValuePair <Key, IMergeSource> > >(data.Length);
foreach (var d in data)
{
d.Seal();
d.Sort();
var chunk = new List <KeyValuePair <Key, IMergeSource> >(d.Count);
chunk.AddRange(d.Select(kvp => new KeyValuePair <Key, IMergeSource>(kvp.Key.Clone() as Key, new LongData(kvp.Value))));
dataToMerge.Add(chunk);
}
Key currentKey = null;
foreach (var kvp in KeyedDataMerge <LongData> .MergeSorted(dataToMerge))
{
Assert.IsTrue(currentKey < kvp.Key);
currentKey = kvp.Key.Clone() as Key;
var expectedData = expected[kvp.Key[0]];
Assert.AreEqual(expectedData.Sum(), kvp.Value.Value);
expected.Remove(kvp.Key[0]);
}
}
public void CanMergeEmptyData()
{
Assert.IsFalse(KeyedDataMerge <LongData> .MergeSorted(
new[]
{
new List <KeyValuePair <Key, IMergeSource> >(),
new List <KeyValuePair <Key, IMergeSource> >(),
}).Any());
}
public void CanMergeZeroDepthData()
{
var left = new LongData(867);
var right = new LongData(5309);
var leftData = new List <KeyValuePair <Key, IMergeSource> >
{
new KeyValuePair <Key, IMergeSource>(new Key(new uint[0]), left),
};
var rightData = new List <KeyValuePair <Key, IMergeSource> >
{
new KeyValuePair <Key, IMergeSource>(new Key(new uint[0]), right),
};
var merged = KeyedDataMerge <LongData> .MergeSorted(new[] { leftData, rightData }).ToList();
Assert.AreEqual(1, merged.Count);
Assert.AreEqual(left.Value + right.Value, merged[0].Value.Value);
}
public void CanMergeDataWhereOneSideIsEmpty()
{
var firstKey = new LongData(867);
var secondKey = new LongData(5309);
var fullData = new List <KeyValuePair <Key, IMergeSource> >
{
new KeyValuePair <Key, IMergeSource>(new Key(new[] { (uint)firstKey.Value }), firstKey),
new KeyValuePair <Key, IMergeSource>(new Key(new[] { (uint)secondKey.Value }), secondKey),
};
var emptyData = new List <KeyValuePair <Key, IMergeSource> >();
// Validate both left and right being the full side.
foreach (var pair in
new[]
{
new[] { fullData, emptyData },
new[] { emptyData, fullData },
})
{
var count = 0;
foreach (var kvp in KeyedDataMerge <LongData> .MergeSorted(pair))
{
++count;
if (kvp.Key.Values[0] == (uint)firstKey.Value)
{
Assert.AreEqual(firstKey.Value, kvp.Value.Value);
}
else if (kvp.Key.Values[0] == (uint)secondKey.Value)
{
Assert.AreEqual(secondKey.Value, kvp.Value.Value);
}
else
{
Assert.Fail();
}
}
Assert.AreEqual(2, count);
}
}
public void MergeTreesCombinesData()
{
const int possibleValuesByDimension = 100;
var leftData = new List <KeyValuePair <Key, IMergeSource> >();
var rightData = new List <KeyValuePair <Key, IMergeSource> >();
var insertKey = new Key(new uint[3]);
// Create a reasonably complex pair of trees which overlap when all keys are divisible by 6, and are
// somewhat sparse.
var expectedCount = 0;
var expectedOverlap = 0;
for (uint d1 = 0; d1 < possibleValuesByDimension; ++d1)
{
insertKey.Values[0] = d1;
var left1 = d1 % 2 == 0;
var right1 = d1 % 3 == 0;
for (uint d2 = 0; d2 < possibleValuesByDimension; ++d2)
{
insertKey.Values[1] = d2;
var left2 = d2 % 2 == 0;
var right2 = d2 % 3 == 0;
for (uint d3 = 0; d3 < possibleValuesByDimension; ++d3)
{
insertKey.Values[2] = d3;
var left3 = d3 % 2 == 0;
var right3 = d3 % 3 == 0;
var value = (d1 + d2 + d3);
var wroteValue = false;
if (left1 && left2 && left3)
{
leftData.Add(new KeyValuePair <Key, IMergeSource>(new Key(insertKey), new LongData((int)value)));
wroteValue = true;
}
if (right1 && right2 && right3)
{
rightData.Add(new KeyValuePair <Key, IMergeSource>(new Key(insertKey), new LongData((int)value)));
expectedOverlap += (wroteValue ? 1 : 0);
wroteValue = true;
}
expectedCount += (wroteValue ? 1 : 0);
}
}
}
var count = 0;
var overlapCount = 0;
foreach (var kvp in KeyedDataMerge <LongData> .MergeSorted(new[] { leftData, rightData }))
{
var key = kvp.Key;
var value = kvp.Value;
++count;
var expectedValues = new List <uint>();
var match = false;
if (key.Values[0] % 2 == 0 && key.Values[1] % 2 == 0 && key.Values[2] % 2 == 0)
{
expectedValues.Add(key.Values[0] + key.Values[1] + key.Values[2]);
match = true;
}
if (key.Values[0] % 3 == 0 && key.Values[1] % 3 == 0 && key.Values[2] % 3 == 0)
{
if (match)
{
++overlapCount;
}
expectedValues.Add(key.Values[0] + key.Values[1] + key.Values[2]);
}
Assert.AreEqual(expectedValues.Sum(x => x), value.Value);
}
Assert.AreEqual(expectedCount, count);
Assert.AreEqual(expectedOverlap, overlapCount);
}
