private static async Task PracticeL2ToL3Merge(string dbFolder)
{
using var blockStore = new ProtoSharded(2_560);
var l2 = new SortedStorageLayer(2, dbFolder, blockStore, 0, 0);
var l3 = new SortedStorageLayer(3, dbFolder, blockStore, 0, 0);
MergeInFile(l2, l3);
}
private static async Task CheckLayer(string dbFolder, int level)
{
using var blockStore = new Core.Storage.Blocks.CachePrototype.ProtoSharded(2_560);
var sortedLayer = new SortedStorageLayer(level, dbFolder, blockStore, 0);
await sortedLayer.LoadLayer();
var firstLast = sortedLayer.GetFirstAndLastKeys().ToArray();
for (var outer = 0; outer < firstLast.Length; outer++)
{
for (var inner = 0; inner < firstLast.Length; inner++)
{
if (inner == outer)
{
continue;
}
var i = firstLast[inner];
var o = firstLast[outer];
var compareStart = i.firstKey.Span.SequenceCompareTo(o.firstKey.Span);
if (compareStart < 0)
{
// The start is before the block so overlap if the finish is after the
// next start
var compareFinish = i.lastKey.Span.SequenceCompareTo(o.firstKey.Span);
if (compareFinish > 0)
{
throw new InvalidOperationException("We found an overlap");
}
}
if (compareStart > 0)
{
// The start is after the start of the outer block if the start is before
// the end then there is an overlap
var compareFinish = i.firstKey.Span.SequenceCompareTo(o.lastKey.Span);
if (compareFinish < 0)
{
throw new InvalidOperationException("We found an overlap");
}
}
}
}
}
private static void MergeInFile(SortedStorageLayer l2, SortedStorageLayer l3)
{
if (l3.NumberOfTables == 0)
{
Console.WriteLine("Moved table down with no merge");
// Move table down one level
return;
}
var overlapCounts = new int[l2.NumberOfTables];
for (var i = 0; i < l2.GetTables().Length; i++)
{
var t = l2.GetTables()[i];
// Check if there is overlap
var overlapCount = 0;
foreach (var l3t in l3.GetTables())
{
if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0)
{
continue;
}
if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0)
{
continue;
}
overlapCount++;
break;
}
if (overlapCount == 0)
{
// Move table down one level
Console.WriteLine("Moved table down with no merge");
return;
}
overlapCounts[i] = overlapCount;
}
var min = overlapCounts.Min();
var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index;
// Found with min overlap so merge it
return;
}
static async Task Main(string[] args)
{
//var merge = new MergeBenchmark();
//merge.GlobalSetup();
//await merge.MergeFiles();
//return;
var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
return;
var dbFolder = "D:\\stress";
var outputFolder = "D:\\stressoutput";
Directory.Delete(outputFolder, true);
Directory.CreateDirectory(outputFolder);
var dbOptions = new TrimDatabaseOptions();
using var blockstore = dbOptions.BlockCache();
var filenames = Directory.GetFiles(dbFolder);
var files = new TableFile[filenames.Length];
for (var i = 0; i < files.Length; i++)
{
var tf = new TableFile(filenames[i], blockstore);
await tf.LoadAsync();
files[i] = tf;
}
var merger = new TableFileMerger(files.Select(f => f.GetAsyncEnumerator()).ToArray());
var storageLayer = new SortedStorageLayer(5, dbFolder, blockstore, 10 * 1024 * 1024, 100);
var mWriter = new TableFileMergeWriter(storageLayer, blockstore);
await mWriter.WriteFromMerger(merger);
}
