private static async Task PracticeL2ToL3Merge(string dbFolder) { using var blockStore = new ProtoSharded(2_560); var l2 = new SortedStorageLayer(2, dbFolder, blockStore, 0, 0); var l3 = new SortedStorageLayer(3, dbFolder, blockStore, 0, 0); MergeInFile(l2, l3); }
private static async Task CheckLayer(string dbFolder, int level) { using var blockStore = new Core.Storage.Blocks.CachePrototype.ProtoSharded(2_560); var sortedLayer = new SortedStorageLayer(level, dbFolder, blockStore, 0); await sortedLayer.LoadLayer(); var firstLast = sortedLayer.GetFirstAndLastKeys().ToArray(); for (var outer = 0; outer < firstLast.Length; outer++) { for (var inner = 0; inner < firstLast.Length; inner++) { if (inner == outer) { continue; } var i = firstLast[inner]; var o = firstLast[outer]; var compareStart = i.firstKey.Span.SequenceCompareTo(o.firstKey.Span); if (compareStart < 0) { // The start is before the block so overlap if the finish is after the // next start var compareFinish = i.lastKey.Span.SequenceCompareTo(o.firstKey.Span); if (compareFinish > 0) { throw new InvalidOperationException("We found an overlap"); } } if (compareStart > 0) { // The start is after the start of the outer block if the start is before // the end then there is an overlap var compareFinish = i.firstKey.Span.SequenceCompareTo(o.lastKey.Span); if (compareFinish < 0) { throw new InvalidOperationException("We found an overlap"); } } } } }
private static void MergeInFile(SortedStorageLayer l2, SortedStorageLayer l3) { if (l3.NumberOfTables == 0) { Console.WriteLine("Moved table down with no merge"); // Move table down one level return; } var overlapCounts = new int[l2.NumberOfTables]; for (var i = 0; i < l2.GetTables().Length; i++) { var t = l2.GetTables()[i]; // Check if there is overlap var overlapCount = 0; foreach (var l3t in l3.GetTables()) { if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0) { continue; } if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0) { continue; } overlapCount++; break; } if (overlapCount == 0) { // Move table down one level Console.WriteLine("Moved table down with no merge"); return; } overlapCounts[i] = overlapCount; } var min = overlapCounts.Min(); var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index; // Found with min overlap so merge it return; }
static async Task Main(string[] args) { //var merge = new MergeBenchmark(); //merge.GlobalSetup(); //await merge.MergeFiles(); //return; var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); return; var dbFolder = "D:\\stress"; var outputFolder = "D:\\stressoutput"; Directory.Delete(outputFolder, true); Directory.CreateDirectory(outputFolder); var dbOptions = new TrimDatabaseOptions(); using var blockstore = dbOptions.BlockCache(); var filenames = Directory.GetFiles(dbFolder); var files = new TableFile[filenames.Length]; for (var i = 0; i < files.Length; i++) { var tf = new TableFile(filenames[i], blockstore); await tf.LoadAsync(); files[i] = tf; } var merger = new TableFileMerger(files.Select(f => f.GetAsyncEnumerator()).ToArray()); var storageLayer = new SortedStorageLayer(5, dbFolder, blockstore, 10 * 1024 * 1024, 100); var mWriter = new TableFileMergeWriter(storageLayer, blockstore); await mWriter.WriteFromMerger(merger); }