private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer) { // We can't merge downwards at the bottom layer if (unsortedLayer.Level == _database.StorageLayers.Count) { return; } var nextLayer = _database.StorageLayers[unsortedLayer.Level]; var tables = unsortedLayer.GetTables(); var oldestTable = tables[0]; // Get all of the overlapping tables var overlapped = GetOverlappingTables(oldestTable, nextLayer); overlapped.Insert(0, oldestTable); Console.WriteLine($"Begin merging with {overlapped.Count} tables"); var sw = Stopwatch.StartNew(); await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray())) { // Begin writing out to disk var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level); await writer.WriteFromMerger(merger); nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped); unsortedLayer.RemoveTable(oldestTable); } foreach (var file in overlapped) { file.Dispose(); System.IO.File.Delete(file.FileName); } Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms"); }
private async Task MergeSortedLayer(SortedStorageLayer sortedStorage) { var layerBelow = _database.StorageLayers[sortedStorage.Level]; if (layerBelow.NumberOfTables == 0) { var file = sortedStorage.GetTables()[0]; await MoveTableToLowerLevel(sortedStorage, layerBelow, file); return; } var overlapCounts = new int[sortedStorage.NumberOfTables]; for (var i = 0; i < sortedStorage.GetTables().Length; i++) { var t = sortedStorage.GetTables()[i]; // Check if there is overlap var overlapCount = 0; foreach (var l3t in layerBelow.GetTables()) { if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0) { continue; } if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0) { continue; } overlapCount++; } if (overlapCount == 0) { // Move table down one level await MoveTableToLowerLevel(sortedStorage, layerBelow, t); return; } overlapCounts[i] = overlapCount; } var min = overlapCounts.Min(); var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index; var upperTable = sortedStorage.GetTables()[indexOfMin]; var overlapping = GetOverlappingTables(upperTable, layerBelow); overlapping.Insert(0, upperTable); Console.WriteLine($"Merging from {sortedStorage.Level}"); var sw = Stopwatch.StartNew(); await using (var merger = new TableFileMerger(overlapping.Select(ol => ol.GetAsyncEnumerator()).ToArray())) { // Begin writing out to disk var writer = new TableFileMergeWriter(layerBelow, _database.BlockCache); await writer.WriteFromMerger(merger); layerBelow.AddAndRemoveTableFiles(writer.NewTableFiles, overlapping); sortedStorage.RemoveTable(upperTable); } Console.WriteLine($"Merge for level {sortedStorage.Level} took {sw.ElapsedMilliseconds}ms"); foreach (var file in overlapping) { file.Dispose(); System.IO.File.Delete(file.FileName); } // Found with min overlap so merge it return; }