public IOScheduler(int maxSkiplistBacklog, UnsortedStorageLayer storageLayer, TrimDatabase database) { _storageLayer = storageLayer; _database = database; _sortedStrategy = (sl) => false; _unsortedStrategy = (sl) => sl.NumberOfTables > (sl.MaxFilesAtLayer * 0.8); _channel = Channel.CreateBounded <MemoryTable>(new BoundedChannelOptions(maxSkiplistBacklog)); _writerTask = WriteInMemoryTable(); _mergeTask = CheckForMerge(); }
private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer) { // We can't merge downwards at the bottom layer if (unsortedLayer.Level == _database.StorageLayers.Count) { return; } var nextLayer = _database.StorageLayers[unsortedLayer.Level]; var tables = unsortedLayer.GetTables(); var oldestTable = tables[0]; // Get all of the overlapping tables var overlapped = GetOverlappingTables(oldestTable, nextLayer); overlapped.Insert(0, oldestTable); Console.WriteLine($"Begin merging with {overlapped.Count} tables"); var sw = Stopwatch.StartNew(); await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray())) { // Begin writing out to disk var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level); await writer.WriteFromMerger(merger); nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped); unsortedLayer.RemoveTable(oldestTable); } foreach (var file in overlapped) { file.Dispose(); System.IO.File.Delete(file.FileName); } Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms"); }