private async Task <bool> WriteBlockAsync(TableFileMerger merger) { var memBlock = _filePipe.GetMemory(FileConsts.PageSize); var fullMemBlock = memBlock; var currentOffset = _metaData.BlockCount * FileConsts.PageSize; _metaData.AddBlockOffset(currentOffset, merger.Current.Key.ToArray()); do { //if (merger.Current.IsDeleted) //{ // // If the key doesn't exist in a level below we do not need to store a tombstone we can just remove completely // var doesExist = await _database.DoesKeyExistBelowLevel(merger.Current.Key.ToArray(), _lowestLevel); // if (doesExist == SearchResult.NotFound || doesExist == SearchResult.Deleted) continue; // sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length; //} //else //{ //} if (!WriteBlock(merger, ref memBlock)) { return(false); } } while (await merger.MoveNextAsync()); memBlock.Span.Fill(0); _filePipe.Advance(FileConsts.PageSize); _currentFileSize += FileConsts.PageSize; return(true); }
public async Task WriteFromMerger(TableFileMerger merger) { await merger.MoveNextAsync(); while (true) { // Set the first key as we must be at the start of a new file if (_metaData.FirstKey.Length == 0) { StartNewFile(); _metaData.FirstKey = merger.Current.Key.ToArray(); } var mergerFinished = await WriteBlockAsync(merger); await _filePipe.FlushAsync(); if (_currentFileSize >= _layer.MaxFileSize || mergerFinished) { await CloseOffCurrentTable(); _metaData.FirstKey = Array.Empty <byte>(); if (mergerFinished) { return; } } } }
private async Task <bool> WriteBlockAsync(TableFileMerger merger) { var memBlock = _filePipe.GetMemory(FileConsts.PageSize); var fullMemBlock = memBlock; var currentOffset = _metaData.BlockCount * FileConsts.PageSize; _metaData.AddBlockOffset(currentOffset, merger.Current.Key.ToArray()); do { var sizeNeeded = 0; sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length + merger.Current.Value.Length; if (sizeNeeded > memBlock.Length) { memBlock.Span.Fill(0); _filePipe.Advance(FileConsts.PageSize); _currentFileSize += FileConsts.PageSize; return(false); } _metaData.Count++; _metaData.Filter.AddKey(merger.Current.Key); BinaryPrimitives.WriteInt32LittleEndian(memBlock.Span, merger.Current.Key.Length); memBlock = memBlock[sizeof(int)..];
private async Task <bool> WriteBlock(TableFileMerger merger) { var memBlock = _filePipe.GetMemory(FileConsts.PageSize); var fullMemBlock = memBlock; var currentOffset = _metaData.BlockCount * FileConsts.PageSize; _metaData.AddBlockOffset(currentOffset, merger.Current.Key.ToArray()); do { var sizeNeeded = 0; if (merger.Current.IsDeleted) { var doesExist = await _database.DoesKeyExistBelowLevel(merger.Current.Key.ToArray(), _lowestLevel); if (doesExist == SearchResult.NotFound || doesExist == SearchResult.Deleted) { continue; } sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length; } else { sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length + merger.Current.Value.Length; } if (sizeNeeded > memBlock.Length) { memBlock.Span.Fill(0); _filePipe.Advance(FileConsts.PageSize); _currentFileSize += FileConsts.PageSize; return(false); } _metaData.Count++; _metaData.Filter.AddKey(merger.Current.Key); BinaryPrimitives.WriteInt32LittleEndian(memBlock.Span, merger.Current.Key.Length); memBlock = memBlock[sizeof(int)..];
private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer) { // We can't merge downwards at the bottom layer if (unsortedLayer.Level == _database.StorageLayers.Count) { return; } var nextLayer = _database.StorageLayers[unsortedLayer.Level]; var tables = unsortedLayer.GetTables(); var oldestTable = tables[0]; // Get all of the overlapping tables var overlapped = GetOverlappingTables(oldestTable, nextLayer); overlapped.Insert(0, oldestTable); Console.WriteLine($"Begin merging with {overlapped.Count} tables"); var sw = Stopwatch.StartNew(); await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray())) { // Begin writing out to disk var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level); await writer.WriteFromMerger(merger); nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped); unsortedLayer.RemoveTable(oldestTable); } foreach (var file in overlapped) { file.Dispose(); System.IO.File.Delete(file.FileName); } Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms"); }
private bool WriteBlock(TableFileMerger merger, ref Memory <byte> memBlock) { var span = memBlock.Span; var remainingLength = span.Length; var key = merger.Current.Key; var value = merger.Current.Value; var keyLength = key.Length; var valueLength = value.Length; var sizeNeeded = (sizeof(int) * 2) + keyLength + valueLength; if (sizeNeeded > remainingLength) { span.Fill(0); _filePipe.Advance(FileConsts.PageSize); _currentFileSize += FileConsts.PageSize; return(false); } _metaData.Count++; _metaData.Filter.AddKey(key); ref var currentPointer = ref MemoryMarshal.GetReference(span);
private async Task MergeSortedLayer(SortedStorageLayer sortedStorage) { var layerBelow = _database.StorageLayers[sortedStorage.Level]; if (layerBelow.NumberOfTables == 0) { var file = sortedStorage.GetTables()[0]; await MoveTableToLowerLevel(sortedStorage, layerBelow, file); return; } var overlapCounts = new int[sortedStorage.NumberOfTables]; for (var i = 0; i < sortedStorage.GetTables().Length; i++) { var t = sortedStorage.GetTables()[i]; // Check if there is overlap var overlapCount = 0; foreach (var l3t in layerBelow.GetTables()) { if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0) { continue; } if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0) { continue; } overlapCount++; } if (overlapCount == 0) { // Move table down one level await MoveTableToLowerLevel(sortedStorage, layerBelow, t); return; } overlapCounts[i] = overlapCount; } var min = overlapCounts.Min(); var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index; var upperTable = sortedStorage.GetTables()[indexOfMin]; var overlapping = GetOverlappingTables(upperTable, layerBelow); overlapping.Insert(0, upperTable); Console.WriteLine($"Merging from {sortedStorage.Level}"); var sw = Stopwatch.StartNew(); await using (var merger = new TableFileMerger(overlapping.Select(ol => ol.GetAsyncEnumerator()).ToArray())) { // Begin writing out to disk var writer = new TableFileMergeWriter(layerBelow, _database.BlockCache); await writer.WriteFromMerger(merger); layerBelow.AddAndRemoveTableFiles(writer.NewTableFiles, overlapping); sortedStorage.RemoveTable(upperTable); } Console.WriteLine($"Merge for level {sortedStorage.Level} took {sw.ElapsedMilliseconds}ms"); foreach (var file in overlapping) { file.Dispose(); System.IO.File.Delete(file.FileName); } // Found with min overlap so merge it return; }