private async Task <bool> WriteBlockAsync(TableFileMerger merger)
{
var memBlock = _filePipe.GetMemory(FileConsts.PageSize);
var fullMemBlock = memBlock;
var currentOffset = _metaData.BlockCount * FileConsts.PageSize;
_metaData.AddBlockOffset(currentOffset, merger.Current.Key.ToArray());
do
{
//if (merger.Current.IsDeleted)
//{
// // If the key doesn't exist in a level below we do not need to store a tombstone we can just remove completely
// var doesExist = await _database.DoesKeyExistBelowLevel(merger.Current.Key.ToArray(), _lowestLevel);
// if (doesExist == SearchResult.NotFound || doesExist == SearchResult.Deleted) continue;
// sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length;
//}
//else
//{
//}
if (!WriteBlock(merger, ref memBlock))
{
return(false);
}
} while (await merger.MoveNextAsync());
memBlock.Span.Fill(0);
_filePipe.Advance(FileConsts.PageSize);
_currentFileSize += FileConsts.PageSize;
return(true);
}
public async Task WriteFromMerger(TableFileMerger merger)
{
await merger.MoveNextAsync();
while (true)
{
// Set the first key as we must be at the start of a new file
if (_metaData.FirstKey.Length == 0)
{
StartNewFile();
_metaData.FirstKey = merger.Current.Key.ToArray();
}
var mergerFinished = await WriteBlockAsync(merger);
await _filePipe.FlushAsync();
if (_currentFileSize >= _layer.MaxFileSize || mergerFinished)
{
await CloseOffCurrentTable();
_metaData.FirstKey = Array.Empty <byte>();
if (mergerFinished)
{
return;
}
}
}
}
private async Task <bool> WriteBlockAsync(TableFileMerger merger)
{
var memBlock = _filePipe.GetMemory(FileConsts.PageSize);
var fullMemBlock = memBlock;
var currentOffset = _metaData.BlockCount * FileConsts.PageSize;
_metaData.AddBlockOffset(currentOffset, merger.Current.Key.ToArray());
do
{
var sizeNeeded = 0;
sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length + merger.Current.Value.Length;
if (sizeNeeded > memBlock.Length)
{
memBlock.Span.Fill(0);
_filePipe.Advance(FileConsts.PageSize);
_currentFileSize += FileConsts.PageSize;
return(false);
}
_metaData.Count++;
_metaData.Filter.AddKey(merger.Current.Key);
BinaryPrimitives.WriteInt32LittleEndian(memBlock.Span, merger.Current.Key.Length);
memBlock = memBlock[sizeof(int)..];
private async Task <bool> WriteBlock(TableFileMerger merger)
{
var memBlock = _filePipe.GetMemory(FileConsts.PageSize);
var fullMemBlock = memBlock;
var currentOffset = _metaData.BlockCount * FileConsts.PageSize;
_metaData.AddBlockOffset(currentOffset, merger.Current.Key.ToArray());
do
{
var sizeNeeded = 0;
if (merger.Current.IsDeleted)
{
var doesExist = await _database.DoesKeyExistBelowLevel(merger.Current.Key.ToArray(), _lowestLevel);
if (doesExist == SearchResult.NotFound || doesExist == SearchResult.Deleted)
{
continue;
}
sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length;
}
else
{
sizeNeeded = (sizeof(int) * 2) + merger.Current.Key.Length + merger.Current.Value.Length;
}
if (sizeNeeded > memBlock.Length)
{
memBlock.Span.Fill(0);
_filePipe.Advance(FileConsts.PageSize);
_currentFileSize += FileConsts.PageSize;
return(false);
}
_metaData.Count++;
_metaData.Filter.AddKey(merger.Current.Key);
BinaryPrimitives.WriteInt32LittleEndian(memBlock.Span, merger.Current.Key.Length);
memBlock = memBlock[sizeof(int)..];
private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer)
{
// We can't merge downwards at the bottom layer
if (unsortedLayer.Level == _database.StorageLayers.Count)
{
return;
}
var nextLayer = _database.StorageLayers[unsortedLayer.Level];
var tables = unsortedLayer.GetTables();
var oldestTable = tables[0];
// Get all of the overlapping tables
var overlapped = GetOverlappingTables(oldestTable, nextLayer);
overlapped.Insert(0, oldestTable);
Console.WriteLine($"Begin merging with {overlapped.Count} tables");
var sw = Stopwatch.StartNew();
await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
{
// Begin writing out to disk
var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level);
await writer.WriteFromMerger(merger);
nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped);
unsortedLayer.RemoveTable(oldestTable);
}
foreach (var file in overlapped)
{
file.Dispose();
System.IO.File.Delete(file.FileName);
}
Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms");
}
private bool WriteBlock(TableFileMerger merger, ref Memory <byte> memBlock)
{
var span = memBlock.Span;
var remainingLength = span.Length;
var key = merger.Current.Key;
var value = merger.Current.Value;
var keyLength = key.Length;
var valueLength = value.Length;
var sizeNeeded = (sizeof(int) * 2) + keyLength + valueLength;
if (sizeNeeded > remainingLength)
{
span.Fill(0);
_filePipe.Advance(FileConsts.PageSize);
_currentFileSize += FileConsts.PageSize;
return(false);
}
_metaData.Count++;
_metaData.Filter.AddKey(key);
ref var currentPointer = ref MemoryMarshal.GetReference(span);
private async Task MergeSortedLayer(SortedStorageLayer sortedStorage)
{
var layerBelow = _database.StorageLayers[sortedStorage.Level];
if (layerBelow.NumberOfTables == 0)
{
var file = sortedStorage.GetTables()[0];
await MoveTableToLowerLevel(sortedStorage, layerBelow, file);
return;
}
var overlapCounts = new int[sortedStorage.NumberOfTables];
for (var i = 0; i < sortedStorage.GetTables().Length; i++)
{
var t = sortedStorage.GetTables()[i];
// Check if there is overlap
var overlapCount = 0;
foreach (var l3t in layerBelow.GetTables())
{
if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0)
{
continue;
}
if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0)
{
continue;
}
overlapCount++;
}
if (overlapCount == 0)
{
// Move table down one level
await MoveTableToLowerLevel(sortedStorage, layerBelow, t);
return;
}
overlapCounts[i] = overlapCount;
}
var min = overlapCounts.Min();
var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index;
var upperTable = sortedStorage.GetTables()[indexOfMin];
var overlapping = GetOverlappingTables(upperTable, layerBelow);
overlapping.Insert(0, upperTable);
Console.WriteLine($"Merging from {sortedStorage.Level}");
var sw = Stopwatch.StartNew();
await using (var merger = new TableFileMerger(overlapping.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
{
// Begin writing out to disk
var writer = new TableFileMergeWriter(layerBelow, _database.BlockCache);
await writer.WriteFromMerger(merger);
layerBelow.AddAndRemoveTableFiles(writer.NewTableFiles, overlapping);
sortedStorage.RemoveTable(upperTable);
}
Console.WriteLine($"Merge for level {sortedStorage.Level} took {sw.ElapsedMilliseconds}ms");
foreach (var file in overlapping)
{
file.Dispose();
System.IO.File.Delete(file.FileName);
}
// Found with min overlap so merge it
return;
}
