private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer)
{
// We can't merge downwards at the bottom layer
if (unsortedLayer.Level == _database.StorageLayers.Count)
{
return;
}
var nextLayer = _database.StorageLayers[unsortedLayer.Level];
var tables = unsortedLayer.GetTables();
var oldestTable = tables[0];
// Get all of the overlapping tables
var overlapped = GetOverlappingTables(oldestTable, nextLayer);
overlapped.Insert(0, oldestTable);
Console.WriteLine($"Begin merging with {overlapped.Count} tables");
var sw = Stopwatch.StartNew();
await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
{
// Begin writing out to disk
var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level);
await writer.WriteFromMerger(merger);
nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped);
unsortedLayer.RemoveTable(oldestTable);
}
foreach (var file in overlapped)
{
file.Dispose();
System.IO.File.Delete(file.FileName);
}
Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms");
}
private async Task MergeSortedLayer(SortedStorageLayer sortedStorage)
{
var layerBelow = _database.StorageLayers[sortedStorage.Level];
if (layerBelow.NumberOfTables == 0)
{
var file = sortedStorage.GetTables()[0];
await MoveTableToLowerLevel(sortedStorage, layerBelow, file);
return;
}
var overlapCounts = new int[sortedStorage.NumberOfTables];
for (var i = 0; i < sortedStorage.GetTables().Length; i++)
{
var t = sortedStorage.GetTables()[i];
// Check if there is overlap
var overlapCount = 0;
foreach (var l3t in layerBelow.GetTables())
{
if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0)
{
continue;
}
if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0)
{
continue;
}
overlapCount++;
}
if (overlapCount == 0)
{
// Move table down one level
await MoveTableToLowerLevel(sortedStorage, layerBelow, t);
return;
}
overlapCounts[i] = overlapCount;
}
var min = overlapCounts.Min();
var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index;
var upperTable = sortedStorage.GetTables()[indexOfMin];
var overlapping = GetOverlappingTables(upperTable, layerBelow);
overlapping.Insert(0, upperTable);
Console.WriteLine($"Merging from {sortedStorage.Level}");
var sw = Stopwatch.StartNew();
await using (var merger = new TableFileMerger(overlapping.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
{
// Begin writing out to disk
var writer = new TableFileMergeWriter(layerBelow, _database.BlockCache);
await writer.WriteFromMerger(merger);
layerBelow.AddAndRemoveTableFiles(writer.NewTableFiles, overlapping);
sortedStorage.RemoveTable(upperTable);
}
Console.WriteLine($"Merge for level {sortedStorage.Level} took {sw.ElapsedMilliseconds}ms");
foreach (var file in overlapping)
{
file.Dispose();
System.IO.File.Delete(file.FileName);
}
// Found with min overlap so merge it
return;
}
