Пример #1
0
        private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer)
        {
            // We can't merge downwards at the bottom layer
            if (unsortedLayer.Level == _database.StorageLayers.Count)
            {
                return;
            }

            var nextLayer = _database.StorageLayers[unsortedLayer.Level];

            var tables      = unsortedLayer.GetTables();
            var oldestTable = tables[0];

            // Get all of the overlapping tables
            var overlapped = GetOverlappingTables(oldestTable, nextLayer);

            overlapped.Insert(0, oldestTable);

            Console.WriteLine($"Begin merging with {overlapped.Count} tables");
            var sw = Stopwatch.StartNew();

            await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
            {
                // Begin writing out to disk
                var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level);
                await writer.WriteFromMerger(merger);

                nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped);
                unsortedLayer.RemoveTable(oldestTable);
            }

            foreach (var file in overlapped)
            {
                file.Dispose();
                System.IO.File.Delete(file.FileName);
            }

            Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms");
        }
Пример #2
0
        private async Task MergeSortedLayer(SortedStorageLayer sortedStorage)
        {
            var layerBelow = _database.StorageLayers[sortedStorage.Level];

            if (layerBelow.NumberOfTables == 0)
            {
                var file = sortedStorage.GetTables()[0];
                await MoveTableToLowerLevel(sortedStorage, layerBelow, file);

                return;
            }

            var overlapCounts = new int[sortedStorage.NumberOfTables];

            for (var i = 0; i < sortedStorage.GetTables().Length; i++)
            {
                var t = sortedStorage.GetTables()[i];

                // Check if there is overlap
                var overlapCount = 0;

                foreach (var l3t in layerBelow.GetTables())
                {
                    if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0)
                    {
                        continue;
                    }
                    if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0)
                    {
                        continue;
                    }
                    overlapCount++;
                }

                if (overlapCount == 0)
                {
                    // Move table down one level
                    await MoveTableToLowerLevel(sortedStorage, layerBelow, t);

                    return;
                }
                overlapCounts[i] = overlapCount;
            }

            var min         = overlapCounts.Min();
            var indexOfMin  = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index;
            var upperTable  = sortedStorage.GetTables()[indexOfMin];
            var overlapping = GetOverlappingTables(upperTable, layerBelow);

            overlapping.Insert(0, upperTable);

            Console.WriteLine($"Merging from {sortedStorage.Level}");
            var sw = Stopwatch.StartNew();

            await using (var merger = new TableFileMerger(overlapping.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
            {
                // Begin writing out to disk
                var writer = new TableFileMergeWriter(layerBelow, _database.BlockCache);
                await writer.WriteFromMerger(merger);

                layerBelow.AddAndRemoveTableFiles(writer.NewTableFiles, overlapping);
                sortedStorage.RemoveTable(upperTable);
            }
            Console.WriteLine($"Merge for level {sortedStorage.Level} took {sw.ElapsedMilliseconds}ms");
            foreach (var file in overlapping)
            {
                file.Dispose();
                System.IO.File.Delete(file.FileName);
            }

            // Found with min overlap so merge it
            return;
        }