public async Task MergeFiles2() { var merger = new TableFileMerger(_tableFiles.Select(f => f.GetAsyncEnumerator()).ToArray()); var mWriter = new TableFileMergeWriter2(_storageLayer, _cache, loadNewFiles: false); await mWriter.WriteFromMerger(merger); }
public async Task CheckMergeWorks() { var firstList = GetAsync(new byte[] { 1, 5, 7, 10 }); var secondList = GetAsync(new byte[] { 1, 6, 7, 10, 12 }); var merger = new TableFileMerger(new[] { firstList.GetAsyncEnumerator(), secondList.GetAsyncEnumerator() }); var result = new List <int>(); while (await merger.MoveNextAsync()) { result.Add(merger.Current.Key[0]); } var expected = new List <int>() { 1, 5, 6, 7, 10, 12 }; Assert.Equal(expected, result); }
public async Task OverlappedThreeWay() { var firstList = GetAsync(new byte[] { 10, 20, 30, 40, 50, 60 }); var secondList = GetAsync(new byte[] { 5, 10, 20, 30 }); var thirdList = GetAsync(new byte[] { 50, 60, 70, 80, 90, 100, 110 }); var merger = new TableFileMerger(new[] { firstList.GetAsyncEnumerator(), secondList.GetAsyncEnumerator(), thirdList.GetAsyncEnumerator() }); var result = new List <int>(); while (await merger.MoveNextAsync()) { result.Add(merger.Current.Key[0]); } var expected = new List <int>() { 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110 }; Assert.Equal(expected, result); }
static async Task Main(string[] args) { //var merge = new MergeBenchmark(); //merge.GlobalSetup(); //await merge.MergeFiles(); //return; var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); return; var dbFolder = "D:\\stress"; var outputFolder = "D:\\stressoutput"; Directory.Delete(outputFolder, true); Directory.CreateDirectory(outputFolder); var dbOptions = new TrimDatabaseOptions(); using var blockstore = dbOptions.BlockCache(); var filenames = Directory.GetFiles(dbFolder); var files = new TableFile[filenames.Length]; for (var i = 0; i < files.Length; i++) { var tf = new TableFile(filenames[i], blockstore); await tf.LoadAsync(); files[i] = tf; } var merger = new TableFileMerger(files.Select(f => f.GetAsyncEnumerator()).ToArray()); var storageLayer = new SortedStorageLayer(5, dbFolder, blockstore, 10 * 1024 * 1024, 100); var mWriter = new TableFileMergeWriter(storageLayer, blockstore); await mWriter.WriteFromMerger(merger); }