static void Main(string[] args) { // prueba 1 int[] temp1 = { 1, 3, 4, 7, 9, 13, 15, 17 }; int[] temp2 = { 3, 6, 9, 12, 45, 78 }; int[] temp3 = { 4, 6, 8, 13, 46, 57 }; //foreach (var el in Merge.MergeEnumerators<int>(new ComparaEnteros(), ((IEnumerable<int>)temp1).GetEnumerator(), ((IEnumerable<int>)temp2).GetEnumerator(), // ((IEnumerable<int>)temp3).GetEnumerator())) // Console.WriteLine(el); IEnumerator <int> my = new MergeEnumerator <int>(new ComparaEnteros(), ((IEnumerable <int>)temp1).GetEnumerator(), ((IEnumerable <int>)temp2).GetEnumerator(), ((IEnumerable <int>)temp3).GetEnumerator()); while (my.MoveNext()) { Console.WriteLine(my.Current); } my.Reset(); Console.WriteLine(); while (my.MoveNext()) { Console.WriteLine(my.Current); } }
public void TestMergeIterator() { Random r = new Random(); int totalElements = 0; // Create 10 randomly sized lists of random numbers List <IEnumerable <int> > collections = new List <IEnumerable <int> >(); for (int i = 0; i < 10; i++) { List <int> randomData = new List <int>(); for (int j = 0; j < r.Next(100); j++) { randomData.Add(r.Next()); totalElements++; } collections.Add(randomData); } // Sort all the individual lists var sortedCollections = collections.Select(list => list.OrderBy(i => i).AsEnumerable()); // Now scan through the merged list and make sure the result is ordered int lastNum = int.MinValue; int numElements = 0; foreach (var num in MergeEnumerator.Merge(sortedCollections)) { Assert.LessOrEqual(lastNum, num); lastNum = num; numElements++; } Assert.AreEqual(totalElements, numElements); }
public void TestEmptyMergeIterator() { var enumerators = new List <IEnumerable <int> >(); Assert.AreEqual(0, MergeEnumerator.Merge(enumerators).Count()); enumerators.Add(new List <int>().OrderBy(e => e)); enumerators.Add(new List <int>().OrderBy(e => e)); enumerators.Add(new List <int>().OrderBy(e => e)); Assert.AreEqual(0, MergeEnumerator.Merge(enumerators).Count()); }
public void LevelMergeReadTest() { string path = Path.GetFullPath("TestData\\LevelMergeReadTest"); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } int num_tables_to_merge = 4; int items_per_table = 2500; int totalData = 0; for (int i = 0; i < num_tables_to_merge; i++) { var mt = new MemTable(); for (int j = 0; j < items_per_table; j++) { var randKey = Key.Random(40); var randVal = Value.Random(512); mt.Add(randKey, randVal); } mt.WriteToSortedBlockTable("TestData\\LevelMergeReadTest", 0, i); totalData += mt.Size; } var tables = new List <IEnumerable <KeyValuePair <Key, Value> > >(); var sbts = new List <SortedBlockTable>(); var cache = new RazorCache(); for (int j = 0; j < num_tables_to_merge; j++) { var sbt = new SortedBlockTable(cache, "TestData\\LevelMergeReadTest", 0, j); tables.Add(sbt.Enumerate()); sbts.Add(sbt); } int ct = 0; Key key = Key.FromBytes(new byte[] { 0, 0 }); var timer = new Stopwatch(); timer.Start(); foreach (var pair in MergeEnumerator.Merge(tables, p => p.Key)) { Assert.True(key.CompareTo(pair.Key) < 0); key = pair.Key; ct++; } timer.Stop(); sbts.ForEach(s => s.Close()); Console.WriteLine("Scanned through a multilevel merge at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); }
private FileMetadata WriteMergedTable(Version version, MergeEnumerator mergeEnumerator, ref int count) { var newFileMeta = new FileMetadata { FileNumber = version.GetNewFileNumber() }; var newFileInfo = new FileInfo(GetTableFileName(newFileMeta.FileNumber)); using FileStream newFileStream = newFileInfo.Create(); var creator = new TableCreator(newFileStream); byte[] smallestKey = null; byte[] largestKey = null; ReadOnlyMemory <byte> prevKey = null; while (mergeEnumerator.MoveNext()) { BlockEntry entry = mergeEnumerator.Current; count++; ReadOnlyMemory <byte> key = entry.Key; if (prevKey.Length != 0) { if (prevKey.Span.UserKey().SequenceEqual(key.Span.UserKey())) { Log.Warn($"Duplicate keys - Prev Key: {prevKey.ToHexString()}, Key: {key.ToHexString()}"); continue; } } prevKey = key; creator.Add(key.Span, entry.Data.Span); smallestKey ??= key.ToArray(); largestKey = key.ToArray(); if (creator.CurrentSize > Options.MaxTableFileSize) { break; } } creator.Finish(); newFileMeta.SmallestKey = smallestKey; newFileMeta.LargestKey = largestKey; newFileInfo.Refresh(); newFileMeta.FileSize = (ulong)newFileInfo.Length; return(newFileMeta); }
public void TestDuplicateMergeIterator() { var enumerators = new List <IEnumerable <int> >(); enumerators.Add(Enumerable.Range(1, 10)); Assert.AreEqual(55, MergeEnumerator.Merge(enumerators).Sum()); enumerators = new List <IEnumerable <int> >(); enumerators.Add(Enumerable.Range(1, 10)); enumerators.Add(Enumerable.Range(2, 9)); enumerators.Add(Enumerable.Range(1, 10)); Assert.AreEqual(55, MergeEnumerator.Merge(enumerators).Sum()); enumerators = new List <IEnumerable <int> >(); enumerators.Add(new List <int>()); enumerators.Add(Enumerable.Range(2, 9)); enumerators.Add(Enumerable.Range(1, 10)); Assert.AreEqual(55, MergeEnumerator.Merge(enumerators).Sum()); }
public InnerHandler(IAsyncEnumerator <TSource> source, MergeEnumerator parent, CancellationTokenSource cts) { _source = source; _parent = parent; _cts = cts; }
private static IEnumerable <SegmentationInput.Segment> SplitOverlappingSegments(List <SegmentationInput.Segment[]> sampleSegments) { var starts = MergeEnumerator.Merge(sampleSegments.Select(segments => segments.Select(segment => segment.start))); var ends = MergeEnumerator.Merge(sampleSegments.Select(segments => segments.Select(segment => segment.end))); var partitions = MergeEnumerator.Merge(new[]
public void CompactNumeric() { var keys = new List <byte[]>(); DirectoryInfo dir = TestUtils.GetTestDirectory(false); // Setup new database and generate values enough to create 2 level 0 tables with overlapping keys. // We use this when we run the real test. ulong idx = 0; var options = new Options() { LevelSizeBaseFactor = 10, RetainAllFiles = true }; List <FileMetadata> level0Files; Version version = null; using (var db = new Database(dir, true, options)) { db.Open(); for (int j = 0; j < 4; j++) { for (int i = 0; i < 8000; i++) { byte[] key = BitConverter.GetBytes(idx++); byte[] data = TestUtils.FillArrayWithRandomBytes(1000, 128); db.Put(key, data); keys.Add(key); } } level0Files = new List <FileMetadata>(db.Level0Tables); version = db.Version; db.Close(); } ((Hierarchy)LogManager.GetRepository(Assembly.GetEntryAssembly())).Root.Level = Level.Warn; { Log.Warn($"Reading {keys.Count} values using regular db.get()"); using (var db = new Database(dir, false, options)) { db.Open(); ulong count = 0; ulong countMissing = 0; foreach (byte[] key in keys) { byte[] value = db.Get(key); if (value == null) { Log.Error($"Missing key {key.ToHexString()} at idx:{count}, {countMissing++}"); } count++; } db.Close(); } } return; //{ // Log.Warn($"Reading {keys.Count} values, from log files"); // List<byte[]> keysToRemove = new List<byte[]>(keys); // FileInfo[] logFiles = dir.GetFiles("*.log"); // foreach (FileInfo fileInfo in logFiles) // { // Log.Warn($"Reading from {fileInfo.Name}. Have {keysToRemove.Count} keys left"); // using var reader = new LogReader(fileInfo.Open(FileMode.Open)); // var cache = new MemCache(); // cache.Load(reader); // foreach (byte[] key in keysToRemove.Take(5000).ToArray()) // { // if (cache.Get(key).State == ResultState.Exist) // { // keysToRemove.Remove(key); // } // } // } // Assert.AreEqual(0, keysToRemove.Count); //} int keysInLevel0 = 0; var keysInCurrentLog = new List <byte[]>(); { Log.Warn($"Reading {keys.Count} values, from level0 files"); List <byte[]> keysToRemove = new List <byte[]>(keys); var enumerators = new List <TableEnumerator>(); foreach (FileMetadata fileMeta in level0Files.OrderBy(f => f.FileNumber)) { string filePath = Path.Combine(dir.FullName, $"{fileMeta.FileNumber:000000}.ldb"); var fileInfo = new FileInfo(filePath); Log.Warn($"Reading from {fileInfo.Name}. Have {keysToRemove.Count} keys left"); var table = new Table(fileInfo); foreach (byte[] key in keysToRemove.ToArray()) { if (table.Get(key).State == ResultState.Exist) { keysInLevel0++; keysToRemove.Remove(key); } } enumerators.Add((TableEnumerator)table.GetEnumerator()); } Assert.Less(0, keysInLevel0); // Read the remaining from current log file { string filePath = Path.Combine(dir.FullName, $"{version.LogNumber:000000}.log"); var fileInfo = new FileInfo(filePath); Log.Warn($"Reading remaining {keysToRemove.Count} values from current log {fileInfo.Name}"); using var reader = new LogReader(fileInfo.Open(FileMode.Open)); var cache = new MemCache(); cache.Load(reader); foreach (byte[] key in keysToRemove.ToArray()) { if (cache.Get(key).State == ResultState.Exist) { keysInCurrentLog.Add(key); keysToRemove.Remove(key); } } Assert.AreEqual(0, keysToRemove.Count); } { Log.Warn($"Reading {keysInLevel0} values, based on merge enumerator of all level0 table files"); var enumerator = new MergeEnumerator(enumerators); int enumCount = 0; while (enumerator.MoveNext()) { enumCount++; } Assert.AreEqual(keysInLevel0, enumCount); // Close the tables foreach (TableEnumerator tableEnumerator in enumerators) { tableEnumerator.TEST_Close(); } } } { var keysLeftToRemove = new List <byte[]>(keys).Except(keysInCurrentLog).ToList(); Log.Warn($"Reading {keysLeftToRemove.Count} values, from all level+1 files + current level0"); var level1Enumerators = new List <TableEnumerator>(); FileInfo[] tableFiles = dir.GetFiles("*.ldb"); foreach (var fileInfo in tableFiles.OrderBy(f => f.Name)) { if (level0Files.Any(f => $"{f.FileNumber:000000}.ldb" == fileInfo.Name)) { if (version.GetFiles(0).All(f => $"{f.FileNumber:000000}.ldb" != fileInfo.Name)) { continue; } Log.Warn($"Reading current level0 file {fileInfo.Name}"); } Log.Warn($"Reading from {fileInfo.Name}. Have {keysLeftToRemove.Count} keys left"); var table = new Table(fileInfo); table.Initialize(); level1Enumerators.Add((TableEnumerator)table.GetEnumerator()); foreach (byte[] key in keysLeftToRemove.ToArray()) { if (table.Get(key).State == ResultState.Exist) { keysLeftToRemove.Remove(key); } } } //Assert.AreEqual(0, keysLeftToRemove.Count); // FAIL { keysLeftToRemove = new List <byte[]>(keys).Except(keysInCurrentLog).ToList(); Log.Warn($"Reading {keysLeftToRemove.Count} values, from all level+1 files + current level0 using merge enumerator"); var enumerator = new MergeEnumerator(level1Enumerators); int enumCount = 0; while (enumerator.MoveNext()) { enumCount++; if (enumerator.Current != null) { byte[] key = enumerator.Current.Key.Span.UserKey().ToArray(); keysLeftToRemove.RemoveAll(bytes => new BytewiseComparator().Compare(bytes, key) == 0); } else { Log.Warn($"Current in enumerator is null"); } } Assert.AreEqual(keys.Count - keysInCurrentLog.Count, enumCount, "Expected to have count of all keys"); Assert.AreEqual(0, keysLeftToRemove.Count, "Expected to have found all keys"); foreach (TableEnumerator tableEnumerator in level1Enumerators) { tableEnumerator.TEST_Close(); } } } Log.Warn($"Done!"); }
private void TryCompactLevelN() { Log.Debug($"Trying to do merge compact of level n to level .."); if (!_dbLock.IsWriteLockHeld) { Log.Error("Panic!"); throw new PanicException("Tried to compact without holding a write lock. Abort!"); } Version version = _manifest.CurrentVersion; bool manifestIsDirty = false; foreach (KeyValuePair <int, List <FileMetadata> > kvp in version.Levels.Where(kvp => kvp.Key > 0).ToArray()) { int level = kvp.Key; List <FileMetadata> files = kvp.Value; if (files.Count > Math.Pow(Options.LevelSizeBaseFactor, level)) { manifestIsDirty = true; Log.Warn($"Want to compact level {level} with {files.Count} files."); while (true) { FileMetadata fileToCompact = GetNextFileToCompactForLevel(files, version.GetCompactPointer(level)); if (fileToCompact == null) { break; // Nah, shouldn't happen, right. } if (files.Count <= Math.Pow(Options.LevelSizeBaseFactor, level)) { break; } List <FileMetadata> overlappingFiles = version.GetOverlappingFiles(level + 1, fileToCompact.SmallestKey, fileToCompact.LargestKey); if (overlappingFiles.Count == 0) { // Level+1 has no files overlapping. So we can basically just move this file up the level. Log.Warn($"Want to compact level {level}, file {fileToCompact.FileNumber} and level {level + 1} had No overlapping files. Simple move."); version.RemoveFile(level, fileToCompact.FileNumber); version.AddFile(level + 1, fileToCompact); version.SetCompactPointer(level, fileToCompact.LargestKey); } else { Log.Warn($"Want to compact level {level}, file {fileToCompact.FileNumber} and level {level + 1} had {overlappingFiles.Count} overlapping files"); var mergeFiles = new List <FileMetadata>(overlappingFiles); mergeFiles.Add(fileToCompact); var tableEnumerators = new List <TableEnumerator>(); foreach (FileMetadata metadata in mergeFiles.OrderBy(f => f.SmallestKey, new BytewiseComparator())) { metadata.Table ??= _manifest.GetTable(metadata.FileNumber); metadata.Table.Initialize(); Log.Debug($"Add enumerator for {metadata.FileNumber}, idx len {metadata.Table._blockIndex.Length}"); tableEnumerators.Add((TableEnumerator)metadata.Table.GetEnumerator()); } var mergeEnumerator = new MergeEnumerator(tableEnumerators); var newFiles = new List <FileMetadata>(); int count = 0; while (true) { FileMetadata newFileMeta = WriteMergedTable(version, mergeEnumerator, ref count); if (newFileMeta.SmallestKey == null) { break; } version.SetCompactPointer(level, newFileMeta.LargestKey); newFiles.Add(newFileMeta); } foreach (FileMetadata originalFile in mergeFiles) { version.RemoveFile(overlappingFiles.Contains(originalFile) ? level + 1 : level, originalFile.FileNumber); } foreach (var fileMetadata in newFiles) { version.AddFile(level + 1, fileMetadata); } } } //break; } } if (manifestIsDirty) { var manifestFileName = new FileInfo(GetManifestFileName(_manifest.CurrentVersion.GetNewFileNumber())); using (var writer = new LogWriter(manifestFileName)) { _manifest.Save(writer); } using (StreamWriter currentStream = File.CreateText(GetCurrentFileName())) { currentStream.WriteLine(manifestFileName.Name); currentStream.Close(); } Manifest.Print(Log, version); CleanOldFiles(); } Log.Debug($"Done merge compact of level n to level .."); }
private void TryCompactLevel0() { Log.Debug($"Trying to do merge compact of level 0 to level 1.."); if (!_dbLock.IsWriteLockHeld) { Log.Error("Panic!"); throw new PanicException("Tried to compact without holding a write lock. Abort!"); } Version version = _manifest.CurrentVersion; List <FileMetadata> level0FilesToCompact = version.GetFiles(0); if (level0FilesToCompact.Count < (int)Options.MaxNumberOfLogFiles) { Log.Debug($"No level 0 to 1 compact needed at this time."); return; } byte[] smallestKey = null; byte[] largestKey = null; foreach (FileMetadata metadata in level0FilesToCompact.OrderBy(f => f.SmallestKey, new InternalKeyComparator())) { smallestKey ??= metadata.SmallestKey; largestKey = metadata.LargestKey; } Log.Debug($"Smallest Key: {smallestKey.ToHexString()}, Largest Key: {largestKey.ToHexString()}"); var filesToCompact = new List <FileMetadata>(level0FilesToCompact); // Add overlapping level 1 files List <FileMetadata> overlappingFiles = version.GetOverlappingFiles(1, smallestKey, largestKey); filesToCompact.AddRange(overlappingFiles); Log.Warn($"Overlapping files: {overlappingFiles.Count}"); var files = new List <TableEnumerator>(); foreach (FileMetadata metadata in filesToCompact.OrderBy(f => f.SmallestKey, new BytewiseComparator())) { metadata.Table ??= _manifest.GetTable(metadata.FileNumber); metadata.Table.Initialize(); Log.Debug($"Add enumerator for {metadata.FileNumber}, idx len {metadata.Table._blockIndex.Length}"); files.Add((TableEnumerator)metadata.Table.GetEnumerator()); } var mergeEnumerator = new MergeEnumerator(files); int count = 0; var newFiles = new List <FileMetadata>(); while (true) { FileMetadata newFileMeta = WriteMergedTable(version, mergeEnumerator, ref count); if (newFileMeta.SmallestKey == null) { break; } newFiles.Add(newFileMeta); } if (Log.IsDebugEnabled) { Log.Debug($"Compaction L0 * {level0FilesToCompact.Count} -> L1 * {newFiles.Count}"); foreach (FileMetadata file in level0FilesToCompact) { Log.Debug($"Compacted L0@{file.FileNumber}"); } foreach (FileMetadata file in overlappingFiles) { Log.Debug($"Compacted L1@{file.FileNumber}"); } foreach (FileMetadata newFile in newFiles) { Log.Debug($"Created L1@{newFile.FileNumber}"); } } var newVersion = new Version(version); newVersion.LogNumber = _logNumber; foreach (FileMetadata originalFile in filesToCompact) { newVersion.RemoveFile(overlappingFiles.Contains(originalFile) ? 1 : 0, originalFile.FileNumber); } foreach (var fileMetadata in newFiles) { newVersion.AddFile(1, fileMetadata); } Manifest.Print(Log, newVersion); // Update manifest with new version _manifest.CurrentVersion = newVersion; var manifestFileName = new FileInfo(GetManifestFileName(newVersion.GetNewFileNumber())); using (var writer = new LogWriter(manifestFileName)) { _manifest.Save(writer); } using (StreamWriter currentStream = File.CreateText(GetCurrentFileName())) { currentStream.WriteLine(manifestFileName.Name); currentStream.Close(); } CleanOldFiles(); Log.Debug($"Done with merge compact of level 0 to level 1.."); }
public InnerHandler(IAsyncEnumerator <TSource> source, MergeEnumerator parent) { _source = source; _parent = parent; }