static void Main(string[] args)
{
// prueba 1
int[] temp1 = { 1, 3, 4, 7, 9, 13, 15, 17 };
int[] temp2 = { 3, 6, 9, 12, 45, 78 };
int[] temp3 = { 4, 6, 8, 13, 46, 57 };
//foreach (var el in Merge.MergeEnumerators<int>(new ComparaEnteros(), ((IEnumerable<int>)temp1).GetEnumerator(), ((IEnumerable<int>)temp2).GetEnumerator(),
// ((IEnumerable<int>)temp3).GetEnumerator()))
// Console.WriteLine(el);
IEnumerator <int> my = new MergeEnumerator <int>(new ComparaEnteros(), ((IEnumerable <int>)temp1).GetEnumerator(), ((IEnumerable <int>)temp2).GetEnumerator(),
((IEnumerable <int>)temp3).GetEnumerator());
while (my.MoveNext())
{
Console.WriteLine(my.Current);
}
my.Reset();
Console.WriteLine();
while (my.MoveNext())
{
Console.WriteLine(my.Current);
}
}
public void TestMergeIterator()
{
Random r = new Random();
int totalElements = 0;
// Create 10 randomly sized lists of random numbers
List <IEnumerable <int> > collections = new List <IEnumerable <int> >();
for (int i = 0; i < 10; i++)
{
List <int> randomData = new List <int>();
for (int j = 0; j < r.Next(100); j++)
{
randomData.Add(r.Next());
totalElements++;
}
collections.Add(randomData);
}
// Sort all the individual lists
var sortedCollections = collections.Select(list => list.OrderBy(i => i).AsEnumerable());
// Now scan through the merged list and make sure the result is ordered
int lastNum = int.MinValue;
int numElements = 0;
foreach (var num in MergeEnumerator.Merge(sortedCollections))
{
Assert.LessOrEqual(lastNum, num);
lastNum = num;
numElements++;
}
Assert.AreEqual(totalElements, numElements);
}
public void TestEmptyMergeIterator()
{
var enumerators = new List <IEnumerable <int> >();
Assert.AreEqual(0, MergeEnumerator.Merge(enumerators).Count());
enumerators.Add(new List <int>().OrderBy(e => e));
enumerators.Add(new List <int>().OrderBy(e => e));
enumerators.Add(new List <int>().OrderBy(e => e));
Assert.AreEqual(0, MergeEnumerator.Merge(enumerators).Count());
}
public void LevelMergeReadTest()
{
string path = Path.GetFullPath("TestData\\LevelMergeReadTest");
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
int num_tables_to_merge = 4;
int items_per_table = 2500;
int totalData = 0;
for (int i = 0; i < num_tables_to_merge; i++)
{
var mt = new MemTable();
for (int j = 0; j < items_per_table; j++)
{
var randKey = Key.Random(40);
var randVal = Value.Random(512);
mt.Add(randKey, randVal);
}
mt.WriteToSortedBlockTable("TestData\\LevelMergeReadTest", 0, i);
totalData += mt.Size;
}
var tables = new List <IEnumerable <KeyValuePair <Key, Value> > >();
var sbts = new List <SortedBlockTable>();
var cache = new RazorCache();
for (int j = 0; j < num_tables_to_merge; j++)
{
var sbt = new SortedBlockTable(cache, "TestData\\LevelMergeReadTest", 0, j);
tables.Add(sbt.Enumerate());
sbts.Add(sbt);
}
int ct = 0;
Key key = Key.FromBytes(new byte[] { 0, 0 });
var timer = new Stopwatch();
timer.Start();
foreach (var pair in MergeEnumerator.Merge(tables, p => p.Key))
{
Assert.True(key.CompareTo(pair.Key) < 0);
key = pair.Key;
ct++;
}
timer.Stop();
sbts.ForEach(s => s.Close());
Console.WriteLine("Scanned through a multilevel merge at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
}
private FileMetadata WriteMergedTable(Version version, MergeEnumerator mergeEnumerator, ref int count)
{
var newFileMeta = new FileMetadata {
FileNumber = version.GetNewFileNumber()
};
var newFileInfo = new FileInfo(GetTableFileName(newFileMeta.FileNumber));
using FileStream newFileStream = newFileInfo.Create();
var creator = new TableCreator(newFileStream);
byte[] smallestKey = null;
byte[] largestKey = null;
ReadOnlyMemory <byte> prevKey = null;
while (mergeEnumerator.MoveNext())
{
BlockEntry entry = mergeEnumerator.Current;
count++;
ReadOnlyMemory <byte> key = entry.Key;
if (prevKey.Length != 0)
{
if (prevKey.Span.UserKey().SequenceEqual(key.Span.UserKey()))
{
Log.Warn($"Duplicate keys - Prev Key: {prevKey.ToHexString()}, Key: {key.ToHexString()}");
continue;
}
}
prevKey = key;
creator.Add(key.Span, entry.Data.Span);
smallestKey ??= key.ToArray();
largestKey = key.ToArray();
if (creator.CurrentSize > Options.MaxTableFileSize)
{
break;
}
}
creator.Finish();
newFileMeta.SmallestKey = smallestKey;
newFileMeta.LargestKey = largestKey;
newFileInfo.Refresh();
newFileMeta.FileSize = (ulong)newFileInfo.Length;
return(newFileMeta);
}
public void TestDuplicateMergeIterator()
{
var enumerators = new List <IEnumerable <int> >();
enumerators.Add(Enumerable.Range(1, 10));
Assert.AreEqual(55, MergeEnumerator.Merge(enumerators).Sum());
enumerators = new List <IEnumerable <int> >();
enumerators.Add(Enumerable.Range(1, 10));
enumerators.Add(Enumerable.Range(2, 9));
enumerators.Add(Enumerable.Range(1, 10));
Assert.AreEqual(55, MergeEnumerator.Merge(enumerators).Sum());
enumerators = new List <IEnumerable <int> >();
enumerators.Add(new List <int>());
enumerators.Add(Enumerable.Range(2, 9));
enumerators.Add(Enumerable.Range(1, 10));
Assert.AreEqual(55, MergeEnumerator.Merge(enumerators).Sum());
}
public InnerHandler(IAsyncEnumerator <TSource> source, MergeEnumerator parent, CancellationTokenSource cts)
{
_source = source;
_parent = parent;
_cts = cts;
}
private static IEnumerable <SegmentationInput.Segment> SplitOverlappingSegments(List <SegmentationInput.Segment[]> sampleSegments)
{
var starts = MergeEnumerator.Merge(sampleSegments.Select(segments => segments.Select(segment => segment.start)));
var ends = MergeEnumerator.Merge(sampleSegments.Select(segments => segments.Select(segment => segment.end)));
var partitions = MergeEnumerator.Merge(new[]
public void CompactNumeric()
{
var keys = new List <byte[]>();
DirectoryInfo dir = TestUtils.GetTestDirectory(false);
// Setup new database and generate values enough to create 2 level 0 tables with overlapping keys.
// We use this when we run the real test.
ulong idx = 0;
var options = new Options()
{
LevelSizeBaseFactor = 10,
RetainAllFiles = true
};
List <FileMetadata> level0Files;
Version version = null;
using (var db = new Database(dir, true, options))
{
db.Open();
for (int j = 0; j < 4; j++)
{
for (int i = 0; i < 8000; i++)
{
byte[] key = BitConverter.GetBytes(idx++);
byte[] data = TestUtils.FillArrayWithRandomBytes(1000, 128);
db.Put(key, data);
keys.Add(key);
}
}
level0Files = new List <FileMetadata>(db.Level0Tables);
version = db.Version;
db.Close();
}
((Hierarchy)LogManager.GetRepository(Assembly.GetEntryAssembly())).Root.Level = Level.Warn;
{
Log.Warn($"Reading {keys.Count} values using regular db.get()");
using (var db = new Database(dir, false, options))
{
db.Open();
ulong count = 0;
ulong countMissing = 0;
foreach (byte[] key in keys)
{
byte[] value = db.Get(key);
if (value == null)
{
Log.Error($"Missing key {key.ToHexString()} at idx:{count}, {countMissing++}");
}
count++;
}
db.Close();
}
}
return;
//{
// Log.Warn($"Reading {keys.Count} values, from log files");
// List<byte[]> keysToRemove = new List<byte[]>(keys);
// FileInfo[] logFiles = dir.GetFiles("*.log");
// foreach (FileInfo fileInfo in logFiles)
// {
// Log.Warn($"Reading from {fileInfo.Name}. Have {keysToRemove.Count} keys left");
// using var reader = new LogReader(fileInfo.Open(FileMode.Open));
// var cache = new MemCache();
// cache.Load(reader);
// foreach (byte[] key in keysToRemove.Take(5000).ToArray())
// {
// if (cache.Get(key).State == ResultState.Exist)
// {
// keysToRemove.Remove(key);
// }
// }
// }
// Assert.AreEqual(0, keysToRemove.Count);
//}
int keysInLevel0 = 0;
var keysInCurrentLog = new List <byte[]>();
{
Log.Warn($"Reading {keys.Count} values, from level0 files");
List <byte[]> keysToRemove = new List <byte[]>(keys);
var enumerators = new List <TableEnumerator>();
foreach (FileMetadata fileMeta in level0Files.OrderBy(f => f.FileNumber))
{
string filePath = Path.Combine(dir.FullName, $"{fileMeta.FileNumber:000000}.ldb");
var fileInfo = new FileInfo(filePath);
Log.Warn($"Reading from {fileInfo.Name}. Have {keysToRemove.Count} keys left");
var table = new Table(fileInfo);
foreach (byte[] key in keysToRemove.ToArray())
{
if (table.Get(key).State == ResultState.Exist)
{
keysInLevel0++;
keysToRemove.Remove(key);
}
}
enumerators.Add((TableEnumerator)table.GetEnumerator());
}
Assert.Less(0, keysInLevel0);
// Read the remaining from current log file
{
string filePath = Path.Combine(dir.FullName, $"{version.LogNumber:000000}.log");
var fileInfo = new FileInfo(filePath);
Log.Warn($"Reading remaining {keysToRemove.Count} values from current log {fileInfo.Name}");
using var reader = new LogReader(fileInfo.Open(FileMode.Open));
var cache = new MemCache();
cache.Load(reader);
foreach (byte[] key in keysToRemove.ToArray())
{
if (cache.Get(key).State == ResultState.Exist)
{
keysInCurrentLog.Add(key);
keysToRemove.Remove(key);
}
}
Assert.AreEqual(0, keysToRemove.Count);
}
{
Log.Warn($"Reading {keysInLevel0} values, based on merge enumerator of all level0 table files");
var enumerator = new MergeEnumerator(enumerators);
int enumCount = 0;
while (enumerator.MoveNext())
{
enumCount++;
}
Assert.AreEqual(keysInLevel0, enumCount);
// Close the tables
foreach (TableEnumerator tableEnumerator in enumerators)
{
tableEnumerator.TEST_Close();
}
}
}
{
var keysLeftToRemove = new List <byte[]>(keys).Except(keysInCurrentLog).ToList();
Log.Warn($"Reading {keysLeftToRemove.Count} values, from all level+1 files + current level0");
var level1Enumerators = new List <TableEnumerator>();
FileInfo[] tableFiles = dir.GetFiles("*.ldb");
foreach (var fileInfo in tableFiles.OrderBy(f => f.Name))
{
if (level0Files.Any(f => $"{f.FileNumber:000000}.ldb" == fileInfo.Name))
{
if (version.GetFiles(0).All(f => $"{f.FileNumber:000000}.ldb" != fileInfo.Name))
{
continue;
}
Log.Warn($"Reading current level0 file {fileInfo.Name}");
}
Log.Warn($"Reading from {fileInfo.Name}. Have {keysLeftToRemove.Count} keys left");
var table = new Table(fileInfo);
table.Initialize();
level1Enumerators.Add((TableEnumerator)table.GetEnumerator());
foreach (byte[] key in keysLeftToRemove.ToArray())
{
if (table.Get(key).State == ResultState.Exist)
{
keysLeftToRemove.Remove(key);
}
}
}
//Assert.AreEqual(0, keysLeftToRemove.Count); // FAIL
{
keysLeftToRemove = new List <byte[]>(keys).Except(keysInCurrentLog).ToList();
Log.Warn($"Reading {keysLeftToRemove.Count} values, from all level+1 files + current level0 using merge enumerator");
var enumerator = new MergeEnumerator(level1Enumerators);
int enumCount = 0;
while (enumerator.MoveNext())
{
enumCount++;
if (enumerator.Current != null)
{
byte[] key = enumerator.Current.Key.Span.UserKey().ToArray();
keysLeftToRemove.RemoveAll(bytes => new BytewiseComparator().Compare(bytes, key) == 0);
}
else
{
Log.Warn($"Current in enumerator is null");
}
}
Assert.AreEqual(keys.Count - keysInCurrentLog.Count, enumCount, "Expected to have count of all keys");
Assert.AreEqual(0, keysLeftToRemove.Count, "Expected to have found all keys");
foreach (TableEnumerator tableEnumerator in level1Enumerators)
{
tableEnumerator.TEST_Close();
}
}
}
Log.Warn($"Done!");
}
private void TryCompactLevelN()
{
Log.Debug($"Trying to do merge compact of level n to level ..");
if (!_dbLock.IsWriteLockHeld)
{
Log.Error("Panic!");
throw new PanicException("Tried to compact without holding a write lock. Abort!");
}
Version version = _manifest.CurrentVersion;
bool manifestIsDirty = false;
foreach (KeyValuePair <int, List <FileMetadata> > kvp in version.Levels.Where(kvp => kvp.Key > 0).ToArray())
{
int level = kvp.Key;
List <FileMetadata> files = kvp.Value;
if (files.Count > Math.Pow(Options.LevelSizeBaseFactor, level))
{
manifestIsDirty = true;
Log.Warn($"Want to compact level {level} with {files.Count} files.");
while (true)
{
FileMetadata fileToCompact = GetNextFileToCompactForLevel(files, version.GetCompactPointer(level));
if (fileToCompact == null)
{
break; // Nah, shouldn't happen, right.
}
if (files.Count <= Math.Pow(Options.LevelSizeBaseFactor, level))
{
break;
}
List <FileMetadata> overlappingFiles = version.GetOverlappingFiles(level + 1, fileToCompact.SmallestKey, fileToCompact.LargestKey);
if (overlappingFiles.Count == 0)
{
// Level+1 has no files overlapping. So we can basically just move this file up the level.
Log.Warn($"Want to compact level {level}, file {fileToCompact.FileNumber} and level {level + 1} had No overlapping files. Simple move.");
version.RemoveFile(level, fileToCompact.FileNumber);
version.AddFile(level + 1, fileToCompact);
version.SetCompactPointer(level, fileToCompact.LargestKey);
}
else
{
Log.Warn($"Want to compact level {level}, file {fileToCompact.FileNumber} and level {level + 1} had {overlappingFiles.Count} overlapping files");
var mergeFiles = new List <FileMetadata>(overlappingFiles);
mergeFiles.Add(fileToCompact);
var tableEnumerators = new List <TableEnumerator>();
foreach (FileMetadata metadata in mergeFiles.OrderBy(f => f.SmallestKey, new BytewiseComparator()))
{
metadata.Table ??= _manifest.GetTable(metadata.FileNumber);
metadata.Table.Initialize();
Log.Debug($"Add enumerator for {metadata.FileNumber}, idx len {metadata.Table._blockIndex.Length}");
tableEnumerators.Add((TableEnumerator)metadata.Table.GetEnumerator());
}
var mergeEnumerator = new MergeEnumerator(tableEnumerators);
var newFiles = new List <FileMetadata>();
int count = 0;
while (true)
{
FileMetadata newFileMeta = WriteMergedTable(version, mergeEnumerator, ref count);
if (newFileMeta.SmallestKey == null)
{
break;
}
version.SetCompactPointer(level, newFileMeta.LargestKey);
newFiles.Add(newFileMeta);
}
foreach (FileMetadata originalFile in mergeFiles)
{
version.RemoveFile(overlappingFiles.Contains(originalFile) ? level + 1 : level, originalFile.FileNumber);
}
foreach (var fileMetadata in newFiles)
{
version.AddFile(level + 1, fileMetadata);
}
}
}
//break;
}
}
if (manifestIsDirty)
{
var manifestFileName = new FileInfo(GetManifestFileName(_manifest.CurrentVersion.GetNewFileNumber()));
using (var writer = new LogWriter(manifestFileName))
{
_manifest.Save(writer);
}
using (StreamWriter currentStream = File.CreateText(GetCurrentFileName()))
{
currentStream.WriteLine(manifestFileName.Name);
currentStream.Close();
}
Manifest.Print(Log, version);
CleanOldFiles();
}
Log.Debug($"Done merge compact of level n to level ..");
}
private void TryCompactLevel0()
{
Log.Debug($"Trying to do merge compact of level 0 to level 1..");
if (!_dbLock.IsWriteLockHeld)
{
Log.Error("Panic!");
throw new PanicException("Tried to compact without holding a write lock. Abort!");
}
Version version = _manifest.CurrentVersion;
List <FileMetadata> level0FilesToCompact = version.GetFiles(0);
if (level0FilesToCompact.Count < (int)Options.MaxNumberOfLogFiles)
{
Log.Debug($"No level 0 to 1 compact needed at this time.");
return;
}
byte[] smallestKey = null;
byte[] largestKey = null;
foreach (FileMetadata metadata in level0FilesToCompact.OrderBy(f => f.SmallestKey, new InternalKeyComparator()))
{
smallestKey ??= metadata.SmallestKey;
largestKey = metadata.LargestKey;
}
Log.Debug($"Smallest Key: {smallestKey.ToHexString()}, Largest Key: {largestKey.ToHexString()}");
var filesToCompact = new List <FileMetadata>(level0FilesToCompact);
// Add overlapping level 1 files
List <FileMetadata> overlappingFiles = version.GetOverlappingFiles(1, smallestKey, largestKey);
filesToCompact.AddRange(overlappingFiles);
Log.Warn($"Overlapping files: {overlappingFiles.Count}");
var files = new List <TableEnumerator>();
foreach (FileMetadata metadata in filesToCompact.OrderBy(f => f.SmallestKey, new BytewiseComparator()))
{
metadata.Table ??= _manifest.GetTable(metadata.FileNumber);
metadata.Table.Initialize();
Log.Debug($"Add enumerator for {metadata.FileNumber}, idx len {metadata.Table._blockIndex.Length}");
files.Add((TableEnumerator)metadata.Table.GetEnumerator());
}
var mergeEnumerator = new MergeEnumerator(files);
int count = 0;
var newFiles = new List <FileMetadata>();
while (true)
{
FileMetadata newFileMeta = WriteMergedTable(version, mergeEnumerator, ref count);
if (newFileMeta.SmallestKey == null)
{
break;
}
newFiles.Add(newFileMeta);
}
if (Log.IsDebugEnabled)
{
Log.Debug($"Compaction L0 * {level0FilesToCompact.Count} -> L1 * {newFiles.Count}");
foreach (FileMetadata file in level0FilesToCompact)
{
Log.Debug($"Compacted L0@{file.FileNumber}");
}
foreach (FileMetadata file in overlappingFiles)
{
Log.Debug($"Compacted L1@{file.FileNumber}");
}
foreach (FileMetadata newFile in newFiles)
{
Log.Debug($"Created L1@{newFile.FileNumber}");
}
}
var newVersion = new Version(version);
newVersion.LogNumber = _logNumber;
foreach (FileMetadata originalFile in filesToCompact)
{
newVersion.RemoveFile(overlappingFiles.Contains(originalFile) ? 1 : 0, originalFile.FileNumber);
}
foreach (var fileMetadata in newFiles)
{
newVersion.AddFile(1, fileMetadata);
}
Manifest.Print(Log, newVersion);
// Update manifest with new version
_manifest.CurrentVersion = newVersion;
var manifestFileName = new FileInfo(GetManifestFileName(newVersion.GetNewFileNumber()));
using (var writer = new LogWriter(manifestFileName))
{
_manifest.Save(writer);
}
using (StreamWriter currentStream = File.CreateText(GetCurrentFileName()))
{
currentStream.WriteLine(manifestFileName.Name);
currentStream.Close();
}
CleanOldFiles();
Log.Debug($"Done with merge compact of level 0 to level 1..");
}
public InnerHandler(IAsyncEnumerator <TSource> source, MergeEnumerator parent)
{
_source = source;
_parent = parent;
}
