public void AddingContinueToNewFileAfterReopenWithCorruption()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
fileCollection.SimulateCorruptionBySetSize(20 + 16);
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.True(2 <= fileCollection.GetCount());
}
}
public void FastCleanUpOnStartRemovesUselessFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[1024]);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.EraseAll();
tr.Commit();
}
Assert.Equal(3u, fileCollection.GetCount()); // 3 Logs
}
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
Console.WriteLine(db.CalcStats());
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
}
}
}
public void AddingContinueToNewFileAfterReopenWithCorruption()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
fileCollection.SimulateCorruptionBySetSize(20 + 16);
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.True(2 <= fileCollection.GetCount());
}
}
public void CompactionWaitsForFinishingOldTransactionsBeforeRemovingFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024, null))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
Task.Run(() =>
{
db.Compact(new CancellationToken());
});
Thread.Sleep(2000);
Console.WriteLine(db.CalcStats());
Assert.True(4 <= fileCollection.GetCount()); // 2 Logs, 1 Value, 1 KeyIndex, (optinal 1 Unknown (old KeyIndex))
longTr.Dispose();
Thread.Sleep(1000);
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
public void AddingContinueToSameFileAfterReopen()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.Equal(2u, fileCollection.GetCount()); // Log + Index
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
Console.WriteLine(
"Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "realpath":
{
var res = PlatformMethods.Instance.RealPath(args[0]);
if (res == null)
{
Console.WriteLine("Error resolving real path for " + args[0]);
}
else
{
Console.WriteLine(res);
}
break;
}
case "nicedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "interactive":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo(true);
Interactive(iterator, visitor);
break;
}
case "comparedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToConsoleVisitorForComparison();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "comparesplitdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToFilesVisitorForComparison(HashType.Crc32);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "diskdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
using var tst = File.CreateText(Path.Combine(args[0], "dump.txt"));
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToFileVisitorNice(tst);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dumpnull":
case "null":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "check":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var transaction = kdb.StartReadOnlyTransaction();
var keyValueCount = transaction.GetKeyValueCount();
transaction.FindFirstKey();
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey();
}
break;
}
case "stat":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "statm": // Stat but by old managed implementation
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "kvi":
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "kvim": // Kvi but by old managed implementation
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "fileheaders":
{
using var dfc = new OnDiskFileCollection(args[0]);
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyIndex:
{
details =
$"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}";
var usedFiles = keyIndex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
break;
}
case IFileTransactionLog trlog:
details = $"Previous File Id: {trlog.PreviousFileId}";
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine(
$"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "export":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var tr = kdb.StartReadOnlyTransaction();
using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
KeyValueDBExportImporter.Export(tr, st);
break;
}
case "import":
{
using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc);
using var tr = kdb.StartTransaction();
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
break;
}
case "leaks":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
Console.WriteLine("Leaks: ");
odb.Open(kdb, false);
odb.DumpLeaks();
break;
}
case "leakscode":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
odb.DumpLeaksCode();
break;
}
case "frequency":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleFrequencyVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
visitor.OutputStatistic();
}
break;
case "size":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleSizeVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
break;
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
public void HugeTest()
{
const int keyCount = 100;
using (var fileCollection = CreateTestFileCollection())
{
_sw.Start();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
var key = new byte[100];
var value = new byte[100000000];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
value[100] = (byte)(i / 100);
value[200] = (byte)(i % 100);
tr.CreateOrUpdateKeyValue(key, value);
tr.Commit();
}
}
}
_sw.Stop();
Console.WriteLine("Time to create 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100))
{
throw new InvalidDataException();
}
if (value[200] != (byte)(i % 100))
{
throw new InvalidDataException();
}
}
}
_sw.Stop();
Console.WriteLine("Time to read all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
_sw.Restart();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open2 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100))
{
throw new InvalidDataException();
}
if (value[200] != (byte)(i % 100))
{
throw new InvalidDataException();
}
}
}
_sw.Stop();
Console.WriteLine("Time to read2 all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
}
}
public void AddingContinueToSameFileAfterReopen()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.Equal(2u, fileCollection.GetCount()); // Log + Index
}
}
public void FastCleanUpOnStartRemovesUselessFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[1024]);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.EraseAll();
tr.Commit();
}
Assert.Equal(3u, fileCollection.GetCount()); // 3 Logs
}
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
Console.WriteLine(db.CalcStats());
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "diskdump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
using (var tst = File.CreateText(Path.Combine(args[0], "dump.txt")))
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToFileVisitorNice(tst);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "fileheaders":
{
using (var dfc = new OnDiskFileCollection(args[0]))
{
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyindex:
{
details =
$"KVCount:{keyindex.KeyValueCount} CommitUlong:{keyindex.CommitUlong} TrLogFileId:{keyindex.TrLogFileId} TrLogOffset:{keyindex.TrLogOffset}";
var usedFiles = keyindex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
break;
}
case IFileTransactionLog trlog:
details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine($"Opened in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine($"Final compaction in {sw.Elapsed.TotalSeconds:F1}");
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "snapshot.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
case "import":
{
using (var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat")))
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartTransaction())
{
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "dump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToStringVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
var text = visitor.ToString();
Console.WriteLine(text);
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
public void CompactionWaitsForFinishingOldTransactionsBeforeRemovingFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
db.Compact();
Thread.Sleep(2000);
Console.WriteLine(db.CalcStats());
Assert.True(4 <= fileCollection.GetCount()); // 2 Logs, 1 Value, 1 KeyIndex, (optinal 1 Unknown (old KeyIndex))
longTr.Dispose();
Thread.Sleep(1000);
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
Console.WriteLine(
"Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check, findsplitbrain, fulldiskdump, trldump");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "realpath":
{
var res = PlatformMethods.Instance.RealPath(args[0]);
if (res == null)
{
Console.WriteLine("Error resolving real path for " + args[0]);
}
else
{
Console.WriteLine(res);
}
break;
}
case "nicedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "interactive":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo(true);
Interactive(iterator, visitor);
break;
}
case "comparedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToConsoleVisitorForComparison();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "comparesplitdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
using var visitor = new ToFilesVisitorForComparison(HashType.Crc32);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "fulldiskdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
using var visitor = new ToFilesVisitorWithSecondaryKeys();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "diskdump":
{
var dbDir = args[0];
var openUpToCommitUlong = args.Length >= 3 ? (ulong?)ulong.Parse(args[2]) : null;
var fileName = Path.Combine(dbDir, "dump.txt");
DiskDump(dbDir, fileName, openUpToCommitUlong);
break;
}
case "dump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dumpnull":
case "null":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "check":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var transaction = kdb.StartReadOnlyTransaction();
var keyValueCount = transaction.GetKeyValueCount();
transaction.FindFirstKey(ReadOnlySpan <byte> .Empty);
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey(ReadOnlySpan <byte> .Empty);
}
break;
}
case "stat":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "statm": // Stat but by old managed implementation
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "kvi":
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "kvim": // Kvi but by old managed implementation
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "fileheaders":
{
using var dfc = new OnDiskFileCollection(args[0]);
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyIndex:
{
details =
$"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}\n";
details += LoadUsedFilesFromKvi(keyIndex, fcfi, fi.Key);
break;
}
case IFileTransactionLog trlog:
details = $"Previous File Id: {trlog.PreviousFileId}";
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine(
$"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "export":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var tr = kdb.StartReadOnlyTransaction();
using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
KeyValueDBExportImporter.Export(tr, st);
break;
}
case "import":
{
using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc);
using var tr = kdb.StartTransaction();
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
break;
}
case "leaks":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
Console.WriteLine("Leaks: ");
odb.Open(kdb, false);
odb.DumpLeaks();
break;
}
case "leakscode":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
odb.DumpLeaksCode();
break;
}
case "frequency":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleFrequencyVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
visitor.OutputStatistic();
}
break;
case "size":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleSizeVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
break;
case "findsplitbrain":
{
if (args.Length != 6)
{
Console.WriteLine(
"usage: ODBDump Eagle_0 findsplitbrain Eagle_1 1000 100000 INestedEventTable");
return;
}
var startEvent = ulong.Parse(args[3]);
var endEvent = ulong.Parse(args[4]);
var relationName = args[5];
var(found, lastGood, firstBad) =
FindSplitBrain(args[0], args[2], relationName, startEvent, endEvent);
if (found)
{
Console.WriteLine($"Split occured between {lastGood} and {firstBad}");
DiskDump(args[0], $"dump_{lastGood}_0.txt", lastGood);
DiskDump(args[2], $"dump_{lastGood}_1.txt", lastGood);
DiskDump(args[0], $"dump_{firstBad}_0.txt", firstBad);
DiskDump(args[2], $"dump_{firstBad}_1.txt", firstBad);
}
}
break;
case "trldump":
{
ITrlVisitor visitor = new ConsoleTrlVisitor();
using var dfc = new OnDiskFileCollection(args[0]);
foreach (var file in dfc.Enumerate())
{
try
{
visitor.StartFile(file.Index, file.GetSize());
var reader = new TrlFileReader(file);
reader.Iterate(visitor);
}
catch (Exception e)
{
visitor.OperationDetail("Failure " + e.Message);
}
visitor.EndOperation();
}
}
break;
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "fileheaders":
{
using (var dfc = new OnDiskFileCollection(args[0]))
{
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
var keyindex = fi.Value as IKeyIndex;
if (keyindex != null)
{
details = string.Format("KVCount:{0} CommitUlong:{1} TrLogFileId:{2} TrLogOffset:{3}", keyindex.KeyValueCount, keyindex.CommitUlong, keyindex.TrLogFileId, keyindex.TrLogOffset);
var usedFiles = keyindex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
}
var trlog = fi.Value as IFileTransactionLog;
if (trlog != null)
{
details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
public void HugeTest()
{
const int keyCount = 100;
using (var fileCollection = CreateTestFileCollection())
{
_sw.Start();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
var key = new byte[100];
var value = new byte[100000000];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
value[100] = (byte)(i / 100);
value[200] = (byte)(i % 100);
tr.CreateOrUpdateKeyValue(key, value);
tr.Commit();
}
}
}
_sw.Stop();
Console.WriteLine("Time to create 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100)) throw new InvalidDataException();
if (value[200] != (byte)(i % 100)) throw new InvalidDataException();
}
}
_sw.Stop();
Console.WriteLine("Time to read all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
_sw.Restart();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open2 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100)) throw new InvalidDataException();
if (value[200] != (byte)(i % 100)) throw new InvalidDataException();
}
}
_sw.Stop();
Console.WriteLine("Time to read2 all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
}
}
