public void CanOpenDbAfterDeletingAndCompacting()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 4096,
OpenUpToCommitUlong = null,
PreserveHistoryUpToCommitUlong = null,
CompactorScheduler = null,
};
using (var kvDb = new KeyValueDB(options))
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
tr.CreateOrUpdateKeyValue(new byte[5], new byte[3000]);
tr.CreateOrUpdateKeyValue(new byte[6], new byte[2000]);
tr.Commit();
}
kvDb.Compact(CancellationToken.None);
}
using (var kvDb = new KeyValueDB(options))
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
tr.FindFirstKey();
tr.EraseCurrent();
tr.Commit();
}
kvDb.Compact(CancellationToken.None);
}
using (var kvDb = new KeyValueDB(options))
{
// If there is error in KVI 3 it will create new KVI 4, but there is no problem in KVI 3
Assert.Null(kvDb.FileCollection.FileInfoByIdx(4));
}
}
}
public void CompactionWaitsForFinishingOldTransactionsBeforeRemovingFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024, null))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
Task.Run(() =>
{
db.Compact(new CancellationToken());
});
Thread.Sleep(2000);
Console.WriteLine(db.CalcStats());
Assert.True(4 <= fileCollection.GetCount()); // 2 Logs, 1 Value, 1 KeyIndex, (optinal 1 Unknown (old KeyIndex))
longTr.Dispose();
Thread.Sleep(1000);
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
Console.WriteLine(
"Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "realpath":
{
var res = PlatformMethods.Instance.RealPath(args[0]);
if (res == null)
{
Console.WriteLine("Error resolving real path for " + args[0]);
}
else
{
Console.WriteLine(res);
}
break;
}
case "nicedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "interactive":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo(true);
Interactive(iterator, visitor);
break;
}
case "comparedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToConsoleVisitorForComparison();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "comparesplitdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToFilesVisitorForComparison(HashType.Crc32);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "diskdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
using var tst = File.CreateText(Path.Combine(args[0], "dump.txt"));
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToFileVisitorNice(tst);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dumpnull":
case "null":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "check":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var transaction = kdb.StartReadOnlyTransaction();
var keyValueCount = transaction.GetKeyValueCount();
transaction.FindFirstKey();
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey();
}
break;
}
case "stat":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "statm": // Stat but by old managed implementation
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "kvi":
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "kvim": // Kvi but by old managed implementation
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "fileheaders":
{
using var dfc = new OnDiskFileCollection(args[0]);
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyIndex:
{
details =
$"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}";
var usedFiles = keyIndex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
break;
}
case IFileTransactionLog trlog:
details = $"Previous File Id: {trlog.PreviousFileId}";
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine(
$"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "export":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var tr = kdb.StartReadOnlyTransaction();
using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
KeyValueDBExportImporter.Export(tr, st);
break;
}
case "import":
{
using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc);
using var tr = kdb.StartTransaction();
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
break;
}
case "leaks":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
Console.WriteLine("Leaks: ");
odb.Open(kdb, false);
odb.DumpLeaks();
break;
}
case "leakscode":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
odb.DumpLeaksCode();
break;
}
case "frequency":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleFrequencyVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
visitor.OutputStatistic();
}
break;
case "size":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleSizeVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
break;
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
public void CanRoolback()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
Compression = new NoCompressionStrategy()
}))
{
for (var i = 1; i < 100; i++)
{
using (var tr = kv.StartTransaction())
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[200]);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[0]);
tr.SetCommitUlong((ulong)i);
tr.Commit();
}
if (i % 5 == 0)
{
kv.Compact(new System.Threading.CancellationToken());
}
if (i == 50)
{
kv.PreserveHistoryUpToCommitUlong = (ulong)i;
}
}
}
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
OpenUpToCommitUlong = 50,
PreserveHistoryUpToCommitUlong = 80,
Compression = new NoCompressionStrategy()
}))
{
using (var tr = kv.StartTransaction())
{
Assert.Equal(50, tr.GetKeyValueCount());
}
kv.Compact(new System.Threading.CancellationToken());
}
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
OpenUpToCommitUlong = 80,
PreserveHistoryUpToCommitUlong = 80,
Compression = new NoCompressionStrategy()
}))
{
using (var tr = kv.StartTransaction())
{
Assert.Equal(80, tr.GetKeyValueCount());
}
}
// Openning without long enough preserving in previous open, removed posibility to rollback before it
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
OpenUpToCommitUlong = 50,
PreserveHistoryUpToCommitUlong = 80,
Compression = new NoCompressionStrategy()
}))
{
using (var tr = kv.StartTransaction())
{
Assert.Equal(80, tr.GetKeyValueCount());
}
}
}
}
public void CompatorShouldNotBePesimistDespiteRunningTransactions()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
{
for (var i = 0; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
tr.Commit();
}
}
kvDb.Compact(new System.Threading.CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
using (var trOlder = kvDb.StartReadOnlyTransaction())
{
for (var i = 0; i < 50; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i * 2);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.Commit();
}
}
while (kvDb.Compact(new System.Threading.CancellationToken()))
{
;
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 2, fileCountAfterFirstCompaction + 50);
}
for (var i = 0; i < 4; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
tr.Commit();
}
}
while (kvDb.Compact(new System.Threading.CancellationToken()))
{
;
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
}
}
public void CompatorShouldNotBePesimist()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
OpenUpToCommitUlong = null,
PreserveHistoryUpToCommitUlong = null,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
{
for (var i = 0; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
tr.SetCommitUlong((ulong)i);
tr.Commit();
}
}
kvDb.PreserveHistoryUpToCommitUlong = 100;
kvDb.Compact(new System.Threading.CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
for (var i = 0; i < 50; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.SetCommitUlong(100 + (ulong)i);
tr.Commit();
}
}
kvDb.PreserveHistoryUpToCommitUlong = 150;
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 1, fileCountAfterFirstCompaction + 3);
using (var trOlder = kvDb.StartReadOnlyTransaction())
{
for (var i = 50; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.SetCommitUlong(100 + (ulong)i);
tr.Commit();
}
}
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
kvDb.PreserveHistoryUpToCommitUlong = 200;
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange <uint>(fileCollection.GetCount(), 1, 4);
}
}
}
public void ReopenAndReopenWithRoolbackDoesNotCorruptDB()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 4096,
Compression = new NoCompressionStrategy()
}))
{
for (var i = 1; i < 60; i++)
{
using (var tr = kv.StartTransaction())
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
tr.SetCommitUlong((ulong)i);
tr.Commit();
}
if (i % 2 == 0)
{
using (var tr = kv.StartTransaction())
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i - 1);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.Commit();
}
}
if (i % 5 == 0)
{
kv.Compact(new System.Threading.CancellationToken());
}
if (i == 50)
{
kv.PreserveHistoryUpToCommitUlong = (ulong)i;
}
}
}
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 4096,
PreserveHistoryUpToCommitUlong = 50,
Compression = new NoCompressionStrategy()
}))
{
kv.Compact(new System.Threading.CancellationToken());
ReadAllValues(kv);
}
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 4096,
PreserveHistoryUpToCommitUlong = 50,
OpenUpToCommitUlong = 50,
Compression = new NoCompressionStrategy()
}))
{
ReadAllValues(kv);
}
}
}
public void Run()
{
var collection = new InMemoryFileCollection();
using (var kvDb = new KeyValueDB(collection, new NoCompressionStrategy(), 100 * 1024 * 1024))
{
ulong itemsCount = 0;
using (var objDb = new ObjectDB())
{
objDb.Open(kvDb, false);
Console.WriteLine("started generating");
using (var tr = objDb.StartWritingTransaction().Result)
{
var objects = tr.Singleton <SmallObjects>();
while (true)
{
objects.Items.Add(itemsCount, new SmallObject()
{
Id = itemsCount, Label = "bu"
});
if (itemsCount % 1_000_000 == 0)
{
Console.WriteLine("Generated {0}", itemsCount);
}
if (itemsCount % 1000 == 0 && collection.GetCount() == 20)
{
break;
}
itemsCount++;
}
tr.Commit();
}
Console.WriteLine("finished generating");
using (var tr = objDb.StartWritingTransaction().Result)
{
var objects = tr.Singleton <SmallObjects>();
itemsCount = (ulong)objects.Items.Count;
tr.Commit();
}
Console.WriteLine("removing items started");
using (var tr = objDb.StartWritingTransaction().Result)
{
var objects = tr.Singleton <SmallObjects>();
for (ulong i = 0; i < itemsCount / 5; i++)
{
if (i % 2 == 0)
{
continue;
}
objects.Items.Remove(i);
}
tr.Commit();
}
Console.WriteLine("removing items finished");
var transactionCreationStarted = new ManualResetEventSlim(false);
var compactionFinished = new ManualResetEventSlim(false);
Task.Run(() =>
{
Console.WriteLine("Started waiting for transaction creating");
transactionCreationStarted.Wait();
Console.WriteLine("Started Compacting");
Trace.Assert(kvDb.Compact(CancellationToken.None));
Console.WriteLine("Finished Compacting");
compactionFinished.Set();
});
Console.WriteLine("Started concurrent transaction creation");
long msMax = 0;
long average = 0;
long iterations = 0;
Stopwatch watch = new Stopwatch();
while (true)
{
var compactionFinishedBeforeLasttransaction = compactionFinished.IsSet;
iterations++;
watch.Start();
var task = objDb.StartWritingTransaction();
if (!transactionCreationStarted.IsSet)
{
transactionCreationStarted.Set();
}
task.Wait();
var ms = watch.ElapsedMilliseconds;
average += ms;
msMax = Math.Max(ms, msMax);
watch.Reset();
using (var tr = task.Result)
{
tr.Commit();
}
if ((compactionFinishedBeforeLasttransaction && compactionFinished.IsSet))
{
break;
}
}
Console.WriteLine("Finished concurrent transaction creation, longest transaction create time was {0}ms, " +
"average {1}ms, iterations {2}", msMax, average / (double)iterations, iterations);
}
}
}
public void PreapprovedCommitAndCompaction()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartWritingTransaction().Result)
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
db.Compact(new CancellationToken());
using (var tr = db.StartWritingTransaction().Result)
{
tr.EraseRange(0, 0);
tr.Commit();
}
db.Compact(new CancellationToken());
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.False(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(Key2));
}
}
}
}
}
public void CompactionDoesNotRemoveStillUsedFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024, null))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
db.Compact(new CancellationToken());
Assert.Equal(3u, fileCollection.GetCount()); // 2 Logs, 1 KeyIndex
longTr.Dispose();
db.Compact(new CancellationToken());
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "diskdump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
using (var tst = File.CreateText(Path.Combine(args[0], "dump.txt")))
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToFileVisitorNice(tst);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "fileheaders":
{
using (var dfc = new OnDiskFileCollection(args[0]))
{
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyindex:
{
details =
$"KVCount:{keyindex.KeyValueCount} CommitUlong:{keyindex.CommitUlong} TrLogFileId:{keyindex.TrLogFileId} TrLogOffset:{keyindex.TrLogOffset}";
var usedFiles = keyindex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
break;
}
case IFileTransactionLog trlog:
details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine($"Opened in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine($"Final compaction in {sw.Elapsed.TotalSeconds:F1}");
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "snapshot.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
case "import":
{
using (var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat")))
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartTransaction())
{
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "dump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToStringVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
var text = visitor.ToString();
Console.WriteLine(text);
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
public void CompactionWaitsForFinishingOldTransactionsBeforeRemovingFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
db.Compact();
Thread.Sleep(2000);
Console.WriteLine(db.CalcStats());
Assert.True(4 <= fileCollection.GetCount()); // 2 Logs, 1 Value, 1 KeyIndex, (optinal 1 Unknown (old KeyIndex))
longTr.Dispose();
Thread.Sleep(1000);
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
Console.WriteLine(
"Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check, findsplitbrain, fulldiskdump, trldump");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "realpath":
{
var res = PlatformMethods.Instance.RealPath(args[0]);
if (res == null)
{
Console.WriteLine("Error resolving real path for " + args[0]);
}
else
{
Console.WriteLine(res);
}
break;
}
case "nicedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "interactive":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo(true);
Interactive(iterator, visitor);
break;
}
case "comparedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToConsoleVisitorForComparison();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "comparesplitdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
using var visitor = new ToFilesVisitorForComparison(HashType.Crc32);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "fulldiskdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
using var visitor = new ToFilesVisitorWithSecondaryKeys();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "diskdump":
{
var dbDir = args[0];
var openUpToCommitUlong = args.Length >= 3 ? (ulong?)ulong.Parse(args[2]) : null;
var fileName = Path.Combine(dbDir, "dump.txt");
DiskDump(dbDir, fileName, openUpToCommitUlong);
break;
}
case "dump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dumpnull":
case "null":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "check":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var transaction = kdb.StartReadOnlyTransaction();
var keyValueCount = transaction.GetKeyValueCount();
transaction.FindFirstKey(ReadOnlySpan <byte> .Empty);
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey(ReadOnlySpan <byte> .Empty);
}
break;
}
case "stat":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "statm": // Stat but by old managed implementation
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "kvi":
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "kvim": // Kvi but by old managed implementation
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "fileheaders":
{
using var dfc = new OnDiskFileCollection(args[0]);
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyIndex:
{
details =
$"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}\n";
details += LoadUsedFilesFromKvi(keyIndex, fcfi, fi.Key);
break;
}
case IFileTransactionLog trlog:
details = $"Previous File Id: {trlog.PreviousFileId}";
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine(
$"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "export":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var tr = kdb.StartReadOnlyTransaction();
using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
KeyValueDBExportImporter.Export(tr, st);
break;
}
case "import":
{
using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc);
using var tr = kdb.StartTransaction();
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
break;
}
case "leaks":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
Console.WriteLine("Leaks: ");
odb.Open(kdb, false);
odb.DumpLeaks();
break;
}
case "leakscode":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
odb.DumpLeaksCode();
break;
}
case "frequency":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleFrequencyVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
visitor.OutputStatistic();
}
break;
case "size":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleSizeVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
break;
case "findsplitbrain":
{
if (args.Length != 6)
{
Console.WriteLine(
"usage: ODBDump Eagle_0 findsplitbrain Eagle_1 1000 100000 INestedEventTable");
return;
}
var startEvent = ulong.Parse(args[3]);
var endEvent = ulong.Parse(args[4]);
var relationName = args[5];
var(found, lastGood, firstBad) =
FindSplitBrain(args[0], args[2], relationName, startEvent, endEvent);
if (found)
{
Console.WriteLine($"Split occured between {lastGood} and {firstBad}");
DiskDump(args[0], $"dump_{lastGood}_0.txt", lastGood);
DiskDump(args[2], $"dump_{lastGood}_1.txt", lastGood);
DiskDump(args[0], $"dump_{firstBad}_0.txt", firstBad);
DiskDump(args[2], $"dump_{firstBad}_1.txt", firstBad);
}
}
break;
case "trldump":
{
ITrlVisitor visitor = new ConsoleTrlVisitor();
using var dfc = new OnDiskFileCollection(args[0]);
foreach (var file in dfc.Enumerate())
{
try
{
visitor.StartFile(file.Index, file.GetSize());
var reader = new TrlFileReader(file);
reader.Iterate(visitor);
}
catch (Exception e)
{
visitor.OperationDetail("Failure " + e.Message);
}
visitor.EndOperation();
}
}
break;
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "fileheaders":
{
using (var dfc = new OnDiskFileCollection(args[0]))
{
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
var keyindex = fi.Value as IKeyIndex;
if (keyindex != null)
{
details = string.Format("KVCount:{0} CommitUlong:{1} TrLogFileId:{2} TrLogOffset:{3}", keyindex.KeyValueCount, keyindex.CommitUlong, keyindex.TrLogFileId, keyindex.TrLogOffset);
var usedFiles = keyindex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
}
var trlog = fi.Value as IFileTransactionLog;
if (trlog != null)
{
details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
