public static void DumpLeaks(this IObjectDB db)
{
using (var tr = db.StartReadOnlyTransaction())
using (var visitor = new FindUnusedKeysVisitor())
{
visitor.ImportAllKeys(tr);
var iterator = visitor.Iterate(tr);
visitor.DumpUnseenKeys();
var leakedObjects = new List <ulong>();
foreach (var unseenKey in visitor.UnseenKeys())
{
if (unseenKey.Key[0] == 1)
{
try
{
var r = new ByteArrayReader(unseenKey.Key);
r.SkipUInt8();
var oid = r.ReadVUInt64();
leakedObjects.Add(oid);
}
catch (Exception ex)
{
Console.WriteLine($"Leak found but error has occured while reading: {ex.Message}");
}
}
}
if (leakedObjects.Count > 0)
{
Console.WriteLine("--- OBJECTS ---");
var consoleVisitor = new ToConsoleVisitorNice();
foreach (var oid in leakedObjects)
{
iterator.IterateUnseenOid(oid, consoleVisitor);
Console.WriteLine("------");
}
}
}
}
static void Interactive(ODBIterator iterator, ToConsoleVisitorNice visitor)
{
Console.WriteLine("Enter command:");
var currentRelationId = -1;
while (true)
{
var line = Console.ReadLine();
var words = line.Split(' ');
switch (words[0])
{
default:
Console.WriteLine("Unknown command " + words[0]);
goto case "help";
case "":
case "h":
case "help":
Console.WriteLine("Commands help:");
Console.WriteLine("l list");
Console.WriteLine("e exit");
continue;
case "s":
case "select":
if (words.Length == 1)
{
Console.WriteLine("Select command help:");
Console.WriteLine("select relation id");
continue;
}
if (long.TryParse(words[1], out var selectId))
{
if (currentRelationId >= 0)
{
iterator.IterateRelationRow(iterator.RelationId2Info[(uint)currentRelationId],
selectId);
}
break;
}
switch (words[1])
{
case "r":
case "relation":
if (words.Length == 3)
{
if (uint.TryParse(words[2], out var id) && iterator.RelationId2Info.ContainsKey(id))
{
currentRelationId = (int)id;
}
break;
}
break;
}
break;
case "l":
case "list":
if (words.Length == 1)
{
Console.WriteLine("List command help:");
Console.WriteLine("list relation");
Console.WriteLine("list nonempty relations");
continue;
}
switch (words[1])
{
case "r":
case "relation":
if (words.Length == 2)
{
foreach (var rel in iterator.RelationId2Info.Values.OrderBy(r => r.Id))
{
Console.WriteLine(rel.Id + " " + rel.Name + " " + rel.RowCount);
}
}
break;
case "n":
case "nonempty":
if (words.Length == 3)
{
switch (words[2])
{
case "r":
case "relations":
foreach (var rel in iterator.RelationId2Info.Values
.Where(r => r.RowCount > 0).OrderBy(r => r.RowCount))
{
Console.WriteLine(rel.Id + " " + rel.Name + " " + rel.RowCount);
}
break;
}
}
break;
}
break;
case "e":
case "exit":
return;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
Console.WriteLine(
"Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "realpath":
{
var res = PlatformMethods.Instance.RealPath(args[0]);
if (res == null)
{
Console.WriteLine("Error resolving real path for " + args[0]);
}
else
{
Console.WriteLine(res);
}
break;
}
case "nicedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "interactive":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo(true);
Interactive(iterator, visitor);
break;
}
case "comparedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToConsoleVisitorForComparison();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "comparesplitdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToFilesVisitorForComparison(HashType.Crc32);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "diskdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
using var tst = File.CreateText(Path.Combine(args[0], "dump.txt"));
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToFileVisitorNice(tst);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dumpnull":
case "null":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "check":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var transaction = kdb.StartReadOnlyTransaction();
var keyValueCount = transaction.GetKeyValueCount();
transaction.FindFirstKey();
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey();
}
break;
}
case "stat":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "statm": // Stat but by old managed implementation
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "kvi":
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "kvim": // Kvi but by old managed implementation
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "fileheaders":
{
using var dfc = new OnDiskFileCollection(args[0]);
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyIndex:
{
details =
$"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}";
var usedFiles = keyIndex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
break;
}
case IFileTransactionLog trlog:
details = $"Previous File Id: {trlog.PreviousFileId}";
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine(
$"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "export":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var tr = kdb.StartReadOnlyTransaction();
using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
KeyValueDBExportImporter.Export(tr, st);
break;
}
case "import":
{
using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc);
using var tr = kdb.StartTransaction();
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
break;
}
case "leaks":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
Console.WriteLine("Leaks: ");
odb.Open(kdb, false);
odb.DumpLeaks();
break;
}
case "leakscode":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
odb.DumpLeaksCode();
break;
}
case "frequency":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleFrequencyVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
visitor.OutputStatistic();
}
break;
case "size":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleSizeVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
break;
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "diskdump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
using (var tst = File.CreateText(Path.Combine(args[0], "dump.txt")))
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToFileVisitorNice(tst);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "fileheaders":
{
using (var dfc = new OnDiskFileCollection(args[0]))
{
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyindex:
{
details =
$"KVCount:{keyindex.KeyValueCount} CommitUlong:{keyindex.CommitUlong} TrLogFileId:{keyindex.TrLogFileId} TrLogOffset:{keyindex.TrLogOffset}";
var usedFiles = keyindex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
break;
}
case IFileTransactionLog trlog:
details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine($"Opened in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine($"Final compaction in {sw.Elapsed.TotalSeconds:F1}");
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "snapshot.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
case "import":
{
using (var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat")))
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartTransaction())
{
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
Console.WriteLine(
"Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check, findsplitbrain, fulldiskdump, trldump");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "realpath":
{
var res = PlatformMethods.Instance.RealPath(args[0]);
if (res == null)
{
Console.WriteLine("Error resolving real path for " + args[0]);
}
else
{
Console.WriteLine(res);
}
break;
}
case "nicedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "interactive":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo(true);
Interactive(iterator, visitor);
break;
}
case "comparedump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToConsoleVisitorForComparison();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "comparesplitdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
using var visitor = new ToFilesVisitorForComparison(HashType.Crc32);
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "fulldiskdump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
using var visitor = new ToFilesVisitorWithSecondaryKeys();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate(sortTableByNameAsc: true);
break;
}
case "diskdump":
{
var dbDir = args[0];
var openUpToCommitUlong = args.Length >= 3 ? (ulong?)ulong.Parse(args[2]) : null;
var fileName = Path.Combine(dbDir, "dump.txt");
DiskDump(dbDir, fileName, openUpToCommitUlong);
break;
}
case "dump":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "dumpnull":
case "null":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
break;
}
case "check":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var transaction = kdb.StartReadOnlyTransaction();
var keyValueCount = transaction.GetKeyValueCount();
transaction.FindFirstKey(ReadOnlySpan <byte> .Empty);
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey(ReadOnlySpan <byte> .Empty);
}
break;
}
case "stat":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "statm": // Stat but by old managed implementation
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "kvi":
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "kvim": // Kvi but by old managed implementation
{
var sw = Stopwatch.StartNew();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
kdb.CreateKvi(CancellationToken.None);
Console.WriteLine(
$"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
break;
}
case "fileheaders":
{
using var dfc = new OnDiskFileCollection(args[0]);
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
switch (fi.Value)
{
case IKeyIndex keyIndex:
{
details =
$"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}\n";
details += LoadUsedFilesFromKvi(keyIndex, fcfi, fi.Key);
break;
}
case IFileTransactionLog trlog:
details = $"Previous File Id: {trlog.PreviousFileId}";
break;
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
break;
}
case "compact":
{
var sw = new Stopwatch();
sw.Start();
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
kdb.Logger = new ConsoleKvdbLogger();
sw.Stop();
Console.WriteLine(
$"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
sw.Restart();
while (kdb.Compact(new CancellationToken()))
{
sw.Stop();
Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
sw.Restart();
}
sw.Stop();
Console.WriteLine(
$"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
Console.WriteLine(kdb.CalcStats());
break;
}
case "export":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var tr = kdb.StartReadOnlyTransaction();
using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
KeyValueDBExportImporter.Export(tr, st);
break;
}
case "import":
{
using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(dfc);
using var tr = kdb.StartTransaction();
KeyValueDBExportImporter.Import(tr, st);
tr.Commit();
break;
}
case "leaks":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
Console.WriteLine("Leaks: ");
odb.Open(kdb, false);
odb.DumpLeaks();
break;
}
case "leakscode":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
odb.DumpLeaksCode();
break;
}
case "frequency":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleFrequencyVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
visitor.OutputStatistic();
}
break;
case "size":
{
using var dfc = new OnDiskFileCollection(args[0]);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
Compression = new SnappyCompressionStrategy(),
OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var tr = odb.StartTransaction();
var visitor = new ToConsoleSizeVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
break;
case "findsplitbrain":
{
if (args.Length != 6)
{
Console.WriteLine(
"usage: ODBDump Eagle_0 findsplitbrain Eagle_1 1000 100000 INestedEventTable");
return;
}
var startEvent = ulong.Parse(args[3]);
var endEvent = ulong.Parse(args[4]);
var relationName = args[5];
var(found, lastGood, firstBad) =
FindSplitBrain(args[0], args[2], relationName, startEvent, endEvent);
if (found)
{
Console.WriteLine($"Split occured between {lastGood} and {firstBad}");
DiskDump(args[0], $"dump_{lastGood}_0.txt", lastGood);
DiskDump(args[2], $"dump_{lastGood}_1.txt", lastGood);
DiskDump(args[0], $"dump_{firstBad}_0.txt", firstBad);
DiskDump(args[2], $"dump_{firstBad}_1.txt", firstBad);
}
}
break;
case "trldump":
{
ITrlVisitor visitor = new ConsoleTrlVisitor();
using var dfc = new OnDiskFileCollection(args[0]);
foreach (var file in dfc.Enumerate())
{
try
{
visitor.StartFile(file.Index, file.GetSize());
var reader = new TrlFileReader(file);
reader.Iterate(visitor);
}
catch (Exception e)
{
visitor.OperationDetail("Failure " + e.Message);
}
visitor.EndOperation();
}
}
break;
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
var action = "nicedump";
if (args.Length > 1)
{
action = args[1].ToLowerInvariant();
}
switch (action)
{
case "nicedump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitorNice();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dump":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var trkv = kdb.StartReadOnlyTransaction())
using (var tr = odb.StartTransaction())
{
Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
var visitor = new ToConsoleVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "dumpnull":
case "null":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToNullVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
}
}
break;
}
case "stat":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
{
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "fileheaders":
{
using (var dfc = new OnDiskFileCollection(args[0]))
{
var fcfi = new FileCollectionWithFileInfos(dfc);
foreach (var fi in fcfi.FileInfos)
{
var details = "";
var keyindex = fi.Value as IKeyIndex;
if (keyindex != null)
{
details = string.Format("KVCount:{0} CommitUlong:{1} TrLogFileId:{2} TrLogOffset:{3}", keyindex.KeyValueCount, keyindex.CommitUlong, keyindex.TrLogFileId, keyindex.TrLogOffset);
var usedFiles = keyindex.UsedFilesInOlderGenerations;
if (usedFiles != null)
{
details += " UsedFiles:" + string.Join(",", usedFiles);
}
}
var trlog = fi.Value as IFileTransactionLog;
if (trlog != null)
{
details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
}
Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
}
}
break;
}
case "compact":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
{
Console.WriteLine("Starting first compaction");
while (kdb.Compact(new CancellationToken()))
{
Console.WriteLine(kdb.CalcStats());
Console.WriteLine("Another compaction needed");
}
Console.WriteLine(kdb.CalcStats());
}
break;
}
case "export":
{
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var tr = kdb.StartReadOnlyTransaction())
using (var st = File.Create(Path.Combine(args[0], "export.dat")))
{
KeyValueDBExportImporter.Export(tr, st);
}
break;
}
default:
{
Console.WriteLine($"Unknown action: {action}");
break;
}
}
}