static string DumpRelationContent(string dbDir, string relationName, ulong?openUpToCommitUlong) { using var dfc = new OnDiskFileCollection(dbDir); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = openUpToCommitUlong }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); var visitor = new ToStringFastVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.LoadGlobalInfo(); var relationIdInfo = iterator.RelationId2Info.FirstOrDefault(kvp => kvp.Value.Name.EndsWith(relationName)); if (relationIdInfo.Value == null) { return(""); } iterator.IterateRelation(relationIdInfo.Value); return(visitor.ToString()); }
static ObjectDB CreateDb() { var fc = new OnDiskFileCollection("."); var lowDb = new KeyValueDB(fc, new NoCompressionStrategy()); var db = new ObjectDB(); db.Open(lowDb, true); return(db); }
public File(OnDiskFileCollection owner, uint index, string fileName) { _owner = owner; _index = index; _fileName = fileName; _stream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, 1, FileOptions.None); _handle = _stream.SafeFileHandle !; _writer = new Writer(this); }
public void Run(string dbDirectory, CancellationToken cancellationToken) { using var fc = new OnDiskFileCollection(dbDirectory); using var lowDb = new KeyValueDB(fc, new NoCompressionStrategy(), 2000); var values = new List <byte[]>(100); for (var i = 0; i < 100; i++) { values.Add(new byte[i]); } var rnd = new Random(); var mre = new ManualResetEvent(false); var t = Task.Run(async() => { var keyBuffer = new byte[10]; while (true) { using var tr = await lowDb.StartWritingTransaction(); for (var j = 0; j < 10000; j++) { var pos = 0; PackUnpack.PackVInt(keyBuffer, ref pos, j); tr.CreateOrUpdateKeyValue(keyBuffer.AsSpan(0, pos), values[rnd.Next(0, 99)]); if (cancellationToken.IsCancellationRequested) { return; } } Console.WriteLine("w "); tr.Commit(); mre.Set(); } }, cancellationToken); if (!cancellationToken.IsCancellationRequested) { mre.WaitOne(); } var r1 = StartReader(lowDb, "r1", 1, cancellationToken); var r2 = StartReader(lowDb, "r2", 10, cancellationToken); var r3 = StartReader(lowDb, "r3", 100, cancellationToken); Task.WaitAll(t, r1, r2, r3); }
public void CanCreateFile() { using (var dc = new OnDiskFileCollection(_dir)) { var f = dc.AddFile("kvi"); var w = f.GetAppenderWriter(); for (var i = 0; i < 32000; i++) { w.WriteInt32LE(i); } var data = new byte[4]; for (var i = 0; i < 32000; i++) { f.RandomRead(data, 0, 4, (ulong)i * 4, false); Assert.Equal(i, PackUnpack.UnpackInt32LE(data, 0)); } } }
void Initialize() { _dbdir = Path.GetTempPath() + "/deleteMeDB"; Directory.Delete(_dbdir, true); Directory.CreateDirectory(_dbdir); _fc = new OnDiskFileCollection(_dbdir); _kvdb = new KeyValueDB(new KeyValueDBOptions { Compression = new SnappyCompressionStrategy(), FileSplitSize = 100 * 1024 * 1024, FileCollection = _fc }); _kvdb.Logger = this; _odb = new ObjectDB(); _odb.Open(_kvdb, false); using (var tr = _odb.StartWritingTransaction().GetAwaiter().GetResult()) { _personRelation = tr.InitRelation <IPersonTable>("Person"); tr.Commit(); } }
void DoWork5ReadCheck() { _sw.Restart(); using (var fileCollection = new OnDiskFileCollection("data")) using (IKeyValueDB db = new KeyValueDB(fileCollection)) { using (var trCheck = db.StartTransaction()) { long pureDataLengthCheck = 0; while (trCheck.FindNextKey()) { pureDataLengthCheck += trCheck.GetKey().Length + trCheck.GetValue().Length; } if (pureDataLengthCheck != 396130000) { throw new Exception("Bug"); } } } _sw.Stop(); Console.WriteLine("Time: {0,15}ms", _sw.Elapsed.TotalMilliseconds); }
static void DiskDump(string dbDir, string fileName, ulong?openUpToCommitUlong) { using var dfc = new OnDiskFileCollection(dbDir); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = openUpToCommitUlong }); using var odb = new ObjectDB(); using var tst = File.CreateText(fileName); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); tst.WriteLine("CommitUlong: " + tr.GetCommitUlong()); tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToFileVisitorNice(tst); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); }
static void Main(string[] args) { if (args.Length < 1) { Console.WriteLine("Need to have just one parameter with directory of ObjectDB"); Console.WriteLine( "Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check"); return; } var action = "nicedump"; if (args.Length > 1) { action = args[1].ToLowerInvariant(); } switch (action) { case "realpath": { var res = PlatformMethods.Instance.RealPath(args[0]); if (res == null) { Console.WriteLine("Error resolving real path for " + args[0]); } else { Console.WriteLine(res); } break; } case "nicedump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "interactive": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.LoadGlobalInfo(true); Interactive(iterator, visitor); break; } case "comparedump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); var visitor = new ToConsoleVisitorForComparison(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(sortTableByNameAsc: true); break; } case "comparesplitdump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); var visitor = new ToFilesVisitorForComparison(HashType.Crc32); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(sortTableByNameAsc: true); break; } case "diskdump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); using var tst = File.CreateText(Path.Combine(args[0], "dump.txt")); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); tst.WriteLine("CommitUlong: " + tr.GetCommitUlong()); tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToFileVisitorNice(tst); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "dump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "dumpnull": case "null": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var tr = odb.StartTransaction(); var visitor = new ToNullVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "check": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var transaction = kdb.StartReadOnlyTransaction(); var keyValueCount = transaction.GetKeyValueCount(); transaction.FindFirstKey(); for (long kv = 0; kv < keyValueCount; kv++) { transaction.GetKey(); transaction.GetValue(); transaction.FindNextKey(); } break; } case "stat": { var sw = new Stopwatch(); sw.Start(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new BTreeKeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); sw.Stop(); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); Console.WriteLine(kdb.CalcStats()); break; } case "statm": // Stat but by old managed implementation { var sw = new Stopwatch(); sw.Start(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); sw.Stop(); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); Console.WriteLine(kdb.CalcStats()); break; } case "kvi": { var sw = Stopwatch.StartNew(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); sw.Restart(); kdb.CreateKvi(CancellationToken.None); Console.WriteLine( $"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); break; } case "kvim": // Kvi but by old managed implementation { var sw = Stopwatch.StartNew(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); sw.Restart(); kdb.CreateKvi(CancellationToken.None); Console.WriteLine( $"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); break; } case "fileheaders": { using var dfc = new OnDiskFileCollection(args[0]); var fcfi = new FileCollectionWithFileInfos(dfc); foreach (var fi in fcfi.FileInfos) { var details = ""; switch (fi.Value) { case IKeyIndex keyIndex: { details = $"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}"; var usedFiles = keyIndex.UsedFilesInOlderGenerations; if (usedFiles != null) { details += " UsedFiles:" + string.Join(",", usedFiles); } break; } case IFileTransactionLog trlog: details = $"Previous File Id: {trlog.PreviousFileId}"; break; } Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details); } break; } case "compact": { var sw = new Stopwatch(); sw.Start(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null); kdb.Logger = new ConsoleKvdbLogger(); sw.Stop(); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); sw.Restart(); while (kdb.Compact(new CancellationToken())) { sw.Stop(); Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}"); sw.Restart(); } sw.Stop(); Console.WriteLine( $"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); Console.WriteLine(kdb.CalcStats()); break; } case "export": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var tr = kdb.StartReadOnlyTransaction(); using var st = File.Create(Path.Combine(args[0], "snapshot.dat")); KeyValueDBExportImporter.Export(tr, st); break; } case "import": { using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat")); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(dfc); using var tr = kdb.StartTransaction(); KeyValueDBExportImporter.Import(tr, st); tr.Commit(); break; } case "leaks": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); Console.WriteLine("Leaks: "); odb.Open(kdb, false); odb.DumpLeaks(); break; } case "leakscode": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); odb.DumpLeaksCode(); break; } case "frequency": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var tr = odb.StartTransaction(); var visitor = new ToConsoleFrequencyVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); visitor.OutputStatistic(); } break; case "size": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var tr = odb.StartTransaction(); var visitor = new ToConsoleSizeVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } break; default: { Console.WriteLine($"Unknown action: {action}"); break; } } }
static void Main(string[] args) { if (args.Length < 1) { Console.WriteLine("Need to have just one parameter with directory of ObjectDB"); return; } var action = "nicedump"; if (args.Length > 1) { action = args[1].ToLowerInvariant(); } switch (action) { case "nicedump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "diskdump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) using (var tst = File.CreateText(Path.Combine(args[0], "dump.txt"))) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { tst.WriteLine("CommitUlong: " + tr.GetCommitUlong()); tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToFileVisitorNice(tst); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "dump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "dumpnull": case "null": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var tr = odb.StartTransaction()) { var visitor = new ToNullVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "stat": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) { Console.WriteLine(kdb.CalcStats()); } break; } case "fileheaders": { using (var dfc = new OnDiskFileCollection(args[0])) { var fcfi = new FileCollectionWithFileInfos(dfc); foreach (var fi in fcfi.FileInfos) { var details = ""; switch (fi.Value) { case IKeyIndex keyindex: { details = $"KVCount:{keyindex.KeyValueCount} CommitUlong:{keyindex.CommitUlong} TrLogFileId:{keyindex.TrLogFileId} TrLogOffset:{keyindex.TrLogOffset}"; var usedFiles = keyindex.UsedFilesInOlderGenerations; if (usedFiles != null) { details += " UsedFiles:" + string.Join(",", usedFiles); } break; } case IFileTransactionLog trlog: details = string.Format("Previous File Id: {0}", trlog.PreviousFileId); break; } Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details); } } break; } case "compact": { var sw = new Stopwatch(); sw.Start(); using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null)) { kdb.Logger = new ConsoleKvdbLogger(); sw.Stop(); Console.WriteLine($"Opened in {sw.Elapsed.TotalSeconds:F1}"); sw.Restart(); while (kdb.Compact(new CancellationToken())) { sw.Stop(); Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}"); sw.Restart(); } sw.Stop(); Console.WriteLine($"Final compaction in {sw.Elapsed.TotalSeconds:F1}"); Console.WriteLine(kdb.CalcStats()); } break; } case "export": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var tr = kdb.StartReadOnlyTransaction()) using (var st = File.Create(Path.Combine(args[0], "snapshot.dat"))) { KeyValueDBExportImporter.Export(tr, st); } break; } case "import": { using (var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat"))) using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var tr = kdb.StartTransaction()) { KeyValueDBExportImporter.Import(tr, st); tr.Commit(); } break; } default: { Console.WriteLine($"Unknown action: {action}"); break; } } }
static void Main(string[] args) { if (args.Length < 1) { Console.WriteLine("Need to have just one parameter with directory of ObjectDB"); return; } var action = "nicedump"; if (args.Length > 1) { action = args[1].ToLowerInvariant(); } switch (action) { case "nicedump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "dump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "dumpnull": case "null": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var tr = odb.StartTransaction()) { var visitor = new ToNullVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "stat": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) { Console.WriteLine(kdb.CalcStats()); } break; } case "compact": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null)) { Console.WriteLine("Starting first compaction"); while (kdb.Compact(new CancellationToken())) { Console.WriteLine(kdb.CalcStats()); Console.WriteLine("Another compaction needed"); } Console.WriteLine(kdb.CalcStats()); } break; } case "export": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var tr = kdb.StartReadOnlyTransaction()) using (var st = File.Create(Path.Combine(args[0], "export.dat"))) { KeyValueDBExportImporter.Export(tr, st); } break; } default: { Console.WriteLine($"Unknown action: {action}"); break; } } }
static void Main(string[] args) { if (args.Length < 1) { Console.WriteLine("Need to have just one parameter with directory of ObjectDB"); Console.WriteLine( "Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check, findsplitbrain, fulldiskdump, trldump"); return; } var action = "nicedump"; if (args.Length > 1) { action = args[1].ToLowerInvariant(); } switch (action) { case "realpath": { var res = PlatformMethods.Instance.RealPath(args[0]); if (res == null) { Console.WriteLine("Error resolving real path for " + args[0]); } else { Console.WriteLine(res); } break; } case "nicedump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "interactive": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.LoadGlobalInfo(true); Interactive(iterator, visitor); break; } case "comparedump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); var visitor = new ToConsoleVisitorForComparison(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(sortTableByNameAsc: true); break; } case "comparesplitdump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); using var visitor = new ToFilesVisitorForComparison(HashType.Crc32); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(sortTableByNameAsc: true); break; } case "fulldiskdump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); using var visitor = new ToFilesVisitorWithSecondaryKeys(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(sortTableByNameAsc: true); break; } case "diskdump": { var dbDir = args[0]; var openUpToCommitUlong = args.Length >= 3 ? (ulong?)ulong.Parse(args[2]) : null; var fileName = Path.Combine(dbDir, "dump.txt"); DiskDump(dbDir, fileName, openUpToCommitUlong); break; } case "dump": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var trkv = kdb.StartReadOnlyTransaction(); using var tr = odb.StartTransaction(); Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "dumpnull": case "null": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var tr = odb.StartTransaction(); var visitor = new ToNullVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); break; } case "check": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var transaction = kdb.StartReadOnlyTransaction(); var keyValueCount = transaction.GetKeyValueCount(); transaction.FindFirstKey(ReadOnlySpan <byte> .Empty); for (long kv = 0; kv < keyValueCount; kv++) { transaction.GetKey(); transaction.GetValue(); transaction.FindNextKey(ReadOnlySpan <byte> .Empty); } break; } case "stat": { var sw = new Stopwatch(); sw.Start(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); sw.Stop(); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); Console.WriteLine(kdb.CalcStats()); break; } case "statm": // Stat but by old managed implementation { var sw = new Stopwatch(); sw.Start(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); sw.Stop(); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); Console.WriteLine(kdb.CalcStats()); break; } case "kvi": { var sw = Stopwatch.StartNew(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); sw.Restart(); kdb.CreateKvi(CancellationToken.None); Console.WriteLine( $"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); break; } case "kvim": // Kvi but by old managed implementation { var sw = Stopwatch.StartNew(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); sw.Restart(); kdb.CreateKvi(CancellationToken.None); Console.WriteLine( $"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); break; } case "fileheaders": { using var dfc = new OnDiskFileCollection(args[0]); var fcfi = new FileCollectionWithFileInfos(dfc); foreach (var fi in fcfi.FileInfos) { var details = ""; switch (fi.Value) { case IKeyIndex keyIndex: { details = $"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}\n"; details += LoadUsedFilesFromKvi(keyIndex, fcfi, fi.Key); break; } case IFileTransactionLog trlog: details = $"Previous File Id: {trlog.PreviousFileId}"; break; } Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details); } break; } case "compact": { var sw = new Stopwatch(); sw.Start(); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null); kdb.Logger = new ConsoleKvdbLogger(); sw.Stop(); Console.WriteLine( $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); sw.Restart(); while (kdb.Compact(new CancellationToken())) { sw.Stop(); Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}"); sw.Restart(); } sw.Stop(); Console.WriteLine( $"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM"); Console.WriteLine(kdb.CalcStats()); break; } case "export": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var tr = kdb.StartReadOnlyTransaction(); using var st = File.Create(Path.Combine(args[0], "snapshot.dat")); KeyValueDBExportImporter.Export(tr, st); break; } case "import": { using var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat")); using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(dfc); using var tr = kdb.StartTransaction(); KeyValueDBExportImporter.Import(tr, st); tr.Commit(); break; } case "leaks": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); Console.WriteLine("Leaks: "); odb.Open(kdb, false); odb.DumpLeaks(); break; } case "leakscode": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); odb.DumpLeaksCode(); break; } case "frequency": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var tr = odb.StartTransaction(); var visitor = new ToConsoleFrequencyVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); visitor.OutputStatistic(); } break; case "size": { using var dfc = new OnDiskFileCollection(args[0]); using var kdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = dfc, ReadOnly = true, Compression = new SnappyCompressionStrategy(), OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null }); using var odb = new ObjectDB(); odb.Open(kdb, false); using var tr = odb.StartTransaction(); var visitor = new ToConsoleSizeVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } break; case "findsplitbrain": { if (args.Length != 6) { Console.WriteLine( "usage: ODBDump Eagle_0 findsplitbrain Eagle_1 1000 100000 INestedEventTable"); return; } var startEvent = ulong.Parse(args[3]); var endEvent = ulong.Parse(args[4]); var relationName = args[5]; var(found, lastGood, firstBad) = FindSplitBrain(args[0], args[2], relationName, startEvent, endEvent); if (found) { Console.WriteLine($"Split occured between {lastGood} and {firstBad}"); DiskDump(args[0], $"dump_{lastGood}_0.txt", lastGood); DiskDump(args[2], $"dump_{lastGood}_1.txt", lastGood); DiskDump(args[0], $"dump_{firstBad}_0.txt", firstBad); DiskDump(args[2], $"dump_{firstBad}_1.txt", firstBad); } } break; case "trldump": { ITrlVisitor visitor = new ConsoleTrlVisitor(); using var dfc = new OnDiskFileCollection(args[0]); foreach (var file in dfc.Enumerate()) { try { visitor.StartFile(file.Index, file.GetSize()); var reader = new TrlFileReader(file); reader.Iterate(visitor); } catch (Exception e) { visitor.OperationDetail("Failure " + e.Message); } visitor.EndOperation(); } } break; default: { Console.WriteLine($"Unknown action: {action}"); break; } } }
static void Main(string[] args) { if (args.Length < 1) { Console.WriteLine("Need to have just one parameter with directory of ObjectDB"); return; } var action = "nicedump"; if (args.Length > 1) { action = args[1].ToLowerInvariant(); } switch (action) { case "nicedump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitorNice(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "dump": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var trkv = kdb.StartReadOnlyTransaction()) using (var tr = odb.StartTransaction()) { Console.WriteLine("CommitUlong: " + tr.GetCommitUlong()); Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0)); Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1)); var visitor = new ToConsoleVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "dumpnull": case "null": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var odb = new ObjectDB()) { odb.Open(kdb, false); using (var tr = odb.StartTransaction()) { var visitor = new ToNullVisitor(); var iterator = new ODBIterator(tr, visitor); iterator.Iterate(); } } break; } case "stat": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) { Console.WriteLine(kdb.CalcStats()); } break; } case "fileheaders": { using (var dfc = new OnDiskFileCollection(args[0])) { var fcfi = new FileCollectionWithFileInfos(dfc); foreach (var fi in fcfi.FileInfos) { var details = ""; var keyindex = fi.Value as IKeyIndex; if (keyindex != null) { details = string.Format("KVCount:{0} CommitUlong:{1} TrLogFileId:{2} TrLogOffset:{3}", keyindex.KeyValueCount, keyindex.CommitUlong, keyindex.TrLogFileId, keyindex.TrLogOffset); var usedFiles = keyindex.UsedFilesInOlderGenerations; if (usedFiles != null) { details += " UsedFiles:" + string.Join(",", usedFiles); } } var trlog = fi.Value as IFileTransactionLog; if (trlog != null) { details = string.Format("Previous File Id: {0}", trlog.PreviousFileId); } Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key, fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details); } } break; } case "compact": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null)) { Console.WriteLine("Starting first compaction"); while (kdb.Compact(new CancellationToken())) { Console.WriteLine(kdb.CalcStats()); Console.WriteLine("Another compaction needed"); } Console.WriteLine(kdb.CalcStats()); } break; } case "export": { using (var dfc = new OnDiskFileCollection(args[0])) using (var kdb = new KeyValueDB(dfc)) using (var tr = kdb.StartReadOnlyTransaction()) using (var st = File.Create(Path.Combine(args[0], "export.dat"))) { KeyValueDBExportImporter.Export(tr, st); } break; } default: { Console.WriteLine($"Unknown action: {action}"); break; } } }