Ejemplo n.º 1
0
        static void DiskDump(string dbDir, string fileName, ulong?openUpToCommitUlong)
        {
            using var dfc = new OnDiskFileCollection(dbDir);
            using var kdb = new KeyValueDB(new KeyValueDBOptions
            {
                FileCollection      = dfc,
                ReadOnly            = true,
                OpenUpToCommitUlong = openUpToCommitUlong
            });
            using var odb = new ObjectDB();
            using var tst = File.CreateText(fileName);
            odb.Open(kdb, false);
            using var trkv = kdb.StartReadOnlyTransaction();
            using var tr   = odb.StartTransaction();
            tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
            tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
            tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
            var visitor  = new ToFileVisitorNice(tst);
            var iterator = new ODBIterator(tr, visitor);

            iterator.Iterate();
        }
Ejemplo n.º 2
0
        static void Main(string[] args)
        {
            if (args.Length < 1)
            {
                Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
                Console.WriteLine(
                    "Optional second parameter: nicedump, comparedump, diskdump, dump, dumpnull, stat, fileheaders, compact, export, import, leaks, leakscode, size, frequency, interactive, check");
                return;
            }

            var action = "nicedump";

            if (args.Length > 1)
            {
                action = args[1].ToLowerInvariant();
            }

            switch (action)
            {
            case "realpath":
            {
                var res = PlatformMethods.Instance.RealPath(args[0]);
                if (res == null)
                {
                    Console.WriteLine("Error resolving real path for " + args[0]);
                }
                else
                {
                    Console.WriteLine(res);
                }
                break;
            }

            case "nicedump":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var trkv = kdb.StartReadOnlyTransaction();
                using var tr   = odb.StartTransaction();
                Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                var visitor  = new ToConsoleVisitorNice();
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate();

                break;
            }

            case "interactive":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var trkv = kdb.StartReadOnlyTransaction();
                using var tr   = odb.StartTransaction();
                Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                var visitor  = new ToConsoleVisitorNice();
                var iterator = new ODBIterator(tr, visitor);
                iterator.LoadGlobalInfo(true);
                Interactive(iterator, visitor);

                break;
            }

            case "comparedump":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var trkv = kdb.StartReadOnlyTransaction();
                using var tr   = odb.StartTransaction();
                var visitor  = new ToConsoleVisitorForComparison();
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate(sortTableByNameAsc: true);

                break;
            }

            case "comparesplitdump":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var trkv = kdb.StartReadOnlyTransaction();
                using var tr   = odb.StartTransaction();
                var visitor  = new ToFilesVisitorForComparison(HashType.Crc32);
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate(sortTableByNameAsc: true);

                break;
            }

            case "diskdump":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                using var tst = File.CreateText(Path.Combine(args[0], "dump.txt"));
                odb.Open(kdb, false);
                using var trkv = kdb.StartReadOnlyTransaction();
                using var tr   = odb.StartTransaction();
                tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                var visitor  = new ToFileVisitorNice(tst);
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate();

                break;
            }

            case "dump":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var trkv = kdb.StartReadOnlyTransaction();
                using var tr   = odb.StartTransaction();
                Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                var visitor  = new ToConsoleVisitor();
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate();

                break;
            }

            case "dumpnull":
            case "null":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var tr = odb.StartTransaction();
                var visitor  = new ToNullVisitor();
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate();

                break;
            }

            case "check":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var transaction = kdb.StartReadOnlyTransaction();
                var keyValueCount = transaction.GetKeyValueCount();
                transaction.FindFirstKey();
                for (long kv = 0; kv < keyValueCount; kv++)
                {
                    transaction.GetKey();
                    transaction.GetValue();
                    transaction.FindNextKey();
                }

                break;
            }

            case "stat":
            {
                var sw = new Stopwatch();
                sw.Start();
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new BTreeKeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                sw.Stop();
                Console.WriteLine(
                    $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
                Console.WriteLine(kdb.CalcStats());

                break;
            }

            case "statm":     // Stat but by old managed implementation
            {
                var sw = new Stopwatch();
                sw.Start();
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                sw.Stop();
                Console.WriteLine(
                    $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
                Console.WriteLine(kdb.CalcStats());

                break;
            }

            case "kvi":
            {
                var sw = Stopwatch.StartNew();
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new BTreeKeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
                Console.WriteLine(
                    $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
                sw.Restart();
                kdb.CreateKvi(CancellationToken.None);
                Console.WriteLine(
                    $"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");

                break;
            }

            case "kvim":     // Kvi but by old managed implementation
            {
                var sw = Stopwatch.StartNew();
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
                Console.WriteLine(
                    $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
                sw.Restart();
                kdb.CreateKvi(CancellationToken.None);
                Console.WriteLine(
                    $"Created kvi in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");

                break;
            }

            case "fileheaders":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                var fcfi = new FileCollectionWithFileInfos(dfc);
                foreach (var fi in fcfi.FileInfos)
                {
                    var details = "";
                    switch (fi.Value)
                    {
                    case IKeyIndex keyIndex:
                    {
                        details =
                            $"KVCount:{keyIndex.KeyValueCount} CommitUlong:{keyIndex.CommitUlong} TrLogFileId:{keyIndex.TrLogFileId} TrLogOffset:{keyIndex.TrLogOffset}";
                        var usedFiles = keyIndex.UsedFilesInOlderGenerations;
                        if (usedFiles != null)
                        {
                            details += " UsedFiles:" + string.Join(",", usedFiles);
                        }

                        break;
                    }

                    case IFileTransactionLog trlog:
                        details = $"Previous File Id: {trlog.PreviousFileId}";
                        break;
                    }

                    Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
                                      fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
                }

                break;
            }

            case "compact":
            {
                var sw = new Stopwatch();
                sw.Start();
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null);
                kdb.Logger    = new ConsoleKvdbLogger();
                sw.Stop();
                Console.WriteLine(
                    $"Opened in {sw.Elapsed.TotalSeconds:F1}s Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
                sw.Restart();
                while (kdb.Compact(new CancellationToken()))
                {
                    sw.Stop();
                    Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
                    sw.Restart();
                }

                sw.Stop();
                Console.WriteLine(
                    $"Final compaction in {sw.Elapsed.TotalSeconds:F1} Using {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024}MB RAM");
                Console.WriteLine(kdb.CalcStats());

                break;
            }

            case "export":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var tr = kdb.StartReadOnlyTransaction();
                using var st = File.Create(Path.Combine(args[0], "snapshot.dat"));
                KeyValueDBExportImporter.Export(tr, st);

                break;
            }

            case "import":
            {
                using var st  = File.OpenRead(Path.Combine(args[0], "snapshot.dat"));
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(dfc);
                using var tr  = kdb.StartTransaction();
                KeyValueDBExportImporter.Import(tr, st);
                tr.Commit();

                break;
            }

            case "leaks":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                Console.WriteLine("Leaks: ");
                odb.Open(kdb, false);
                odb.DumpLeaks();

                break;
            }

            case "leakscode":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                odb.DumpLeaksCode();

                break;
            }

            case "frequency":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var tr = odb.StartTransaction();
                var visitor  = new ToConsoleFrequencyVisitor();
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate();
                visitor.OutputStatistic();
            }
            break;

            case "size":
            {
                using var dfc = new OnDiskFileCollection(args[0]);
                using var kdb = new KeyValueDB(new KeyValueDBOptions
                    {
                        FileCollection      = dfc,
                        ReadOnly            = true,
                        Compression         = new SnappyCompressionStrategy(),
                        OpenUpToCommitUlong = args.Length >= 3 ? (ulong?) ulong.Parse(args[2]) : null
                    });
                using var odb = new ObjectDB();
                odb.Open(kdb, false);
                using var tr = odb.StartTransaction();
                var visitor  = new ToConsoleSizeVisitor();
                var iterator = new ODBIterator(tr, visitor);
                iterator.Iterate();
            }
            break;

            default:
            {
                Console.WriteLine($"Unknown action: {action}");
                break;
            }
            }
        }
Ejemplo n.º 3
0
        static void Main(string[] args)
        {
            if (args.Length < 1)
            {
                Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
                return;
            }

            var action = "nicedump";

            if (args.Length > 1)
            {
                action = args[1].ToLowerInvariant();
            }

            switch (action)
            {
            case "nicedump":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc))
                        using (var odb = new ObjectDB())
                        {
                            odb.Open(kdb, false);
                            using (var trkv = kdb.StartReadOnlyTransaction())
                                using (var tr = odb.StartTransaction())
                                {
                                    Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                                    Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                                    Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                                    var visitor  = new ToConsoleVisitorNice();
                                    var iterator = new ODBIterator(tr, visitor);
                                    iterator.Iterate();
                                }
                        }

                break;
            }

            case "diskdump":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc))
                        using (var odb = new ObjectDB())
                            using (var tst = File.CreateText(Path.Combine(args[0], "dump.txt")))
                            {
                                odb.Open(kdb, false);
                                using (var trkv = kdb.StartReadOnlyTransaction())
                                    using (var tr = odb.StartTransaction())
                                    {
                                        tst.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                                        tst.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                                        tst.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                                        var visitor  = new ToFileVisitorNice(tst);
                                        var iterator = new ODBIterator(tr, visitor);
                                        iterator.Iterate();
                                    }
                            }

                break;
            }

            case "dump":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc))
                        using (var odb = new ObjectDB())
                        {
                            odb.Open(kdb, false);
                            using (var trkv = kdb.StartReadOnlyTransaction())
                                using (var tr = odb.StartTransaction())
                                {
                                    Console.WriteLine("CommitUlong: " + tr.GetCommitUlong());
                                    Console.WriteLine("Ulong[0] oid: " + trkv.GetUlong(0));
                                    Console.WriteLine("Ulong[1] dictid: " + trkv.GetUlong(1));
                                    var visitor  = new ToConsoleVisitor();
                                    var iterator = new ODBIterator(tr, visitor);
                                    iterator.Iterate();
                                }
                        }

                break;
            }

            case "dumpnull":
            case "null":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc))
                        using (var odb = new ObjectDB())
                        {
                            odb.Open(kdb, false);
                            using (var tr = odb.StartTransaction())
                            {
                                var visitor  = new ToNullVisitor();
                                var iterator = new ODBIterator(tr, visitor);
                                iterator.Iterate();
                            }
                        }

                break;
            }

            case "stat":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc))
                    {
                        Console.WriteLine(kdb.CalcStats());
                    }

                break;
            }

            case "fileheaders":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                {
                    var fcfi = new FileCollectionWithFileInfos(dfc);
                    foreach (var fi in fcfi.FileInfos)
                    {
                        var details = "";
                        switch (fi.Value)
                        {
                        case IKeyIndex keyindex:
                        {
                            details =
                                $"KVCount:{keyindex.KeyValueCount} CommitUlong:{keyindex.CommitUlong} TrLogFileId:{keyindex.TrLogFileId} TrLogOffset:{keyindex.TrLogOffset}";
                            var usedFiles = keyindex.UsedFilesInOlderGenerations;
                            if (usedFiles != null)
                            {
                                details += " UsedFiles:" + string.Join(",", usedFiles);
                            }

                            break;
                        }

                        case IFileTransactionLog trlog:
                            details = string.Format("Previous File Id: {0}", trlog.PreviousFileId);
                            break;
                        }

                        Console.WriteLine("File {0} Guid:{3} Gen:{2} Type:{1} {4}", fi.Key,
                                          fi.Value.FileType.ToString(), fi.Value.Generation, fi.Value.Guid, details);
                    }
                }

                break;
            }

            case "compact":
            {
                var sw = new Stopwatch();
                sw.Start();
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc, new SnappyCompressionStrategy(), 100 * 1024 * 1024, null))
                    {
                        kdb.Logger = new ConsoleKvdbLogger();
                        sw.Stop();
                        Console.WriteLine($"Opened in {sw.Elapsed.TotalSeconds:F1}");
                        sw.Restart();
                        while (kdb.Compact(new CancellationToken()))
                        {
                            sw.Stop();
                            Console.WriteLine($"Compaction iteration in {sw.Elapsed.TotalSeconds:F1}");
                            sw.Restart();
                        }

                        sw.Stop();
                        Console.WriteLine($"Final compaction in {sw.Elapsed.TotalSeconds:F1}");
                        Console.WriteLine(kdb.CalcStats());
                    }

                break;
            }

            case "export":
            {
                using (var dfc = new OnDiskFileCollection(args[0]))
                    using (var kdb = new KeyValueDB(dfc))
                        using (var tr = kdb.StartReadOnlyTransaction())
                            using (var st = File.Create(Path.Combine(args[0], "snapshot.dat")))
                            {
                                KeyValueDBExportImporter.Export(tr, st);
                            }

                break;
            }

            case "import":
            {
                using (var st = File.OpenRead(Path.Combine(args[0], "snapshot.dat")))
                    using (var dfc = new OnDiskFileCollection(args[0]))
                        using (var kdb = new KeyValueDB(dfc))
                            using (var tr = kdb.StartTransaction())
                            {
                                KeyValueDBExportImporter.Import(tr, st);
                                tr.Commit();
                            }

                break;
            }

            default:
            {
                Console.WriteLine($"Unknown action: {action}");
                break;
            }
            }
        }