public async Task TestDifferentCounts()
        {
            string databasePath = @"C:\Temp\Checkpoint\";

            var countWithSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : true);

            var countWithoutSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : false);

            // Content count with SetTotalOrderSeek: 4193451, without: 4231080
            Output.WriteLine($"Content count with SetTotalOrderSeek: {countWithSetTotalOrderSeek}, without: {countWithoutSetTotalOrderSeek}");

            async Task <int> getContentOnMachine(MachineId machineId, bool setTotalOrderSeek)
            {
                var db = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
                {
                    CleanOnInitialize = false,
                    Epoch             = "DM_S220201001ReconcileTest.03312020.0",
                    UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = setTotalOrderSeek,
                }, () => CollectionUtilities.EmptyArray <MachineId>());

                var context = new OperationContext(new Context(Logger));
                await db.StartupAsync(context).ThrowIfFailure();

                var count = db.EnumerateSortedHashesWithContentSizeForMachineId(context, currentMachineId: new MachineId(121)).Count();

                await db.ShutdownAsync(context).ThrowIfFailure();

                return(count);
            }
        }
        /// <inheritdoc />
        protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
        {
            var result = await _database.StartupAsync(context);

            _database.UpdateClusterState(context, _clusterState, false);
            return(result);
        }
        public async Task DoesNotAllowWritingWhenInReadOnlyMode()
        {
            var configuration = new RocksDbContentLocationDatabaseConfiguration(_workingDirectory.Path)
            {
                CleanOnInitialize = false,
            };

            var context = new Context(Logger);
            var ctx     = new OperationContext(context);

            // First, we create the database
            {
                var db = new RocksDbContentLocationDatabase(Clock, configuration, () => new MachineId[] { });
                await db.StartupAsync(ctx).ShouldBeSuccess();

                db.SetGlobalEntry("test", "hello");
                await db.ShutdownAsync(ctx).ShouldBeSuccess();
            }

            configuration.OpenReadOnly = true;
            {
                var db = new RocksDbContentLocationDatabase(Clock, configuration, () => new MachineId[] { });
                await db.StartupAsync(ctx).ShouldBeSuccess();

                db.TryGetGlobalEntry("test", out var readValue);
                readValue.Should().Be("hello");

                Assert.Throws <BuildXLException>(() =>
                {
                    db.SetGlobalEntry("test", "hello2");
                });

                await db.ShutdownAsync(ctx).ShouldBeSuccess();
            }
        }
Exemplo n.º 4
0
        /// <inheritdoc />
        protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
        {
            var cldbStartup = await _database.StartupAsync(context);

            if (!cldbStartup.Succeeded)
            {
                return(cldbStartup);
            }

            return(BoolResult.Success);
        }
        public async Task TestMissingHashes()
        {
            // This test demonstrates that some hashes are missing when we enumerate without setting SetTotalOrderSeek.
            var missingHashes = new string[] {
                "VSO0:004206D0C9111C8494CBAC",
                "VSO0:004214AD89057251C75AB4",
                "VSO0:0042213A319603AEADDACA",
                "VSO0:00422E765196DDCEC5DDCB",
                "VSO0:00424E130937109ACC9FE1",
                "VSO0:00425A5011BC79F9A7B051",
                "VSO0:0042674A7B2782F0F4BEE4",
                "VSO0:004268BF4380D7CEA4786E",
                "VSO0:00426D1DAA182A6D265461",
                "VSO0:0042866D19EB33B8E11C8E",
            }.Select(str => ParseShortHash(str)).ToHashSet();

            var    maxHash      = missingHashes.Max();
            string databasePath = @"C:\Temp\Checkpoint\";

            var hashesWithoutSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : false);

            var hashesWithSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : true);

            foreach (var h in missingHashes)
            {
                // Hash: VSO0: 0042866D19EB33B8E11C, Contains(TotalOrderSeek = True): True, Contains(TotalOrderSeek = false): False
                bool db1Contains = hashesWithSetTotalOrderSeek.Contains(h);
                bool db2Contains = hashesWithoutSetTotalOrderSeek.Contains(h);
                Output.WriteLine($"Hash: {h}, Contains (TotalOrderSeek=True): {db1Contains}, Contains (TotalOrderSeek=false): {db2Contains}");
            }

            Output.WriteLine($"Content count with SetTotalOrderSeek: {hashesWithSetTotalOrderSeek.Count}, without: {hashesWithoutSetTotalOrderSeek.Count}");

            async Task <HashSet <ShortHash> > getContentOnMachine(MachineId machineId, bool setTotalOrderSeek)
            {
                var db = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
                {
                    CleanOnInitialize = false,
                    Epoch             = "DM_S220201001ReconcileTest.03312020.0",
                    UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = setTotalOrderSeek,
                }, () => CollectionUtilities.EmptyArray <MachineId>());

                var context = new OperationContext(new Context(Logger));
                await db.StartupAsync(context).ThrowIfFailure();

                var hashes = db.EnumerateSortedHashesWithContentSizeForMachineId(context, currentMachineId: new MachineId(121))
                             .TakeWhile(h => h.hash < maxHash || h.hash == maxHash).Select(h => h.hash).ToHashSet();

                await db.ShutdownAsync(context).ThrowIfFailure();

                return(hashes);
            }
        }
        public async Task MimicReconcileWithMachineIdFilteringWithNoDeserialization()
        {
            string databasePath = @"C:\Users\seteplia\AppData\Local\Temp\CloudStore\DatabaseTest\";
            var    db           = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
            {
                CleanOnInitialize = false
            }, () => CollectionUtilities.EmptyArray <MachineId>());

            var context = new OperationContext(new Context(Logger));
            await db.StartupAsync(context).ThrowIfFailure();

            var sw        = Stopwatch.StartNew();
            var reconcile = MimicFastReconcileLogic(context, machineId: 42, db);

            sw.Stop();

            Output.WriteLine($"Reconcile by {sw.ElapsedMilliseconds}ms. Added: {reconcile.addedContent.Count}, removed: {reconcile.removedContent.Count}");
        }
        public async Task MimicReconciliationWithFullDatabaseEnumerationByKeys()
        {
            // This is an original and slower version of reconciliation.
            string databasePath = @"C:\Temp\Checkpoint\";
            var    db           = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
            {
                CleanOnInitialize = false
            }, () => CollectionUtilities.EmptyArray <MachineId>());


            var context = new OperationContext(new Context(Logger));
            await db.StartupAsync(context).ThrowIfFailure();

            var sw        = Stopwatch.StartNew();
            var reconcile = MimicOldReconcile(context, machineId: 121, db);

            sw.Stop();

            Output.WriteLine($"Reconcile by {sw.ElapsedMilliseconds}ms. Added: {reconcile.addedContent.Count}, removed: {reconcile.removedContent.Count}");
        }