public async Task TestDifferentCounts()
{
string databasePath = @"C:\Temp\Checkpoint\";
var countWithSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : true);
var countWithoutSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : false);
// Content count with SetTotalOrderSeek: 4193451, without: 4231080
Output.WriteLine($"Content count with SetTotalOrderSeek: {countWithSetTotalOrderSeek}, without: {countWithoutSetTotalOrderSeek}");
async Task <int> getContentOnMachine(MachineId machineId, bool setTotalOrderSeek)
{
var db = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
{
CleanOnInitialize = false,
Epoch = "DM_S220201001ReconcileTest.03312020.0",
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = setTotalOrderSeek,
}, () => CollectionUtilities.EmptyArray <MachineId>());
var context = new OperationContext(new Context(Logger));
await db.StartupAsync(context).ThrowIfFailure();
var count = db.EnumerateSortedHashesWithContentSizeForMachineId(context, currentMachineId: new MachineId(121)).Count();
await db.ShutdownAsync(context).ThrowIfFailure();
return(count);
}
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
var result = await _database.StartupAsync(context);
_database.UpdateClusterState(context, _clusterState, false);
return(result);
}
public async Task DoesNotAllowWritingWhenInReadOnlyMode()
{
var configuration = new RocksDbContentLocationDatabaseConfiguration(_workingDirectory.Path)
{
CleanOnInitialize = false,
};
var context = new Context(Logger);
var ctx = new OperationContext(context);
// First, we create the database
{
var db = new RocksDbContentLocationDatabase(Clock, configuration, () => new MachineId[] { });
await db.StartupAsync(ctx).ShouldBeSuccess();
db.SetGlobalEntry("test", "hello");
await db.ShutdownAsync(ctx).ShouldBeSuccess();
}
configuration.OpenReadOnly = true;
{
var db = new RocksDbContentLocationDatabase(Clock, configuration, () => new MachineId[] { });
await db.StartupAsync(ctx).ShouldBeSuccess();
db.TryGetGlobalEntry("test", out var readValue);
readValue.Should().Be("hello");
Assert.Throws <BuildXLException>(() =>
{
db.SetGlobalEntry("test", "hello2");
});
await db.ShutdownAsync(ctx).ShouldBeSuccess();
}
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
var cldbStartup = await _database.StartupAsync(context);
if (!cldbStartup.Succeeded)
{
return(cldbStartup);
}
return(BoolResult.Success);
}
public async Task TestMissingHashes()
{
// This test demonstrates that some hashes are missing when we enumerate without setting SetTotalOrderSeek.
var missingHashes = new string[] {
"VSO0:004206D0C9111C8494CBAC",
"VSO0:004214AD89057251C75AB4",
"VSO0:0042213A319603AEADDACA",
"VSO0:00422E765196DDCEC5DDCB",
"VSO0:00424E130937109ACC9FE1",
"VSO0:00425A5011BC79F9A7B051",
"VSO0:0042674A7B2782F0F4BEE4",
"VSO0:004268BF4380D7CEA4786E",
"VSO0:00426D1DAA182A6D265461",
"VSO0:0042866D19EB33B8E11C8E",
}.Select(str => ParseShortHash(str)).ToHashSet();
var maxHash = missingHashes.Max();
string databasePath = @"C:\Temp\Checkpoint\";
var hashesWithoutSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : false);
var hashesWithSetTotalOrderSeek = await getContentOnMachine(new MachineId(121), setTotalOrderSeek : true);
foreach (var h in missingHashes)
{
// Hash: VSO0: 0042866D19EB33B8E11C, Contains(TotalOrderSeek = True): True, Contains(TotalOrderSeek = false): False
bool db1Contains = hashesWithSetTotalOrderSeek.Contains(h);
bool db2Contains = hashesWithoutSetTotalOrderSeek.Contains(h);
Output.WriteLine($"Hash: {h}, Contains (TotalOrderSeek=True): {db1Contains}, Contains (TotalOrderSeek=false): {db2Contains}");
}
Output.WriteLine($"Content count with SetTotalOrderSeek: {hashesWithSetTotalOrderSeek.Count}, without: {hashesWithoutSetTotalOrderSeek.Count}");
async Task <HashSet <ShortHash> > getContentOnMachine(MachineId machineId, bool setTotalOrderSeek)
{
var db = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
{
CleanOnInitialize = false,
Epoch = "DM_S220201001ReconcileTest.03312020.0",
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = setTotalOrderSeek,
}, () => CollectionUtilities.EmptyArray <MachineId>());
var context = new OperationContext(new Context(Logger));
await db.StartupAsync(context).ThrowIfFailure();
var hashes = db.EnumerateSortedHashesWithContentSizeForMachineId(context, currentMachineId: new MachineId(121))
.TakeWhile(h => h.hash < maxHash || h.hash == maxHash).Select(h => h.hash).ToHashSet();
await db.ShutdownAsync(context).ThrowIfFailure();
return(hashes);
}
}
public async Task MimicReconcileWithMachineIdFilteringWithNoDeserialization()
{
string databasePath = @"C:\Users\seteplia\AppData\Local\Temp\CloudStore\DatabaseTest\";
var db = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
{
CleanOnInitialize = false
}, () => CollectionUtilities.EmptyArray <MachineId>());
var context = new OperationContext(new Context(Logger));
await db.StartupAsync(context).ThrowIfFailure();
var sw = Stopwatch.StartNew();
var reconcile = MimicFastReconcileLogic(context, machineId: 42, db);
sw.Stop();
Output.WriteLine($"Reconcile by {sw.ElapsedMilliseconds}ms. Added: {reconcile.addedContent.Count}, removed: {reconcile.removedContent.Count}");
}
public async Task MimicReconciliationWithFullDatabaseEnumerationByKeys()
{
// This is an original and slower version of reconciliation.
string databasePath = @"C:\Temp\Checkpoint\";
var db = new RocksDbContentLocationDatabase(TestClock, new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(databasePath))
{
CleanOnInitialize = false
}, () => CollectionUtilities.EmptyArray <MachineId>());
var context = new OperationContext(new Context(Logger));
await db.StartupAsync(context).ThrowIfFailure();
var sw = Stopwatch.StartNew();
var reconcile = MimicOldReconcile(context, machineId: 121, db);
sw.Stop();
Output.WriteLine($"Reconcile by {sw.ElapsedMilliseconds}ms. Added: {reconcile.addedContent.Count}, removed: {reconcile.removedContent.Count}");
}