public void FingerprintToRedisKey()
{
var fp = Fingerprint.Random();
var key = _serializer.ToRedisKey(fp);
Assert.Equal(fp.ToString(), key);
}
public Task GetSelectorsGivesSelectorsInReverseLruOrderAfterGet()
{
var context = new Context(Logger);
var weakFingerprint = Fingerprint.Random();
var selector1 = Selector.Random();
var selector2 = Selector.Random();
var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
return(RunTestAsync(context, async(store, session) =>
{
await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
_clock.Increment();
await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
_clock.Increment();
await session.GetContentHashListAsync(context, strongFingerprint1, Token).ShouldBeSuccess();
_clock.Increment();
List <GetSelectorResult> getSelectorResults = await session.GetSelectors(context, weakFingerprint, Token).ToListAsync();
Assert.Equal(2, getSelectorResults.Count);
GetSelectorResult r1 = getSelectorResults[0];
Assert.True(r1.Succeeded);
Assert.True(r1.Selector == selector1);
GetSelectorResult r2 = getSelectorResults[1];
Assert.True(r2.Succeeded);
Assert.True(r2.Selector == selector2);
}));
}
public Task GetSelectorsGivesSelectors()
{
var weakFingerprint = Fingerprint.Random();
var selector1 = Selector.Random();
var selector2 = Selector.Random();
var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
return(RunTest((context, contentLocationDatabase) =>
{
contentLocationDatabase.AddOrGetContentHashList(
context, strongFingerprint1, contentHashListWithDeterminism1).ShouldBeSuccess();
contentLocationDatabase.AddOrGetContentHashList(
context, strongFingerprint2, contentHashListWithDeterminism2).ShouldBeSuccess();
List <GetSelectorResult> getSelectorResults = contentLocationDatabase.GetSelectors(context, weakFingerprint).ToList();
Assert.Equal(2, getSelectorResults.Count);
GetSelectorResult r1 = getSelectorResults[0].ShouldBeSuccess();
Assert.True(r1.Selector == selector1 || r1.Selector == selector2);
GetSelectorResult r2 = getSelectorResults[1].ShouldBeSuccess();
Assert.True(r2.Selector == selector1 || r2.Selector == selector2);
}));
}
public Task PreferSharedUrgencyHintIsRespected()
{
var context = new Context(Logger);
var weakFingerprint = Fingerprint.Random();
var selector1 = Selector.Random();
var selector2 = Selector.Random();
var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
return(RunTestAsync(context, async session =>
{
await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
_clock.Increment();
await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
_clock.Increment();
var contentHashList = session.GetContentHashListAsync(context, strongFingerprint1, Token, UrgencyHint.PreferShared);
List <GetSelectorResult> getSelectorResults = await session.GetSelectors(context, weakFingerprint, Token).ToListAsync(CancellationToken.None);
Assert.Equal(2, getSelectorResults.Count);
GetSelectorResult r1 = getSelectorResults[0];
Assert.True(r1.Succeeded);
Assert.True(r1.Selector == selector2);
GetSelectorResult r2 = getSelectorResults[1];
Assert.True(r2.Succeeded);
Assert.True(r2.Selector == selector1);
}));
}
public void PartialBinaryRoundtrip()
{
var value = Fingerprint.Random();
Utilities.TestSerializationRoundtrip(value, value.SerializeBytes,
reader => new Fingerprint(value.Length, reader));
}
public void FixedBytesRoundtrip()
{
var v1 = Fingerprint.Random();
var fixedBytes = v1.ToFixedBytes();
var v2 = new Fingerprint(fixedBytes, v1.Length);
Assert.Equal(v1, v2);
}
public void StringRoundtrip()
{
var v1 = Fingerprint.Random();
var serialized = v1.Serialize();
var v2 = new Fingerprint(serialized);
Assert.Equal(v1, v2);
}
public void GetHashCodeSameWhenEqual()
{
var weakFingerprint = Fingerprint.Random();
var selector = Selector.Random();
var v1 = new StrongFingerprint(weakFingerprint, selector);
var v2 = new StrongFingerprint(weakFingerprint, selector);
Assert.Equal(v1.GetHashCode(), v2.GetHashCode());
}
public void EqualsObjectTrue()
{
var weakFingerprint = Fingerprint.Random();
var selector = Selector.Random();
var v1 = new StrongFingerprint(weakFingerprint, selector);
var v2 = new StrongFingerprint(weakFingerprint, selector) as object;
Assert.True(v1.Equals(v2));
}
public Task GarbageCollectionDeletesInLruOrder()
{
var context = new Context(Logger);
var weakFingerprint = Fingerprint.Random();
var selector1 = Selector.Random();
var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
var selector2 = Selector.Random();
var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
return(RunTestAsync(context,
funcAsync: async(store, session) =>
{
await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
_clock.Increment();
await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
_clock.Increment();
// Force update the last access time of the first fingerprint
await session.GetContentHashListAsync(context, strongFingerprint1, Token).ShouldBeSuccess();
_clock.Increment();
RocksDbContentLocationDatabase database = (store as RocksDbMemoizationStore)?.RocksDbDatabase;
Contract.Assert(database != null);
var ctx = new OperationContext(context);
await database.GarbageCollectAsync(ctx).ShouldBeSuccess();
var r1 = database.GetContentHashList(ctx, strongFingerprint1).ShouldBeSuccess().ContentHashListWithDeterminism;
r1.Should().BeEquivalentTo(contentHashListWithDeterminism1);
var r2 = database.GetContentHashList(ctx, strongFingerprint2).ShouldBeSuccess().ContentHashListWithDeterminism;
r2.ContentHashList.Should().BeNull();
r2.Determinism.Should().Be(CacheDeterminism.None);
database.Counters[ContentLocationDatabaseCounters.GarbageCollectMetadataEntriesRemoved].Value.Should().Be(1);
database.Counters[ContentLocationDatabaseCounters.GarbageCollectMetadataEntriesScanned].Value.Should().Be(2);
},
createStoreFunc: createStoreInternal));
// This is needed because type errors arise if you inline
IMemoizationStore createStoreInternal(DisposableDirectory disposableDirectory)
{
return(CreateStore(testDirectory: disposableDirectory, configMutator: (configuration) =>
{
configuration.MetadataGarbageCollectionEnabled = true;
configuration.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = 1;
// Disables automatic GC
configuration.GarbageCollectionInterval = Timeout.InfiniteTimeSpan;
}));
}
}
public void BufferRoundtrip()
{
var buffer = new byte[Fingerprint.MaxLength];
var v1 = Fingerprint.Random();
v1.Serialize(buffer);
var v2 = new Fingerprint(buffer);
Assert.Equal(v1, v2);
}
public void EqualsFalseSameWeakFingerprint()
{
var weakFingerprint = Fingerprint.Random();
var v1 = new StrongFingerprint(weakFingerprint, Selector.Random());
var v2 = new StrongFingerprint(weakFingerprint, Selector.Random());
Assert.False(v1.Equals(v2));
Assert.False(v1 == v2);
Assert.True(v1 != v2);
}
public void EqualsFalseSameSelector()
{
var selector = Selector.Random();
var v1 = new StrongFingerprint(Fingerprint.Random(), selector);
var v2 = new StrongFingerprint(Fingerprint.Random(), selector);
Assert.False(v1.Equals(v2));
Assert.False(v1 == v2);
Assert.True(v1 != v2);
}
public Task GetNonExistingSelectors()
{
IList <Fingerprint> weakFingerprints =
Enumerable.Range(0, _itemCount).Select(x => Fingerprint.Random()).ToList();
return(Run(
nameof(GetNonExistingSelectors),
_nullSetupFunc,
async session => await GetSelectors(_context, session, weakFingerprints)));
}
public Task GetSelectorsGivesZeroTasks()
{
var weakFingerprint = Fingerprint.Random();
return(RunTest((context, contentLocationDatabase) =>
{
IEnumerable <GetSelectorResult> tasks = contentLocationDatabase.GetSelectors(context, weakFingerprint).ToList();
Assert.Equal(0, tasks.Count());
}));
}
public Task GetSelectorsPassThrough()
{
var context = new Context(Logger);
var weakFingerprint = Fingerprint.Random();
return(RunMockSessionTestAsync(context, async session =>
{
await session.GetSelectors(context, weakFingerprint, Token, NonDefaultUrgencyHint).ToListAsync();
Assert.True(_mockMemoizationSession.GetSelectorsParams.Contains(weakFingerprint), $"Expected to find ({weakFingerprint}) in set of GetSelectors calls.");
}));
}
public Task GetSelectorsGivesZeroTasks()
{
var context = new Context(Logger);
var weakFingerprint = Fingerprint.Random();
return(RunReadOnlyTestAsync(context, async session =>
{
IEnumerable <GetSelectorResult> tasks = await session.GetSelectors(context, weakFingerprint, Token).ToList();
Assert.Equal(0, tasks.Count());
}));
}
public void BufferPositiveOffsetRoundtrip()
{
const int offset = 3;
var buffer = new byte[Fingerprint.MaxLength + offset];
var v1 = Fingerprint.Random();
v1.Serialize(buffer, offset);
var v2 = new Fingerprint(buffer, Fingerprint.MaxLength, offset);
Assert.Equal(v1, v2);
}
public void StrongFingerprintFromMemoization()
{
MemoizationStrongFingerprint memoizationStrongFingerprint = new MemoizationStrongFingerprint(
Fingerprint.Random(WeakFingerprintHash.Length),
Selector.Random(HashingType, WeakFingerprintHash.Length));
BuildXLStrongFingerprint buildXLStrongFingerprint = memoizationStrongFingerprint.FromMemoization(CacheId);
Assert.Equal(memoizationStrongFingerprint.WeakFingerprint.ToByteArray(), buildXLStrongFingerprint.WeakFingerprint.ToArray());
Assert.Equal(memoizationStrongFingerprint.Selector.ContentHash.FromMemoization(), buildXLStrongFingerprint.CasElement);
Assert.Equal(memoizationStrongFingerprint.Selector.Output, buildXLStrongFingerprint.HashElement.ToArray());
Assert.Equal(CacheId, buildXLStrongFingerprint.CacheId);
}
private CompositeGraphFingerprint CreateRandomWithModules(params string[] modules)
{
return(new CompositeGraphFingerprint()
{
OverallFingerprint = new ContentFingerprint(Fingerprint.Random(33)),
FilterHash = Fingerprint.Random(33),
EvaluationFilter = new EvaluationFilter(m_symbolTable, m_pathTable, new FullSymbol[0], new AbsolutePath[0], modules.Select(m => StringId.Create(this.m_pathTable.StringTable, m)).ToArray()),
BuildEngineHash = FingerprintUtilities.ZeroFingerprint,
ConfigFileHash = FingerprintUtilities.ZeroFingerprint,
QualifierHash = FingerprintUtilities.ZeroFingerprint,
});
}
public Task TestGetOrAddSelectorsError()
{
return(RunTest(async(context, metadataCache, redisDb) =>
{
var weakFp = Fingerprint.Random();
var errorResult = new GetSelectorResult("Error");
Task <Result <Selector[]> > failure = Task.FromResult(Result.FromError <Selector[]>(errorResult));
var selectorResults = await metadataCache.GetOrAddSelectorsAsync(context, weakFp, fp => failure).ShouldBeError();
Assert.Equal(errorResult.ErrorMessage, selectorResults.ErrorMessage);
// Check Redis data
Assert.Equal(0, redisDb.DbSet.Keys.Count);
}));
}
public Task TestGetOrAddSelectorsEmptyCache()
{
return(RunTest(async(context, metadataCache, redisDb) =>
{
var selectors = new[] { Selector.Random(), Selector.Random(), };
var weakFp = Fingerprint.Random();
await metadataCache.GetOrAddSelectorsAsync(context, weakFp, fp => ToSelectorResult(selectors)).ShouldBeSuccess();
// Check Redis data
Assert.Equal(1, redisDb.DbSet.Keys.Count);
var cacheKey = _redisSerializer.ToRedisKey(weakFp).Prepend(RedisNameSpace);
Assert.True(redisDb.DbSet.ContainsKey(cacheKey));
Assert.Equal(2, redisDb.DbSet[cacheKey].Length);
}));
}
public void FingerprintWithDifferentTopLevelHash()
{
var oldFingerprint = CreateRandomWithModulesAndValues(modules: new[] { "A", "B" }, valueNames: new[] { "v", "v2" });
oldFingerprint.TopLevelHash = Fingerprint.Random(33);
var newFingerprint = CreateRandomWithModulesAndValues(modules: new[] { "A", "B" }, valueNames: new[] { "v", "v2" });
var comparison = oldFingerprint.CompareFingerprint(newFingerprint);
Assert.Equal(GraphCacheMissReason.FingerprintChanged, comparison);
newFingerprint.TopLevelHash = oldFingerprint.TopLevelHash;
var comparison2 = oldFingerprint.CompareFingerprint(newFingerprint);
Assert.Equal(GraphCacheMissReason.NoMiss, comparison2);
}
public Task GarbageCollectionRuns()
{
var context = new Context(Logger);
var weakFingerprint = Fingerprint.Random();
var selector1 = Selector.Random();
var selector2 = Selector.Random();
var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
return(RunTestAsync(context,
funcAsync: async(store, session) => {
await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
_clock.Increment();
// Notice we don't increment the clock here
await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
RocksDbContentLocationDatabase database = (store as RocksDbMemoizationStore)?.Database;
Contract.Assert(database != null);
var ctx = new OperationContext(context);
database.GarbageCollect(ctx);
var r1 = database.GetContentHashList(ctx, strongFingerprint1).ShouldBeSuccess().ContentHashListWithDeterminism;
r1.ContentHashList.Should().BeNull();
r1.Determinism.Should().Be(CacheDeterminism.None);
var r2 = database.GetContentHashList(ctx, strongFingerprint2).ShouldBeSuccess().ContentHashListWithDeterminism;
r2.Should().BeEquivalentTo(contentHashListWithDeterminism2);
},
createStoreFunc: createStoreInternal));
// This is needed because type errors arise if you inline
IMemoizationStore createStoreInternal(DisposableDirectory disposableDirectory)
{
return(CreateStore(testDirectory: disposableDirectory, configMutator: (configuration) =>
{
configuration.MetadataGarbageCollectionEnabled = true;
configuration.MetadataGarbageCollectionProtectionTime = TimeSpan.FromMilliseconds(1);
// Disables automatic GC
configuration.GarbageCollectionInterval = Timeout.InfiniteTimeSpan;
}));
}
}
public void NonDeterministicItemShouldNotBeCached()
{
var cache = ContentHashListWithDeterminismCache.Instance;
var cacheNamespace = "ns";
var strongFingerprint = new StrongFingerprint(Fingerprint.Random(), Selector.Random());
var contentHashList = new ContentHashListWithDeterminism(
new ContentHashList(new ContentHash[] { ContentHash.Random() }),
CacheDeterminism.None);
// Lets check first that the content hash list is not deterministic.
contentHashList.Determinism.IsDeterministic.Should().BeFalse();
cache.AddValue(cacheNamespace, strongFingerprint, contentHashList);
bool foundInCache = cache.TryGetValue(cacheNamespace, strongFingerprint, out _);
foundInCache.Should().BeFalse("The cache should not save non-deterministic content hash lists.");
}
private CompositeGraphFingerprint CreateRandomWithModulesAndValues(string[] modules, string[] valueNames)
{
return(new CompositeGraphFingerprint()
{
OverallFingerprint = new ContentFingerprint(Fingerprint.Random(33)),
FilterHash = Fingerprint.Random(33),
EvaluationFilter = new EvaluationFilter(
m_symbolTable,
m_pathTable,
valueNames.Select(vn => FullSymbol.Create(m_symbolTable, new StringSegment(vn))).ToArray(),
new AbsolutePath[0],
modules.Select(m => StringId.Create(m_pathTable.StringTable, m)).ToArray()),
BuildEngineHash = FingerprintUtilities.ZeroFingerprint,
ConfigFileHash = FingerprintUtilities.ZeroFingerprint,
QualifierHash = FingerprintUtilities.ZeroFingerprint,
TopLevelHash = FingerprintUtilities.ZeroFingerprint,
});
}
public void PartialBinaryRoundtrip()
{
using (var ms = new MemoryStream())
{
using (var writer = new BinaryWriter(ms))
{
var v1 = Fingerprint.Random();
v1.SerializeBytes(writer);
ms.Position = 0;
using (var reader = new BinaryReader(ms))
{
var v2 = new Fingerprint(v1.Length, reader);
Assert.Equal(v1, v2);
}
}
}
}
public void NonDeterministicItemShouldNotBeCached()
{
var cache = BlobContentHashListCache.Instance;
var cacheNamespace = "ns";
var strongFingerprint = new StrongFingerprint(Fingerprint.Random(), Selector.Random());
var contentHashList = new BlobContentHashListWithCacheMetadata(
new BlobContentHashListWithDeterminism(Guid.NewGuid(), CreateBlobIdentifier()),
DateTime.UtcNow);
// Lets check first that the content hash list is not deterministic.
contentHashList.Determinism.IsDeterministic.Should().BeFalse();
cache.AddValue(cacheNamespace, strongFingerprint, contentHashList);
bool foundInCache = cache.TryGetValue(cacheNamespace, strongFingerprint, out _);
foundInCache.Should().BeFalse("The cache should not save non-deterministic content hash lists.");
}
public async Task TestGetOrAddSelectorsExisting()
{
var selectors = new[] { Selector.Random(), Selector.Random(), };
var weakFp = Fingerprint.Random();
var redisValues = new Dictionary <RedisKey, RedisValue[]>
{
{ _redisSerializer.ToRedisKey(weakFp).Prepend(RedisNameSpace), _redisSerializer.ToRedisValues(selectors) },
};
using (var mockDb = new MockRedisDatabase(
SystemClock.Instance,
setData: redisValues))
{
await RunTest(
mockDb,
async (context, metadataCache, redisDb) =>
{
var selectorResult = await metadataCache.GetOrAddSelectorsAsync(
context,
weakFp,
fp =>
{
throw new InvalidOperationException(
"GetFunc not expected to be called since data is already present in the cache");
}).ShouldBeSuccess();
// Check result
Assert.Equal(2, selectorResult.Value.Length);
foreach (Selector result in selectorResult.Value)
{
Assert.True(selectors.Contains(result));
}
});
}
}
/// <summary>
/// Create a random Fingerprint value.
/// </summary>
public static Fingerprint CreateRandom()
{
Contract.Ensures(Contract.Result <Fingerprint>().Length == FingerprintLength, UnexpectedLengthContractViolationMessage);
return(Fingerprint.Random(FingerprintLength));
}
