public void ViaCacheRoundtrip()
{
var guid = CacheDeterminism.NewCacheGuid();
var determinism = CacheDeterminism.ViaCache(guid, DateTime.MaxValue);
Assert.Equal(guid, determinism.EffectiveGuid);
}
public void ContentHashListWithDeterminismProperty()
{
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));
Assert.Equal(contentHashListWithDeterminism, new GetContentHashListResult(contentHashListWithDeterminism).ContentHashListWithDeterminism);
}
public void EqualsFalseContentHashListMismatch()
{
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires);
var v1 = new GetContentHashListResult(new ContentHashListWithDeterminism(ContentHashList.Random(), determinism));
var v2 = new GetContentHashListResult(new ContentHashListWithDeterminism(ContentHashList.Random(), determinism));
Assert.False(v1.Equals(v2));
}
public void GetHashCodeNotSameWhenNotEqual()
{
var v1 = new GetContentHashListResult(new ContentHashListWithDeterminism(
ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)));
var v2 = new GetContentHashListResult(new ContentHashListWithDeterminism(
ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)));
Assert.NotEqual(v1.GetHashCode(), v2.GetHashCode());
}
public void EqualsFalseDeterminismMismatch()
{
var v1 = new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.MaxValue)));
var v2 = new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.MaxValue)));
Assert.False(v1.Equals(v2));
}
public async Task Roundtrip()
{
var fileName = $"{Guid.NewGuid()}.json";
using (var directory = new DisposableDirectory(FileSystem))
{
var sessionId = 42;
var serializedConfig = "Foo";
var pat = Guid.NewGuid().ToString();
var numOperations = 3;
var operations = Enumerable.Range(0, numOperations)
.Select(_ => generateRandomOperation())
.ToList();
var sessionInfo = new HibernatedCacheSessionInfo(sessionId, serializedConfig, pat, operations);
var sessions1 = new HibernatedSessions <HibernatedCacheSessionInfo>(new List <HibernatedCacheSessionInfo> {
sessionInfo
});
await sessions1.WriteAsync(FileSystem, directory.Path, fileName);
FileSystem.HibernatedSessionsExists(directory.Path, fileName).Should().BeTrue();
var fileSize = FileSystem.GetFileSize(directory.Path / fileName);
fileSize.Should().BeGreaterThan(0);
var sessions2 = await FileSystem.ReadHibernatedSessionsAsync <HibernatedCacheSessionInfo>(directory.Path, fileName);
sessions2.Sessions.Count.Should().Be(1);
sessions2.Sessions[0].Id.Should().Be(sessionId);
sessions2.Sessions[0].SerializedSessionConfiguration.Should().Be(serializedConfig);
sessions2.Sessions[0].Pat.Should().Be(pat);
sessions2.Sessions[0].PendingPublishingOperations.Should().BeEquivalentTo(operations);
await FileSystem.DeleteHibernatedSessions(directory.Path, fileName);
PublishingOperation generateRandomOperation()
{
var amountOfHashes = 3;
var hashes = Enumerable.Range(0, amountOfHashes).Select(_ => ContentHash.Random()).ToArray();
var contentHashList = new ContentHashList(hashes);
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddMilliseconds(ThreadSafeRandom.Generator.Next()));
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random());
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
return(new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListWithDeterminism,
StrongFingerprint = strongFingerprint
});
}
}
}
public void EqualsTrueNotReferenceEqualContentHashList()
{
var contentHashList = ContentHashList.Random();
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(contentHashList, determinism);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(contentHashList, determinism);
var v1 = new GetContentHashListResult(contentHashListWithDeterminism1);
var v2 = new GetContentHashListResult(contentHashListWithDeterminism2);
Assert.True(v1.Equals(v2));
}
public async Task DisconnectMostRemoteCacheAddNewReconnect()
{
string cacheId = "MutlipleCacheRemote";
ICache testCache = await InitializeCacheAsync(NewCache(cacheId, true, false, true)).SuccessAsync();
VerticalCacheAggregator lowerVert = testCache as VerticalCacheAggregator;
XAssert.IsNotNull(lowerVert);
CallbackCacheWrapper callbackCache = lowerVert.RemoteCache as CallbackCacheWrapper;
XAssert.IsNotNull(callbackCache);
VerticalCacheAggregator upperVert = callbackCache.WrappedCache as VerticalCacheAggregator;
XAssert.IsNotNull(upperVert);
ICacheSession session = await testCache.CreateSessionAsync().SuccessAsync();
VerticalAggregatorDisconnectTests.DisconnectCache(upperVert.RemoteCache);
FullCacheRecord cacheRecord = await FakeBuild.DoPipAsync(session, "Test Pip");
VerticalAggregatorDisconnectTests.ConnectCache(upperVert.RemoteCache);
// Now query each cache, and verify only the remote content is in each.
var aggregatorStats = new Dictionary <string, double>();
var remoteDeterminism = CacheDeterminism.ViaCache(upperVert.RemoteCache.CacheGuid, CacheDeterminism.NeverExpires);
foreach (var currentCache in new Tuple <ICache, CacheDeterminism, string, int, Dictionary <string, double> >[]
{
new Tuple <ICache, CacheDeterminism, string, int, Dictionary <string, double> >(testCache, remoteDeterminism, lowerVert.LocalCache.CacheId, 3, aggregatorStats),
new Tuple <ICache, CacheDeterminism, string, int, Dictionary <string, double> >(lowerVert.LocalCache, remoteDeterminism, lowerVert.LocalCache.CacheId, 1, null),
new Tuple <ICache, CacheDeterminism, string, int, Dictionary <string, double> >(upperVert.LocalCache, remoteDeterminism, upperVert.LocalCache.CacheId, 1, null),
new Tuple <ICache, CacheDeterminism, string, int, Dictionary <string, double> >(upperVert.RemoteCache, remoteDeterminism, upperVert.RemoteCache.CacheId, 1, null)
})
{
await ValidateItemsInCacheAsync(
currentCache.Item1,
cacheRecord.StrongFingerprint.WeakFingerprint,
new List <CasHash>(cacheRecord.CasEntries),
currentCache.Item2,
cacheRecord.StrongFingerprint.CasElement,
currentCache.Item3,
currentCache.Item4);
}
await testCache.ShutdownAsync().SuccessAsync();
}
public Task AddOrGetContentHashListPassThrough()
{
var context = new Context(Logger);
var strongFingerprint = StrongFingerprint.Random();
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
ContentHashList.Random(),
CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));
return(RunMockSessionTestAsync(context, session =>
{
session.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListWithDeterminism, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess();
Assert.True(_mockMemoizationSession.AddOrGetContentHashListAsyncParams.Contains(new Tuple <StrongFingerprint, ContentHashListWithDeterminism>(strongFingerprint, contentHashListWithDeterminism)));
return Task.FromResult(0);
}));
}
public Task AddExpiredDeterminism()
{
var strongFingerprint = StrongFingerprint.Random();
var expirationUtc = DateTime.UtcNow - TimeSpan.FromDays(7);
var guid = CacheDeterminism.NewCacheGuid();
var determinism = CacheDeterminism.ViaCache(guid, expirationUtc);
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), determinism);
return(RunTest((context, contentLocationDatabase) =>
{
contentLocationDatabase.AddOrGetContentHashList(
context, strongFingerprint, contentHashListWithDeterminism).ShouldBeSuccess();
var result = contentLocationDatabase.GetContentHashList(context, strongFingerprint);
Assert.Equal(CacheDeterminism.None.EffectiveGuid, result.ContentHashListWithDeterminism.Determinism.EffectiveGuid);
}));
}
/// <summary>
/// Determine the Determinism to return.
/// </summary>
protected static CacheDeterminism UnpackDeterminism(ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId)
{
Contract.Assert(cacheMetadata != null);
if (cacheMetadata.ContentHashListWithDeterminism.Determinism.Guid == CacheDeterminism.Tool.Guid)
{
// Value is Tool-deterministic
return(CacheDeterminism.Tool);
}
var expirationUtc = cacheMetadata.GetEffectiveExpirationTimeUtc();
return(expirationUtc == null
? CacheDeterminism.None // Value is unbacked in VSTS
: CacheDeterminism.ViaCache(cacheId, expirationUtc.Value)); // Value is backed in VSTS
}
public Task AddExpiredDeterminism()
{
var context = new Context(Logger);
var strongFingerprint = StrongFingerprint.Random();
var expirationUtc = DateTime.UtcNow - TimeSpan.FromDays(7);
var guid = CacheDeterminism.NewCacheGuid();
var determinism = CacheDeterminism.ViaCache(guid, expirationUtc);
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), determinism);
return(RunTestAsync(context, async session =>
{
await session.AddOrGetContentHashListAsync(
context, strongFingerprint, contentHashListWithDeterminism, Token).ShouldBeSuccess();
var result = await session.GetContentHashListAsync(context, strongFingerprint, Token);
Assert.Equal(CacheDeterminism.None.EffectiveGuid, result.ContentHashListWithDeterminism.Determinism.EffectiveGuid);
}));
}
public void ContentHashListWithDeterminismToRedisValue()
{
var chl = ContentHashList.Random();
var d = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow);
var chlwd = new ContentHashListWithDeterminism(chl, d);
var redisValue = _serializer.ToRedisValue(chlwd);
var expected =
HexUtilities.BytesToHex(d.Serialize()) +
RedisSerializer.RedisValueSeparator +
RedisSerializer.RedisValueExistsSemaphore +
RedisSerializer.RedisValueSeparator +
chl.Serialize() +
RedisSerializer.RedisValueSeparator +
RedisSerializer.RedisValueSeparator;
Assert.Equal(expected, redisValue);
}
/// <inheritdoc />
public override object ReadJson(
JsonReader reader,
Type objectType,
object existingValue,
JsonSerializer serializer)
{
JObject jsonObject = JObject.Load(reader);
byte[] payload = null;
JToken payloadValue;
if (jsonObject.TryGetValue(PayloadFieldName, StringComparison.Ordinal, out payloadValue))
{
payload = HexUtilities.HexToBytes(payloadValue.Value <string>());
}
JToken expirationUtcToken;
var expirationUtc = jsonObject.TryGetValue(ExpirationUtcFieldName, StringComparison.Ordinal, out expirationUtcToken)
? DateTime.FromBinary(expirationUtcToken.Value <long>())
: CacheDeterminism.Expired;
CacheDeterminism determinism =
CacheDeterminism.ViaCache(
Guid.Parse(jsonObject.GetValue(DeterminismFieldName, StringComparison.Ordinal).Value <string>()),
expirationUtc);
var contentHashes = new List <ContentHash>();
foreach (
string contentHashString in jsonObject.GetValue(HashesFieldName, StringComparison.Ordinal).Values <string>())
{
ContentHash deserializedContentHash;
if (!ContentHash.TryParse(contentHashString, out deserializedContentHash))
{
throw new InvalidDataException("Unable to parse hash out of JSON Token");
}
contentHashes.Add(deserializedContentHash);
}
var contentHashList = new ContentHashList(contentHashes.ToArray(), payload);
return(new ContentHashListWithDeterminism(contentHashList, determinism));
}
public async Task SinglePhaseNonDeterministic()
{
const string TestName = nameof(SinglePhaseNonDeterministic);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
PipDefinition[] pips =
{
new PipDefinition("PipA", determinism: CacheDeterminism.SinglePhaseNonDeterministic),
new PipDefinition("PipB", determinism: CacheDeterminism.SinglePhaseNonDeterministic)
};
FullCacheRecord[] records = (await pips.BuildAsync(session)).ToArray();
XAssert.AreEqual(2, records.Length);
// Now, we should be able to redo those adds but swap the CasEntries in the records
// and get them replaced
for (int i = 0; i < 2; i++)
{
// Make sure we have our own strong fingerprint (no cheating by the cache for this one)
var strong = records[i].StrongFingerprint;
strong = new StrongFingerprint(strong.WeakFingerprint, strong.CasElement, strong.HashElement, "testing");
// Validate that the GetCacheEntry produces what we expect
var entries = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(records[i].CasEntries, entries);
// Validate that the other record I am going to do is different
var other = records[1 - i].CasEntries;
XAssert.AreNotEqual(entries, other, "Other record must be different!");
var newRecord = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, other).SuccessAsync();
XAssert.IsNull(newRecord.Record, "Should have replaced");
entries = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(other, entries);
XAssert.AreNotEqual(records[i].CasEntries, entries);
// Try to replace the existing SinglePhaseNonDeterministic case with a recird that is deterministic with a cache
var fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)));
XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with cache deterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType());
// Try to replace the existing SinglePhaseNonDeterministic case with a recird that is deterministic tool
fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.Tool));
XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with tool deterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType());
// Try to replace the existing SinglePhaseNonDeterministic case with a recird that is not deterministic
fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.None));
XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with non-deterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType());
}
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
private static CacheDeterminism RandomCacheDeterminism()
{
return(CacheDeterminism.ViaCache(Guid.NewGuid(), DateTime.UtcNow));
}
public async Task CacheSessionDataIsHibernated()
{
using var testDirectory = new DisposableDirectory(FileSystem);
var cacheDirectory = testDirectory.Path / "Service";
var cacheName = "theCache";
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
[cacheName] = cacheDirectory / "Root"
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem)
{
GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
};
var scenario = "Default";
var server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
var context = new OperationContext(new Context(Logger));
await server.StartupAsync(context).ShouldBeSuccess();
var pat = Guid.NewGuid().ToString();
var publishingConfig = new PublishingConfigDummy
{
PublishAsynchronously = true,
};
using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario);
await clientCache.StartupAsync(context).ShouldBeSuccess();
var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session;
await clientSession.StartupAsync(context).ShouldBeSuccess();
var piecesOfContent = 3;
var putResults = await Task.WhenAll(
Enumerable.Range(0, piecesOfContent)
.Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess()));
var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray());
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1));
var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 });
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
var cts = new CancellationTokenSource();
// Even though publishing is blocking, this should succeed because we're publishing asynchronously.
await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess();
// Allow for the publishing operation to be registered.
await Task.Delay(TimeSpan.FromSeconds(1));
// Simulate a restart.
await server.ShutdownAsync(context).ShouldBeSuccess();
server.Dispose();
server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario: scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
await server.StartupAsync(context).ShouldBeSuccess();
// Session should have been persisted.
var sessionsAndDatas = server.GetCurrentSessions();
sessionsAndDatas.Length.Should().Be(1);
var serverSession = sessionsAndDatas[0].session;
var data = sessionsAndDatas[0].data;
var operation = new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListwithDeterminism,
StrongFingerprint = strongFingerprint
};
var operations = new[] { operation };
data.Name.Should().Be(clientSession.Name);
data.Pat.Should().Be(pat);
data.Capabilities.Should().Be(Capabilities.All);
data.ImplicitPin.Should().Be(ImplicitPin.None);
data.Pins.Should().BeEquivalentTo(new List <string>());
data.PublishingConfig.Should().BeEquivalentTo(publishingConfig);
data.PendingPublishingOperations.Should().BeEquivalentTo(operations);
var hibernateSession = serverSession as IHibernateCacheSession;
hibernateSession.Should().NotBeNull();
var actualPending = hibernateSession.GetPendingPublishingOperations();
actualPending.Should().BeEquivalentTo(operations);
await server.ShutdownAsync(context).ShouldBeSuccess();
await clientSession.ShutdownAsync(context).ShouldBeSuccess();
}
public void EmptyContentHashListWithDeterminismRoundtrip()
{
TestContentHashListWithDeterminismRoundtrip(new ContentHashListWithDeterminism(
ContentHashList.Random(contentHashCount: 0),
CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow)));
}
protected static IEnumerable <object[]> BuildDeterminismMatrix()
{
for (int i = -1; i < determinisms.Length; i++)
{
int localIndex = Math.Max(0, i);
CacheDeterminism startDeterminismLocal = determinisms[localIndex];
CacheDeterminism startDeterminismRemote = determinisms[Math.Max(localIndex, 1)];
CacheDeterminism endDetermismLocal = startDeterminismLocal.IsDeterministicTool || startDeterminismRemote.IsDeterministicTool ? CacheDeterminism.Tool : CacheDeterminism.ViaCache(RemoteReferenceGuild, CacheDeterminism.NeverExpires);
CacheDeterminism endDeterminismRemote = startDeterminismRemote;
yield return(new object[] { BackingStoreTestClass.Self, BackingStoreTestClass.Self, startDeterminismLocal, startDeterminismRemote, endDetermismLocal, endDeterminismRemote });
}
}
/// <nodoc />
public static CacheDeterminism FromGrpc(this CacheDeterminismData input)
{
return(CacheDeterminism.ViaCache(
new Guid(input.Guid.ToByteArray()),
DateTime.SpecifyKind(new DateTime(input.ExpirationUtc), DateTimeKind.Utc)));
}
