/// <summary>
/// Load value from the filesystem.
/// </summary>
public static Guid Load(IAbsFileSystem fileSystem, AbsolutePath filePath)
{
Guid guid;
try
{
// First try reading the GUID file
guid = Read(fileSystem, filePath);
}
catch (Exception e) when(e is CacheException || e is IOException)
{
// If that fails, we likely need to create a Guid
guid = CacheDeterminism.NewCacheGuid();
try
{
fileSystem.CreateDirectory(filePath.Parent);
// Write the Guid file
fileSystem.WriteAllBytes(filePath, Encoding.UTF8.GetBytes(guid.ToString(SerializationFormat)));
}
catch (Exception ex) when(ex is IOException)
{
// If we failed to write the Guid file we may have just missed getting the guid in the first place,
// so let us try to read it again. This failure we let go all the way out.
guid = Read(fileSystem, filePath);
}
}
return(guid);
}
public void ViaCacheRoundtrip()
{
var guid = CacheDeterminism.NewCacheGuid();
var determinism = CacheDeterminism.ViaCache(guid, DateTime.MaxValue);
Assert.Equal(guid, determinism.EffectiveGuid);
}
public void ContentHashListWithDeterminismProperty()
{
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));
Assert.Equal(contentHashListWithDeterminism, new GetContentHashListResult(contentHashListWithDeterminism).ContentHashListWithDeterminism);
}
/// <nodoc />
public MemCache(CacheId cacheId, bool strictMetadataCasCoupling, bool isauthoritative)
{
m_cacheGuid = CacheDeterminism.NewCacheGuid();
m_cacheId = cacheId;
IsAuthoritative = isauthoritative;
m_strictMetadataCasCoupling = strictMetadataCasCoupling;
}
public void NewCacheGuidCreatesNonSpecialValue()
{
var guid = CacheDeterminism.NewCacheGuid();
Assert.NotEqual(CacheDeterminism.None.EffectiveGuid, guid);
Assert.NotEqual(CacheDeterminism.Tool.EffectiveGuid, guid);
Assert.NotEqual(CacheDeterminism.SinglePhaseNonDeterministic.EffectiveGuid, guid);
}
public void EqualsFalseContentHashListMismatch()
{
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires);
var v1 = new GetContentHashListResult(new ContentHashListWithDeterminism(ContentHashList.Random(), determinism));
var v2 = new GetContentHashListResult(new ContentHashListWithDeterminism(ContentHashList.Random(), determinism));
Assert.False(v1.Equals(v2));
}
public void GetHashCodeSameWhenEqual()
{
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));
var v1 = new GetContentHashListResult(contentHashListWithDeterminism);
var v2 = new GetContentHashListResult(contentHashListWithDeterminism);
Assert.Equal(v1.GetHashCode(), v2.GetHashCode());
}
public void EqualsFalseDeterminismMismatch()
{
var v1 = new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.MaxValue)));
var v2 = new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.MaxValue)));
Assert.False(v1.Equals(v2));
}
public async Task Roundtrip()
{
var fileName = $"{Guid.NewGuid()}.json";
using (var directory = new DisposableDirectory(FileSystem))
{
var sessionId = 42;
var serializedConfig = "Foo";
var pat = Guid.NewGuid().ToString();
var numOperations = 3;
var operations = Enumerable.Range(0, numOperations)
.Select(_ => generateRandomOperation())
.ToList();
var sessionInfo = new HibernatedCacheSessionInfo(sessionId, serializedConfig, pat, operations);
var sessions1 = new HibernatedSessions <HibernatedCacheSessionInfo>(new List <HibernatedCacheSessionInfo> {
sessionInfo
});
await sessions1.WriteAsync(FileSystem, directory.Path, fileName);
FileSystem.HibernatedSessionsExists(directory.Path, fileName).Should().BeTrue();
var fileSize = FileSystem.GetFileSize(directory.Path / fileName);
fileSize.Should().BeGreaterThan(0);
var sessions2 = await FileSystem.ReadHibernatedSessionsAsync <HibernatedCacheSessionInfo>(directory.Path, fileName);
sessions2.Sessions.Count.Should().Be(1);
sessions2.Sessions[0].Id.Should().Be(sessionId);
sessions2.Sessions[0].SerializedSessionConfiguration.Should().Be(serializedConfig);
sessions2.Sessions[0].Pat.Should().Be(pat);
sessions2.Sessions[0].PendingPublishingOperations.Should().BeEquivalentTo(operations);
await FileSystem.DeleteHibernatedSessions(directory.Path, fileName);
PublishingOperation generateRandomOperation()
{
var amountOfHashes = 3;
var hashes = Enumerable.Range(0, amountOfHashes).Select(_ => ContentHash.Random()).ToArray();
var contentHashList = new ContentHashList(hashes);
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddMilliseconds(ThreadSafeRandom.Generator.Next()));
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random());
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
return(new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListWithDeterminism,
StrongFingerprint = strongFingerprint
});
}
}
}
private ICache CreateBackendCache(AbsolutePath rootPath)
{
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(configuration);
return(new OneLevelCache(
() => new FileSystemContentStore(FileSystem, SystemClock.Instance, rootPath, configurationModel),
() => new MemoryMemoizationStore(Logger),
CacheDeterminism.NewCacheGuid()));
}
protected override ICache CreateCache(DisposableDirectory testDirectory)
{
var rootPath = testDirectory.Path;
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(configuration);
return(new OneLevelCache(
() => new FileSystemContentStore(FileSystem, SystemClock.Instance, rootPath, configurationModel),
() => new MemoryMemoizationStore(Logger),
CacheDeterminism.NewCacheGuid()));
}
public void EqualsTrueNotReferenceEqualContentHashList()
{
var contentHashList = ContentHashList.Random();
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires);
var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(contentHashList, determinism);
var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(contentHashList, determinism);
var v1 = new GetContentHashListResult(contentHashListWithDeterminism1);
var v2 = new GetContentHashListResult(contentHashListWithDeterminism2);
Assert.True(v1.Equals(v2));
}
private ICache CreateBlockingPublishingCache(AbsolutePath path)
{
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(configuration);
var contentStore = new FileSystemContentStore(FileSystem, SystemClock.Instance, path, configurationModel);
var localCache = new OneLevelCache(
() => contentStore,
() => new MemoryMemoizationStore(Logger),
CacheDeterminism.NewCacheGuid());
return(new PublishingCacheToContentStore(new PublishingCache <OneLevelCache>(localCache, new BlockingPublishingStore(), Guid.NewGuid())));
}
public Task AddOrGetContentHashListPassThrough()
{
var context = new Context(Logger);
var strongFingerprint = StrongFingerprint.Random();
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
ContentHashList.Random(),
CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));
return(RunMockSessionTestAsync(context, session =>
{
session.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListWithDeterminism, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess();
Assert.True(_mockMemoizationSession.AddOrGetContentHashListAsyncParams.Contains(new Tuple <StrongFingerprint, ContentHashListWithDeterminism>(strongFingerprint, contentHashListWithDeterminism)));
return Task.FromResult(0);
}));
}
public Task AddExpiredDeterminism()
{
var strongFingerprint = StrongFingerprint.Random();
var expirationUtc = DateTime.UtcNow - TimeSpan.FromDays(7);
var guid = CacheDeterminism.NewCacheGuid();
var determinism = CacheDeterminism.ViaCache(guid, expirationUtc);
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), determinism);
return(RunTest((context, contentLocationDatabase) =>
{
contentLocationDatabase.AddOrGetContentHashList(
context, strongFingerprint, contentHashListWithDeterminism).ShouldBeSuccess();
var result = contentLocationDatabase.GetContentHashList(context, strongFingerprint);
Assert.Equal(CacheDeterminism.None.EffectiveGuid, result.ContentHashListWithDeterminism.Determinism.EffectiveGuid);
}));
}
public Task AddExpiredDeterminism()
{
var context = new Context(Logger);
var strongFingerprint = StrongFingerprint.Random();
var expirationUtc = DateTime.UtcNow - TimeSpan.FromDays(7);
var guid = CacheDeterminism.NewCacheGuid();
var determinism = CacheDeterminism.ViaCache(guid, expirationUtc);
var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), determinism);
return(RunTestAsync(context, async session =>
{
await session.AddOrGetContentHashListAsync(
context, strongFingerprint, contentHashListWithDeterminism, Token).ShouldBeSuccess();
var result = await session.GetContentHashListAsync(context, strongFingerprint, Token);
Assert.Equal(CacheDeterminism.None.EffectiveGuid, result.ContentHashListWithDeterminism.Determinism.EffectiveGuid);
}));
}
public void ContentHashListWithDeterminismToRedisValue()
{
var chl = ContentHashList.Random();
var d = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow);
var chlwd = new ContentHashListWithDeterminism(chl, d);
var redisValue = _serializer.ToRedisValue(chlwd);
var expected =
HexUtilities.BytesToHex(d.Serialize()) +
RedisSerializer.RedisValueSeparator +
RedisSerializer.RedisValueExistsSemaphore +
RedisSerializer.RedisValueSeparator +
chl.Serialize() +
RedisSerializer.RedisValueSeparator +
RedisSerializer.RedisValueSeparator;
Assert.Equal(expected, redisValue);
}
public async Task SinglePhaseNonDeterministic()
{
const string TestName = nameof(SinglePhaseNonDeterministic);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
PipDefinition[] pips =
{
new PipDefinition("PipA", determinism: CacheDeterminism.SinglePhaseNonDeterministic),
new PipDefinition("PipB", determinism: CacheDeterminism.SinglePhaseNonDeterministic)
};
FullCacheRecord[] records = (await pips.BuildAsync(session)).ToArray();
XAssert.AreEqual(2, records.Length);
// Now, we should be able to redo those adds but swap the CasEntries in the records
// and get them replaced
for (int i = 0; i < 2; i++)
{
// Make sure we have our own strong fingerprint (no cheating by the cache for this one)
var strong = records[i].StrongFingerprint;
strong = new StrongFingerprint(strong.WeakFingerprint, strong.CasElement, strong.HashElement, "testing");
// Validate that the GetCacheEntry produces what we expect
var entries = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(records[i].CasEntries, entries);
// Validate that the other record I am going to do is different
var other = records[1 - i].CasEntries;
XAssert.AreNotEqual(entries, other, "Other record must be different!");
var newRecord = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, other).SuccessAsync();
XAssert.IsNull(newRecord.Record, "Should have replaced");
entries = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(other, entries);
XAssert.AreNotEqual(records[i].CasEntries, entries);
// Try to replace the existing SinglePhaseNonDeterministic case with a recird that is deterministic with a cache
var fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)));
XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with cache deterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType());
// Try to replace the existing SinglePhaseNonDeterministic case with a recird that is deterministic tool
fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.Tool));
XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with tool deterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType());
// Try to replace the existing SinglePhaseNonDeterministic case with a recird that is not deterministic
fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.None));
XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with non-deterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType());
}
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
private Task RunMockReadOnlySessionTestAsync(Context context, Func <IReadOnlyCacheSession, Task> funcAsync)
{
return(RunReadOnlySessionTestAsync(
context,
funcAsync,
testDirectory => new OneLevelCache(() => _mockContentStore, () => _mockMemoizationStore, CacheDeterminism.NewCacheGuid())));
}
public async Task CacheSessionDataIsHibernated()
{
using var testDirectory = new DisposableDirectory(FileSystem);
var cacheDirectory = testDirectory.Path / "Service";
var cacheName = "theCache";
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
[cacheName] = cacheDirectory / "Root"
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem)
{
GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
};
var scenario = "Default";
var server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
var context = new OperationContext(new Context(Logger));
await server.StartupAsync(context).ShouldBeSuccess();
var pat = Guid.NewGuid().ToString();
var publishingConfig = new PublishingConfigDummy
{
PublishAsynchronously = true,
};
using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario);
await clientCache.StartupAsync(context).ShouldBeSuccess();
var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session;
await clientSession.StartupAsync(context).ShouldBeSuccess();
var piecesOfContent = 3;
var putResults = await Task.WhenAll(
Enumerable.Range(0, piecesOfContent)
.Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess()));
var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray());
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1));
var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 });
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
var cts = new CancellationTokenSource();
// Even though publishing is blocking, this should succeed because we're publishing asynchronously.
await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess();
// Allow for the publishing operation to be registered.
await Task.Delay(TimeSpan.FromSeconds(1));
// Simulate a restart.
await server.ShutdownAsync(context).ShouldBeSuccess();
server.Dispose();
server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario: scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
await server.StartupAsync(context).ShouldBeSuccess();
// Session should have been persisted.
var sessionsAndDatas = server.GetCurrentSessions();
sessionsAndDatas.Length.Should().Be(1);
var serverSession = sessionsAndDatas[0].session;
var data = sessionsAndDatas[0].data;
var operation = new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListwithDeterminism,
StrongFingerprint = strongFingerprint
};
var operations = new[] { operation };
data.Name.Should().Be(clientSession.Name);
data.Pat.Should().Be(pat);
data.Capabilities.Should().Be(Capabilities.All);
data.ImplicitPin.Should().Be(ImplicitPin.None);
data.Pins.Should().BeEquivalentTo(new List <string>());
data.PublishingConfig.Should().BeEquivalentTo(publishingConfig);
data.PendingPublishingOperations.Should().BeEquivalentTo(operations);
var hibernateSession = serverSession as IHibernateCacheSession;
hibernateSession.Should().NotBeNull();
var actualPending = hibernateSession.GetPendingPublishingOperations();
actualPending.Should().BeEquivalentTo(operations);
await server.ShutdownAsync(context).ShouldBeSuccess();
await clientSession.ShutdownAsync(context).ShouldBeSuccess();
}
public void EmptyContentHashListWithDeterminismRoundtrip()
{
TestContentHashListWithDeterminismRoundtrip(new ContentHashListWithDeterminism(
ContentHashList.Random(contentHashCount: 0),
CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow)));
}