/// <summary> /// Load value from the filesystem. /// </summary> public static Guid Load(IAbsFileSystem fileSystem, AbsolutePath filePath) { Guid guid; try { // First try reading the GUID file guid = Read(fileSystem, filePath); } catch (Exception e) when(e is CacheException || e is IOException) { // If that fails, we likely need to create a Guid guid = CacheDeterminism.NewCacheGuid(); try { fileSystem.CreateDirectory(filePath.Parent); // Write the Guid file fileSystem.WriteAllBytes(filePath, Encoding.UTF8.GetBytes(guid.ToString(SerializationFormat))); } catch (Exception ex) when(ex is IOException) { // If we failed to write the Guid file we may have just missed getting the guid in the first place, // so let us try to read it again. This failure we let go all the way out. guid = Read(fileSystem, filePath); } } return(guid); }
public void ViaCacheRoundtrip() { var guid = CacheDeterminism.NewCacheGuid(); var determinism = CacheDeterminism.ViaCache(guid, DateTime.MaxValue); Assert.Equal(guid, determinism.EffectiveGuid); }
public void ContentHashListWithDeterminismProperty() { var contentHashListWithDeterminism = new ContentHashListWithDeterminism( ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)); Assert.Equal(contentHashListWithDeterminism, new GetContentHashListResult(contentHashListWithDeterminism).ContentHashListWithDeterminism); }
/// <nodoc /> public MemCache(CacheId cacheId, bool strictMetadataCasCoupling, bool isauthoritative) { m_cacheGuid = CacheDeterminism.NewCacheGuid(); m_cacheId = cacheId; IsAuthoritative = isauthoritative; m_strictMetadataCasCoupling = strictMetadataCasCoupling; }
public void NewCacheGuidCreatesNonSpecialValue() { var guid = CacheDeterminism.NewCacheGuid(); Assert.NotEqual(CacheDeterminism.None.EffectiveGuid, guid); Assert.NotEqual(CacheDeterminism.Tool.EffectiveGuid, guid); Assert.NotEqual(CacheDeterminism.SinglePhaseNonDeterministic.EffectiveGuid, guid); }
public void EqualsFalseContentHashListMismatch() { var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires); var v1 = new GetContentHashListResult(new ContentHashListWithDeterminism(ContentHashList.Random(), determinism)); var v2 = new GetContentHashListResult(new ContentHashListWithDeterminism(ContentHashList.Random(), determinism)); Assert.False(v1.Equals(v2)); }
public void GetHashCodeSameWhenEqual() { var contentHashListWithDeterminism = new ContentHashListWithDeterminism( ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)); var v1 = new GetContentHashListResult(contentHashListWithDeterminism); var v2 = new GetContentHashListResult(contentHashListWithDeterminism); Assert.Equal(v1.GetHashCode(), v2.GetHashCode()); }
public void EqualsFalseDeterminismMismatch() { var v1 = new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(null, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.MaxValue))); var v2 = new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(null, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.MaxValue))); Assert.False(v1.Equals(v2)); }
public async Task Roundtrip() { var fileName = $"{Guid.NewGuid()}.json"; using (var directory = new DisposableDirectory(FileSystem)) { var sessionId = 42; var serializedConfig = "Foo"; var pat = Guid.NewGuid().ToString(); var numOperations = 3; var operations = Enumerable.Range(0, numOperations) .Select(_ => generateRandomOperation()) .ToList(); var sessionInfo = new HibernatedCacheSessionInfo(sessionId, serializedConfig, pat, operations); var sessions1 = new HibernatedSessions <HibernatedCacheSessionInfo>(new List <HibernatedCacheSessionInfo> { sessionInfo }); await sessions1.WriteAsync(FileSystem, directory.Path, fileName); FileSystem.HibernatedSessionsExists(directory.Path, fileName).Should().BeTrue(); var fileSize = FileSystem.GetFileSize(directory.Path / fileName); fileSize.Should().BeGreaterThan(0); var sessions2 = await FileSystem.ReadHibernatedSessionsAsync <HibernatedCacheSessionInfo>(directory.Path, fileName); sessions2.Sessions.Count.Should().Be(1); sessions2.Sessions[0].Id.Should().Be(sessionId); sessions2.Sessions[0].SerializedSessionConfiguration.Should().Be(serializedConfig); sessions2.Sessions[0].Pat.Should().Be(pat); sessions2.Sessions[0].PendingPublishingOperations.Should().BeEquivalentTo(operations); await FileSystem.DeleteHibernatedSessions(directory.Path, fileName); PublishingOperation generateRandomOperation() { var amountOfHashes = 3; var hashes = Enumerable.Range(0, amountOfHashes).Select(_ => ContentHash.Random()).ToArray(); var contentHashList = new ContentHashList(hashes); var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddMilliseconds(ThreadSafeRandom.Generator.Next())); var contentHashListWithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism); var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray()); var selector = new Selector(ContentHash.Random()); var strongFingerprint = new StrongFingerprint(fingerprint, selector); return(new PublishingOperation { ContentHashListWithDeterminism = contentHashListWithDeterminism, StrongFingerprint = strongFingerprint }); } } }
private ICache CreateBackendCache(AbsolutePath rootPath) { var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); var configurationModel = new ConfigurationModel(configuration); return(new OneLevelCache( () => new FileSystemContentStore(FileSystem, SystemClock.Instance, rootPath, configurationModel), () => new MemoryMemoizationStore(Logger), CacheDeterminism.NewCacheGuid())); }
protected override ICache CreateCache(DisposableDirectory testDirectory) { var rootPath = testDirectory.Path; var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); var configurationModel = new ConfigurationModel(configuration); return(new OneLevelCache( () => new FileSystemContentStore(FileSystem, SystemClock.Instance, rootPath, configurationModel), () => new MemoryMemoizationStore(Logger), CacheDeterminism.NewCacheGuid())); }
public void EqualsTrueNotReferenceEqualContentHashList() { var contentHashList = ContentHashList.Random(); var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires); var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(contentHashList, determinism); var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(contentHashList, determinism); var v1 = new GetContentHashListResult(contentHashListWithDeterminism1); var v2 = new GetContentHashListResult(contentHashListWithDeterminism2); Assert.True(v1.Equals(v2)); }
private ICache CreateBlockingPublishingCache(AbsolutePath path) { var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); var configurationModel = new ConfigurationModel(configuration); var contentStore = new FileSystemContentStore(FileSystem, SystemClock.Instance, path, configurationModel); var localCache = new OneLevelCache( () => contentStore, () => new MemoryMemoizationStore(Logger), CacheDeterminism.NewCacheGuid()); return(new PublishingCacheToContentStore(new PublishingCache <OneLevelCache>(localCache, new BlockingPublishingStore(), Guid.NewGuid()))); }
public Task AddOrGetContentHashListPassThrough() { var context = new Context(Logger); var strongFingerprint = StrongFingerprint.Random(); var contentHashListWithDeterminism = new ContentHashListWithDeterminism( ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires)); return(RunMockSessionTestAsync(context, session => { session.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListWithDeterminism, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess(); Assert.True(_mockMemoizationSession.AddOrGetContentHashListAsyncParams.Contains(new Tuple <StrongFingerprint, ContentHashListWithDeterminism>(strongFingerprint, contentHashListWithDeterminism))); return Task.FromResult(0); })); }
public Task AddExpiredDeterminism() { var strongFingerprint = StrongFingerprint.Random(); var expirationUtc = DateTime.UtcNow - TimeSpan.FromDays(7); var guid = CacheDeterminism.NewCacheGuid(); var determinism = CacheDeterminism.ViaCache(guid, expirationUtc); var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), determinism); return(RunTest((context, contentLocationDatabase) => { contentLocationDatabase.AddOrGetContentHashList( context, strongFingerprint, contentHashListWithDeterminism).ShouldBeSuccess(); var result = contentLocationDatabase.GetContentHashList(context, strongFingerprint); Assert.Equal(CacheDeterminism.None.EffectiveGuid, result.ContentHashListWithDeterminism.Determinism.EffectiveGuid); })); }
public Task AddExpiredDeterminism() { var context = new Context(Logger); var strongFingerprint = StrongFingerprint.Random(); var expirationUtc = DateTime.UtcNow - TimeSpan.FromDays(7); var guid = CacheDeterminism.NewCacheGuid(); var determinism = CacheDeterminism.ViaCache(guid, expirationUtc); var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), determinism); return(RunTestAsync(context, async session => { await session.AddOrGetContentHashListAsync( context, strongFingerprint, contentHashListWithDeterminism, Token).ShouldBeSuccess(); var result = await session.GetContentHashListAsync(context, strongFingerprint, Token); Assert.Equal(CacheDeterminism.None.EffectiveGuid, result.ContentHashListWithDeterminism.Determinism.EffectiveGuid); })); }
public void ContentHashListWithDeterminismToRedisValue() { var chl = ContentHashList.Random(); var d = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow); var chlwd = new ContentHashListWithDeterminism(chl, d); var redisValue = _serializer.ToRedisValue(chlwd); var expected = HexUtilities.BytesToHex(d.Serialize()) + RedisSerializer.RedisValueSeparator + RedisSerializer.RedisValueExistsSemaphore + RedisSerializer.RedisValueSeparator + chl.Serialize() + RedisSerializer.RedisValueSeparator + RedisSerializer.RedisValueSeparator; Assert.Equal(expected, redisValue); }
public async Task SinglePhaseNonDeterministic() { const string TestName = nameof(SinglePhaseNonDeterministic); string testCacheId = MakeCacheId(TestName); ICache cache = await CreateCacheAsync(testCacheId); // Now for the session (which we base on the cache ID) string testSessionId = "Session1-" + testCacheId; ICacheSession session = await CreateSessionAsync(cache, testSessionId); PipDefinition[] pips = { new PipDefinition("PipA", determinism: CacheDeterminism.SinglePhaseNonDeterministic), new PipDefinition("PipB", determinism: CacheDeterminism.SinglePhaseNonDeterministic) }; FullCacheRecord[] records = (await pips.BuildAsync(session)).ToArray(); XAssert.AreEqual(2, records.Length); // Now, we should be able to redo those adds but swap the CasEntries in the records // and get them replaced for (int i = 0; i < 2; i++) { // Make sure we have our own strong fingerprint (no cheating by the cache for this one) var strong = records[i].StrongFingerprint; strong = new StrongFingerprint(strong.WeakFingerprint, strong.CasElement, strong.HashElement, "testing"); // Validate that the GetCacheEntry produces what we expect var entries = await session.GetCacheEntryAsync(strong).SuccessAsync(); XAssert.AreEqual(records[i].CasEntries, entries); // Validate that the other record I am going to do is different var other = records[1 - i].CasEntries; XAssert.AreNotEqual(entries, other, "Other record must be different!"); var newRecord = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, other).SuccessAsync(); XAssert.IsNull(newRecord.Record, "Should have replaced"); entries = await session.GetCacheEntryAsync(strong).SuccessAsync(); XAssert.AreEqual(other, entries); XAssert.AreNotEqual(records[i].CasEntries, entries); // Try to replace the existing SinglePhaseNonDeterministic case with a recird that is deterministic with a cache var fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires))); XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with cache deterministic"); XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType()); // Try to replace the existing SinglePhaseNonDeterministic case with a recird that is deterministic tool fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.Tool)); XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with tool deterministic"); XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType()); // Try to replace the existing SinglePhaseNonDeterministic case with a recird that is not deterministic fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(records[i].CasEntries, CacheDeterminism.None)); XAssert.IsFalse(fail.Succeeded, "Should have failed trying to replace SinglePhaseNonDeterministic with non-deterministic"); XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType()); } await CloseSessionAsync(session, testSessionId); await ShutdownCacheAsync(cache, testCacheId); }
private Task RunMockReadOnlySessionTestAsync(Context context, Func <IReadOnlyCacheSession, Task> funcAsync) { return(RunReadOnlySessionTestAsync( context, funcAsync, testDirectory => new OneLevelCache(() => _mockContentStore, () => _mockMemoizationStore, CacheDeterminism.NewCacheGuid()))); }
public async Task CacheSessionDataIsHibernated() { using var testDirectory = new DisposableDirectory(FileSystem); var cacheDirectory = testDirectory.Path / "Service"; var cacheName = "theCache"; var namedCacheRoots = new Dictionary <string, AbsolutePath> { [cacheName] = cacheDirectory / "Root" }; var grpcPort = PortExtensions.GetNextAvailablePort(); var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem) { GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it. }; var scenario = "Default"; var server = new LocalCacheServer( FileSystem, TestGlobal.Logger, scenario, cacheFactory: CreateBlockingPublishingCache, serverConfiguration, Capabilities.All); var context = new OperationContext(new Context(Logger)); await server.StartupAsync(context).ShouldBeSuccess(); var pat = Guid.NewGuid().ToString(); var publishingConfig = new PublishingConfigDummy { PublishAsynchronously = true, }; using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario); await clientCache.StartupAsync(context).ShouldBeSuccess(); var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session; await clientSession.StartupAsync(context).ShouldBeSuccess(); var piecesOfContent = 3; var putResults = await Task.WhenAll( Enumerable.Range(0, piecesOfContent) .Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess())); var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray()); var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1)); var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism); var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray()); var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 }); var strongFingerprint = new StrongFingerprint(fingerprint, selector); var cts = new CancellationTokenSource(); // Even though publishing is blocking, this should succeed because we're publishing asynchronously. await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess(); // Allow for the publishing operation to be registered. await Task.Delay(TimeSpan.FromSeconds(1)); // Simulate a restart. await server.ShutdownAsync(context).ShouldBeSuccess(); server.Dispose(); server = new LocalCacheServer( FileSystem, TestGlobal.Logger, scenario: scenario, cacheFactory: CreateBlockingPublishingCache, serverConfiguration, Capabilities.All); await server.StartupAsync(context).ShouldBeSuccess(); // Session should have been persisted. var sessionsAndDatas = server.GetCurrentSessions(); sessionsAndDatas.Length.Should().Be(1); var serverSession = sessionsAndDatas[0].session; var data = sessionsAndDatas[0].data; var operation = new PublishingOperation { ContentHashListWithDeterminism = contentHashListwithDeterminism, StrongFingerprint = strongFingerprint }; var operations = new[] { operation }; data.Name.Should().Be(clientSession.Name); data.Pat.Should().Be(pat); data.Capabilities.Should().Be(Capabilities.All); data.ImplicitPin.Should().Be(ImplicitPin.None); data.Pins.Should().BeEquivalentTo(new List <string>()); data.PublishingConfig.Should().BeEquivalentTo(publishingConfig); data.PendingPublishingOperations.Should().BeEquivalentTo(operations); var hibernateSession = serverSession as IHibernateCacheSession; hibernateSession.Should().NotBeNull(); var actualPending = hibernateSession.GetPendingPublishingOperations(); actualPending.Should().BeEquivalentTo(operations); await server.ShutdownAsync(context).ShouldBeSuccess(); await clientSession.ShutdownAsync(context).ShouldBeSuccess(); }
public void EmptyContentHashListWithDeterminismRoundtrip() { TestContentHashListWithDeterminismRoundtrip(new ContentHashListWithDeterminism( ContentHashList.Random(contentHashCount: 0), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow))); }