public async Task <Possible <string, Failure> > ProduceFileAsync( CasHash hash, string filename, FileState fileState, UrgencyHint urgencyHint, Guid activityId) { Contract.Requires(!IsClosed); Contract.Requires(filename != null); using (var counter = m_counters.ProduceFileCounter()) { using (var eventing = new ProduceFileActivity(BasicFilesystemCache.EventSource, activityId, this)) { eventing.Start(hash, filename, fileState, urgencyHint); try { FileUtilities.CreateDirectory(Path.GetDirectoryName(filename)); await m_cache.CopyFromCasAsync(hash, filename); counter.FileSize(new FileInfo(filename).Length); } catch (Exception e) { counter.Fail(); return(eventing.StopFailure(new ProduceFileFailure(CacheId, hash, filename, e))); } return(eventing.Returns(filename)); } } }
public async Task <Possible <string, Failure> > ProduceFileAsync( CasHash hash, string filename, FileState fileState, UrgencyHint urgencyHint, Guid activityId) { Possible <Stream, Failure> casStream = await GetStreamAsync(hash, urgencyHint, activityId); if (!casStream.Succeeded) { return(casStream.Failure); } try { Directory.CreateDirectory(Path.GetDirectoryName(filename)); using (FileStream fs = new FileStream(filename, FileMode.CreateNew, FileAccess.Write)) { await casStream.Result.CopyToAsync(fs); } } catch (Exception e) { return(new ProduceFileFailure(CacheId, hash, filename, e)); } return(filename); }
/// <summary> /// Failure of an observed input list filter /// </summary> /// <param name="cacheId">Cache ID where failure happened</param> /// <param name="weak">Weak fingerprint component of the strong fingerprint</param> /// <param name="casElement">The CasElement component of the strong fingerprint</param> /// <param name="hashElement">The HashElement component of the strong fingerprint</param> /// <param name="message">Details about the failure</param> public InputListFilterFailure(string cacheId, WeakFingerprintHash weak, CasHash casElement, Hash hashElement, string message) { Contract.Requires(message != null); m_strongFingerprint = new StrongFingerprint(weak, casElement, hashElement, cacheId); m_message = message; }
public async Task <Possible <Stream, Failure> > GetStreamAsync(CasHash hash, UrgencyHint urgencyHint, Guid activityId) { Contract.Requires(!IsClosed); using (var counter = m_counters.GetStreamCounter()) { using (var eventing = new GetStreamActivity(BasicFilesystemCache.EventSource, activityId, this)) { eventing.Start(hash, urgencyHint); if (!m_pinnedToCas.ContainsKey(hash)) { counter.Miss(); return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, hash))); } try { Stream result = CasHash.NoItem.Equals(hash) ? Stream.Null : await m_cache.ContendedOpenStreamAsync(m_cache.ToPath(hash), FileMode.Open, FileAccess.Read, FileShare.Read, useAsync : true, handlePendingDelete : true); counter.FileSize(result.Length); return(eventing.Returns(result)); } catch (Exception e) { counter.Fail(); return(eventing.StopFailure(new ProduceStreamFailure(CacheId, hash, e))); } } } }
public void CasHashToMemoization() { CasHash hash = RandomHelpers.CreateRandomCasHash(); ContentHash contentHash = hash.ToMemoization(); Assert.Equal(hash.BaseHash.RawHash.ToByteArray(), contentHash.ToHashByteArray()); }
/// <summary> /// Create a fingerprint from a content hash. /// </summary> public static Hash ToFingerprint(this CasHash casHash) { var hex = casHash.ToString(); var fingerprint = FingerprintUtilities.Hash(hex); return(new Hash(fingerprint)); }
/// <summary> /// Gets the file corresponding to the given CasHash and checks /// to see if the file contents hash to the same CasHash value /// </summary> /// <param name="originalCasHash">CasHash value to check</param> /// <param name="errors">Where any cache errors found get stored</param> private async Task RehashContentsAsync(CasHash originalCasHash, ConcurrentDictionary <CacheError, int> errors) { if (originalCasHash.Equals(CasHash.NoItem)) { // No need to rehash the NoItem cas hash return; } Possible <StreamWithLength, Failure> possibleStream = await m_readOnlySession.GetStreamAsync(originalCasHash); if (!possibleStream.Succeeded) { errors.TryAdd(new CacheError(CacheErrorType.CasHashError, "CasHash " + originalCasHash + " not found in CAS"), 0); return; } using (StreamWithLength stream = possibleStream.Result) { ContentHash contentHash = await ContentHashingUtilities.HashContentStreamAsync(stream); Hash newHash = new Hash(contentHash); CasHash newCasHash = new CasHash(newHash); if (!originalCasHash.Equals(newCasHash)) { errors.TryAdd(new CacheError(CacheErrorType.CasHashError, "The data of CasHash " + originalCasHash + " has been altered in the CAS"), 0); } } }
// Note that this cache is able to remediate even in read-only sessions... public Task <Possible <ValidateContentStatus, Failure> > ValidateContentAsync(CasHash hash, UrgencyHint urgencyHint, Guid activityId) { return(Task.Run <Possible <ValidateContentStatus, Failure> >(() => { if (CasHash.NoItem.Equals(hash)) { return ValidateContentStatus.Ok; } byte[] fileBytes; if (!Cache.CasStorage.TryGetValue(hash, out fileBytes)) { return ValidateContentStatus.Remediated; } CasHash contentHash = new CasHash(ContentHashingUtilities.HashBytes(fileBytes).ToHashByteArray()); if (contentHash == hash) { return ValidateContentStatus.Ok; } // It is a corrupted data entry - remove it Cache.CasStorage.TryRemove(hash, out fileBytes); // Also remove it from the set of pinned content (if it was there) int junk; m_pinnedToCas.TryRemove(hash, out junk); return ValidateContentStatus.Remediated; })); }
/// <summary> /// A fake build with a set number of outputs and /// an OutputList that matches the outputs and is /// used as the "input list" entry for the tests. /// </summary> /// <param name="prefix">The prefix text used to make the build unique</param> /// <param name="outputCount">Number of "output files"</param> /// <param name="forceUniqueOutputs">Determines if the outputs will be unique or not. Unique outputs cannot be verfied by CheckContentsAsync</param> public FakeBuild(string prefix, int outputCount, int startIndex = 0, bool forceUniqueOutputs = false) { Contract.Requires(prefix != null); Contract.Requires(outputCount > 0); StringBuilder inputList = new StringBuilder(); Guid unique = Guid.NewGuid(); Outputs = new StreamWithLength[outputCount]; OutputHashes = new Hash[outputCount]; for (int i = 0; i < outputCount; i++) { string contents = I($"{prefix}:{i + startIndex}"); inputList.Append(contents).Append("\n"); if (forceUniqueOutputs) { contents += ":" + unique.ToString(); } Outputs[i] = contents.AsStream(); OutputHashes[i] = Outputs[i].Stream.AsHash(); } OutputList = inputList.AsStream(); OutputListHash = new CasHash(OutputList.AsHash()); }
public void CasHashRoundTrip() { CasHash hash = RandomHelpers.CreateRandomCasHash(); CasHash roundTrip = hash.ToMemoization().FromMemoization(); Assert.Equal(hash, roundTrip); }
public void ContentHashFromMemoization() { ContentHash contentHash = ContentHashingUtilities.CreateRandom(); CasHash hash = contentHash.FromMemoization(); Assert.Equal(contentHash.ToHashByteArray(), hash.BaseHash.RawHash.ToByteArray()); }
public Task <Possible <string, Failure> > ProduceFileAsync( CasHash hash, string filename, FileState fileState, UrgencyHint urgencyHint, Guid activityId) { var callback = ProduceFileAsyncCallback; if (callback != null) { return(callback( hash, filename, fileState, urgencyHint, activityId, m_realSession)); } else { return(m_realSession.ProduceFileAsync( hash, filename, fileState, urgencyHint, activityId)); } }
/// <summary> /// Gets a file stream from the CAS either directly or by materlializing a file in the temp path and then opening it for read. /// </summary> /// <param name="hash">Hash for CAS entry</param> /// <param name="method">Method used to access CAS</param> /// <param name="session">Cache session</param> /// <returns>A stream pointing to the file contents, or a failure.</returns> private static async Task <Possible <Stream, Failure> > GetStreamAsync(CasHash hash, CasAccessMethod method, ICacheReadOnlySession session) { switch (method) { case CasAccessMethod.Stream: return(await session.GetStreamAsync(hash)); case CasAccessMethod.FileSystem: string filePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); string placedFilePath = await session.ProduceFileAsync(hash, filePath, FileState.ReadOnly).SuccessAsync(); XAssert.AreEqual(filePath, placedFilePath); FileStream fs = new FileStream(placedFilePath, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read); File.Delete(placedFilePath); return(fs); default: throw new NotImplementedException(); } }
public async Task <Possible <string, Failure> > ProduceFileAsync(CasHash hash, string filename, FileState fileState, UrgencyHint urgencyHint, Guid activityId) { using (var eventing = new ProduceFileActivity(CompositingCache.EventSource, activityId, this)) { eventing.Start(hash, filename, fileState, urgencyHint); return(eventing.Returns(await m_casSession.ProduceFileAsync(hash, filename, fileState, urgencyHint, activityId))); } }
public async Task <Possible <Stream, Failure> > GetStreamAsync(CasHash hash, UrgencyHint urgencyHint, Guid activityId) { using (var eventing = new GetStreamActivity(CompositingCache.EventSource, activityId, this)) { eventing.Start(hash, urgencyHint); return(eventing.Returns(await m_casSession.GetStreamAsync(hash, urgencyHint, eventing.Id))); } }
private async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddToCache(ICacheSession session, params string[] thePaths) { WeakFingerprintHash weak = FakeStrongFingerprint.CreateWeak(session.CacheId); CasHash casHash = await AddPathSet(session, thePaths); Hash simpleHash = FakeStrongFingerprint.CreateHash(session.CacheSessionId); return(await session.AddOrGetAsync(weak, casHash, simpleHash, CasEntries.FromCasHashes(casHash))); }
/// <summary> /// Create the failure, including the CasHash and the target filename that failed /// </summary> /// <param name="cacheId">The cacheId where the failure happened</param> /// <param name="casHash">The CasHash that failed</param> /// <param name="filename">The filename that failed</param> public FileAlreadyExistsFailure(string cacheId, CasHash casHash, string filename) { Contract.Requires(cacheId != null); Contract.Requires(filename != null); m_cacheId = cacheId; m_casHash = casHash; m_filename = filename; }
public async Task CorruptionRecovery() { const string TestName = nameof(CorruptionRecovery); string testCacheId = MakeCacheId(TestName); ICache cache = await CreateCacheAsync(testCacheId); string testSessionId = "Session1-" + testCacheId; ICacheSession session = await CreateSessionAsync(cache, testSessionId); // Use the testname to generate a CAS items. CasHash item = (await session.AddToCasAsync(TestName.AsStream())).Success(); // Verify that we can read the content after it was added in // this session since it was pinned using (Stream stream = (await session.GetStreamAsync(item)).Success()) { XAssert.AreEqual(TestName, stream.AsString(), "Failed to read back matching content from cache"); } ValidateContentStatus goodStatus = (await session.ValidateContentAsync(item)).Success(); // We can have implemented ValidateContent and not have a way to test corruption but // we must have implemented ValidateCotent if we have a way to test corruption if (CanTestCorruption || (goodStatus != ValidateContentStatus.NotSupported)) { // We should have returned Ok since the content was not corrupted XAssert.AreEqual(ValidateContentStatus.Ok, goodStatus, "Content should have matched in hash at this point!"); // NoItem should always be valid XAssert.AreEqual(ValidateContentStatus.Ok, (await session.ValidateContentAsync(CasHash.NoItem)).Success(), "NoItem should always be valid!"); // Now, only if we can test corruption (which requires that we can corrupt an item) // do we go down this next path if (CanTestCorruption) { await CorruptCasEntry(cache, item); using (Stream stream = (await session.GetStreamAsync(item)).Success()) { XAssert.AreNotEqual(TestName, stream.AsString(), "Failed to corrupt CAS entry!"); } ValidateContentStatus corruptedStatus = (await session.ValidateContentAsync(item)).Success(); // At this point, caches can do a number of possible things // They can not return OK or NotImplemented (since we already checked that earlier) XAssert.AreNotEqual(ValidateContentStatus.Ok, corruptedStatus, "The item was corrupted - something should have happened"); XAssert.AreNotEqual(ValidateContentStatus.NotSupported, corruptedStatus, "It was implemented a moment earlier"); } } await CloseSessionAsync(session, testSessionId); await ShutdownCacheAsync(cache, testCacheId); }
/// <summary> /// Attempts to pin the specified CasHash. Returns true if the /// pinning succeeds. /// </summary> /// <param name="casHash">CasHash value to attempt to pin</param> private async Task <bool> AttemptToPinAsync(CasHash casHash) { if (casHash.Equals(CasHash.NoItem)) { return(true); } Possible <string, Failure> pinAttempt = await m_readOnlySession.PinToCasAsync(casHash); return(pinAttempt.Succeeded); }
/// <summary> /// Check the fake build via the cache given /// </summary> /// <param name="cache">The cache to read from (uses a read-only session)</param> /// <param name="index">The "index" CasHash</param> /// <param name="entries">The CasHash entries that should match the index</param> /// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param> /// <returns>Success if all worked</returns> /// <remarks> /// This is tied to the FakeBuild where the set of results is /// made as the index file which we use in the strong fingerprint /// and basically describes the set of entries that should be in the CasEntries /// </remarks> public static async Task CheckContentsAsync(ICache cache, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream) { Contract.Requires(cache != null); Contract.Requires(entries.IsValid); ICacheReadOnlySession session = await cache.CreateReadOnlySessionAsync().SuccessAsync(); await CheckContentsAsync(session, index, entries, accessMethod); await session.CloseAsync().SuccessAsync(); }
/// <summary> /// Checks for any errors with the cas element /// and with each of the cas entries /// </summary> /// <param name="casElement">cas element of a strong fingerprint to check</param> /// <param name="casEntries">cas entries to check</param> /// <param name="errors">Where any cache errors found get stored</param> private async Task CheckCas(CasHash casElement, CasEntries casEntries, ConcurrentDictionary <CacheError, int> errors) { // Check CasElement await CheckCasHashAsync(casElement, errors); for (int i = 0; i < casEntries.Count(); i++) { // Check each CasHash CasHash casHash = casEntries[i]; await CheckCasHashAsync(casHash, errors); } }
private async Task <long> PinAndGetStreamSize(ICacheReadOnlySession session, CasHash hash) { long result; await session.PinToCasAsync(hash, CancellationToken.None).SuccessAsync(); using (var stream = await session.GetStreamAsync(hash).SuccessAsync()) { result = stream.Length; } return(result); }
public Task <Possible <StreamWithLength, Failure> > GetStreamAsync(CasHash hash, UrgencyHint urgencyHint, Guid activityId) { var callback = GetStreamAsyncCallback; if (callback != null) { return(callback(hash, urgencyHint, activityId, m_realSession)); } else { return(m_realSession.GetStreamAsync(hash, urgencyHint, activityId)); } }
public Task <Possible <string, Failure> > PinToCasAsync(CasHash hash, UrgencyHint urgencyHint, Guid activityId) { var callback = PinToCasAsyncCallback; if (callback != null) { return(callback(hash, urgencyHint, activityId, m_realSession)); } else { return(m_realSession.PinToCasAsync(hash, urgencyHint, activityId)); } }
/// <summary> /// Start of the activity /// </summary> public void Start(CasHash hash, UrgencyHint urgencyHint) { Start(); if (TraceMethodArgumentsEnabled()) { Write( ParameterOptions, new { CasHash = hash, UrgencyHint = urgencyHint, }); } }
public void PinResultMissFromMemoization() { PinResult pinResult = PinResult.ContentNotFound; CasHash hash = RandomHelpers.CreateRandomCasHash(); Possible <string, Failure> maybe = pinResult.FromMemoization(hash, CacheId); Assert.False(maybe.Succeeded); NoCasEntryFailure failure = maybe.Failure as NoCasEntryFailure; Assert.False(failure == null); string failureMessage = failure.Describe(); Assert.Contains(hash.ToString(), failureMessage); Assert.Contains(CacheId, failureMessage); }
public async Task <Possible <string, Failure> > PinToCasAsync(CasHash hash, UrgencyHint urgencyHint, Guid activityId) { using (var eventing = new PinToCasActivity(CompositingCache.EventSource, activityId, this)) { eventing.Start(hash, urgencyHint); var result = await m_casSession.PinToCasAsync(hash, urgencyHint, eventing.Id); if (result.Succeeded) { PinnedToCas.TryAdd(hash, 0); } return(eventing.Returns(result)); } }
protected override Task CorruptCasEntry(ICache cache, CasHash hash) { // We use this as a Task.Run() just to help prove the test structure // Other caches are likely to need async behavior so we needed to support // that. No return result. This must fail if it can not work. return(Task.Run(() => { BasicFilesystemCache myCache = cache as BasicFilesystemCache; XAssert.IsNotNull(myCache, "Invalid cache passed to TestBasicFilesyste CorruptCasEntry test method"); using (var f = File.AppendText(myCache.ToPath(hash))) { f.Write("!Corrupted!"); } })); }
/// <summary> /// Checks the specified CasHash for potential problems by first /// attempting to pin it and then optionally rehashes the file /// contents. /// </summary> /// <param name="casHash">CasHash value to check</param> /// <param name="errors">Where any cache errors found get stored</param> private Task CheckCasHashAsync(CasHash casHash, ConcurrentDictionary <CacheError, int> errors) { return(AllCasHashes.GetOrAdd(casHash, async(cH) => { if (!(await AttemptToPinAsync(casHash))) { // CasHash failed to be pinned errors.TryAdd(new CacheError(CacheErrorType.CasHashError, "CasHash " + casHash + " not found in CAS"), 0); return 0; } if (m_checkCASContent) { await RehashContentsAsync(casHash, errors); } return 0; })); }
/// <summary> /// Check the fake build via the session given /// </summary> /// <param name="session">Session to use for the check</param> /// <param name="index">The "index" CasHash</param> /// <param name="entries">The CasHash entries that should match the index</param> /// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param> /// <returns>An Async Task</returns> public static async Task CheckContentsAsync(ICacheReadOnlySession session, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream) { string cacheId = await session.PinToCasAsync(index, CancellationToken.None).SuccessAsync("Cannot pin entry {0} to cache {1}", index.ToString(), session.CacheId); string[] expected = (await GetStreamAsync(index, accessMethod, session)).Success().Stream.AsString().Split(s_splitLines, StringSplitOptions.RemoveEmptyEntries); XAssert.AreEqual(expected.Length, entries.Count, "Counts did not match from cache {0}: {1} != {2}", cacheId, expected.Length, entries.Count); for (int i = 0; i < expected.Length; i++) { string casCacheId = await session.PinToCasAsync(entries[i], CancellationToken.None).SuccessAsync(); string entry = (await GetStreamAsync(entries[i], accessMethod, session)).Success().Stream.AsString(); XAssert.AreEqual(expected[i], entry, "CasEntry {0} mismatch from cache {1}: [{2}] != [{3}]", i, casCacheId, expected[i], entry); } }