private async Task <bool> ValidateCacheConnection()
{
var cacheValidationContentHash = m_cacheValidationContentHash.ToContentHash();
IArtifactContentCache contentCache = m_masterService.Environment.Cache.ArtifactContentCache;
var cacheValidationContentRetrieval = await contentCache.TryLoadAvailableContentAsync(new[]
{
cacheValidationContentHash,
});
if (!cacheValidationContentRetrieval.Succeeded)
{
Logger.Log.DistributionFailedToRetrieveValidationContentFromWorkerCacheWithException(
m_appLoggingContext,
Name,
cacheValidationContentHash.ToHex(),
cacheValidationContentRetrieval.Failure.DescribeIncludingInnerFailures());
return(false);
}
if (!cacheValidationContentRetrieval.Result.AllContentAvailable)
{
Logger.Log.DistributionFailedToRetrieveValidationContentFromWorkerCache(
m_appLoggingContext,
Name,
cacheValidationContentHash.ToHex());
return(false);
}
return(true);
}
/// <summary>
/// Stores the path set content to the backing store
/// </summary>
protected virtual async Task <Possible <Unit> > TryStorePathSetContentAsync(ContentHash pathSetHash, MemoryStream pathSetBuffer)
{
Counters.IncrementCounter(PipCachingCounter.StoredPathSetCount);
Counters.AddToCounter(PipCachingCounter.StoredPathSetSize, pathSetBuffer.Length);
// First check to see if the content is already available in the cache since that's a faster noop path
// than storing an already existing PathSet
var result = await ArtifactContentCache.TryLoadAvailableContentAsync(new[] { pathSetHash });
if (result.Succeeded && result.Result.AllContentAvailable)
{
return(Unit.Void);
}
else
{
return((await ArtifactContentCache.TryStoreAsync(pathSetBuffer, pathSetHash)).WithGenericFailure());
}
}
/// <summary>
/// Stores the path set content to the backing store
/// </summary>
protected virtual async Task <Possible <Unit> > TryStorePathSetContentAsync(ContentHash pathSetHash, MemoryStream pathSetBuffer)
{
Counters.IncrementCounter(PipCachingCounter.StoredPathSetCount);
Counters.AddToCounter(PipCachingCounter.StoredPathSetSize, pathSetBuffer.Length);
if (!EngineEnvironmentSettings.SkipExtraneousPins)
{
// First check to see if the content is already available in the cache since that's a faster noop path
// than storing an already existing PathSet
var result = await ArtifactContentCache.TryLoadAvailableContentAsync(new[] { pathSetHash }, Context.CancellationToken);
if (result.Succeeded && result.Result.AllContentAvailable)
{
return(Unit.Void);
}
}
return((await ArtifactContentCache.TryStoreAsync(pathSetBuffer, pathSetHash, StoreArtifactOptions.CacheEntryContent)).WithGenericFailure());
}
/// <inheritdoc />
public async Task <Possible <ContentAvailabilityBatchResult, Failure> > TryLoadAvailableContentAsync(IReadOnlyList <ContentHash> hashes)
{
using (var hashAvailabilityMapWrapper = m_hashAvailabilityMapPool.GetInstance())
using (var hashListWrapper = m_hashListPool.GetInstance())
{
var hashAvailabilityMap = hashAvailabilityMapWrapper.Instance;
var hashList = hashListWrapper.Instance;
var uniqueUnknownAvailabilityHashes = DeduplicateAndGetUnknownAvailabilityHashes(hashes, hashAvailabilityMap, hashList);
bool allContentAvailable = true;
if (uniqueUnknownAvailabilityHashes.Count != 0)
{
// Only query inner cache if there are hashes whose availabilty is unknown
var possibleBatchResult = await m_innerCache.TryLoadAvailableContentAsync(uniqueUnknownAvailabilityHashes);
if (!possibleBatchResult.Succeeded || uniqueUnknownAvailabilityHashes == hashes)
{
// If not successful or the hashes are the same as original hashes just return the result
return(possibleBatchResult);
}
// Populate hash availability map with results from inner cache
foreach (var result in possibleBatchResult.Result.Results)
{
hashAvailabilityMap[result.Hash] = result;
if (!result.IsAvailable)
{
allContentAvailable = false;
}
else
{
// Mark the hash as available for subsequent operations
m_availableContent.TryAdd(result.Hash, result.SourceCache);
}
}
}
ContentAvailabilityResult[] results = new ContentAvailabilityResult[hashes.Count];
for (int i = 0; i < hashes.Count; i++)
{
var hash = hashes[i];
if (hashAvailabilityMap.TryGetValue(hash, out var result))
{
results[i] = result;
}
else
{
throw Contract.AssertFailure(I($"Hash {hash} should be present in availability map."));
}
}
return(new ContentAvailabilityBatchResult(ReadOnlyArray <ContentAvailabilityResult> .FromWithoutCopy(results), allContentAvailable));
}
}
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash(
IArtifactContentCache contentCache,
ContentHash contentHash,
CancellationToken cancellationToken,
BoxRef <long> contentSize = null)
{
if (!EngineEnvironmentSettings.SkipExtraneousPins)
{
Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable =
await contentCache.TryLoadAvailableContentAsync(new[] { contentHash }, cancellationToken);
if (!maybeAvailable.Succeeded)
{
return(maybeAvailable.Failure);
}
bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable;
if (!contentIsAvailable)
{
return(default(Stream));
}
}
var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash);
if (!maybeStream.Succeeded)
{
if (maybeStream.Failure is NoCasEntryFailure)
{
return(default(Stream));
}
return(maybeStream.Failure);
}
Stream stream = maybeStream.Result;
if (contentSize != null)
{
contentSize.Value = stream.Length;
}
return(stream);
}
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash(
IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null)
{
Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable =
await contentCache.TryLoadAvailableContentAsync(new[] { contentHash });
if (!maybeAvailable.Succeeded)
{
return(maybeAvailable.Failure);
}
bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable;
if (!contentIsAvailable)
{
return(default(Stream));
}
var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
Stream stream = maybeStream.Result;
if (contentSize != null)
{
contentSize.Value = stream.Length;
}
return(stream);
}