/// <summary>
/// Attempts to place content at the specified path. The content should have previously been stored or loaded into <paramref name="cache"/>.
/// If not, this operation may fail. This operation may also fail due to I/O-related issues, such as lack of permissions to the destination.
/// Note that the containing directory for <paramref name="path"/> is assumed to be created already.
/// The file is added to the file content table, and may be tracked by a tracker.
/// </summary>
public abstract Task <Possible <ContentMaterializationResult, Failure> > TryMaterializeContentAsync(
IArtifactContentCache cache,
FileRealizationMode fileRealizationModes,
AbsolutePath path,
ContentHash contentHash,
bool trackPath = true,
bool recordPathInFileContentTable = true);
/// <summary>
/// This method is not implemented.
/// </summary>
public override Task <Possible <ContentMaterializationResult, Failure> > TryMaterializeContentAsync(
IArtifactContentCache cache,
FileRealizationMode fileRealizationModes,
AbsolutePath path,
ContentHash contentHash,
bool trackPath = true,
bool recordPathInFileContentTable = true) => throw new NotImplementedException();
/// <summary>
/// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache.
/// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>.
/// </summary>
public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>(
this IArtifactContentCache contentCache,
T valueToSerialize,
BoxRef <long> contentSize = null,
StoreArtifactOptions options = default)
{
return(BondExtensions.TrySerializeAndStoreContent(
valueToSerialize,
async(valueHash, valueBuffer) =>
{
using (var entryStream = new MemoryStream(
valueBuffer.Array,
valueBuffer.Offset,
valueBuffer.Count,
writable: false))
{
Possible <Unit, Failure> maybeStored = await contentCache.TryStoreAsync(
entryStream,
contentHash: valueHash,
options: options);
return maybeStored.WithGenericFailure();
}
},
contentSize));
}
/// <summary>
/// This is a self-contained operation to obtain and deserialize a previously stored structure by its hash.
/// First, this tries to load the given content with <see cref="IArtifactContentCache.TryLoadAvailableContentAsync"/>.
/// If the content is available, returns a Bond-deserialized <typeparamref name="T"/>.
/// If the content is unavailable, returns <c>null</c>.
/// To store a structure (the inverse of this method), use <see cref="TrySerializeAndStoreContent{T}"/>.
/// </summary>
public static async Task <Possible <T, Failure> > TryLoadAndDeserializeContent <T>(
this IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null)
where T : class
{
var maybeStream = await TryGetStreamFromContentHash(contentCache, contentHash, contentSize);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
var stream = maybeStream.Result;
if (stream == null)
{
return(default(T));
}
using (stream)
{
int streamLength = (int)stream.Length;
// Use default bond deserializer
return(DeserializeWithInputStream <T>(stream, streamLength));
}
}
/// <nodoc />
public SinglePhaseFingerprintStoreAdapter(
LoggingContext loggingContext,
PipExecutionContext context,
ITwoPhaseFingerprintStore twoPhaseStore,
IArtifactContentCache contentCache)
: this(loggingContext, context.PathTable, twoPhaseStore, contentCache)
{
Contract.Requires(context != null);
}
/// <summary>
/// Creates a <see cref="EngineCache"/> which owns the provided facets.
/// The facets will be disposed when the new instance is disposed.
/// </summary>
public EngineCache(
IArtifactContentCache contentCache,
ITwoPhaseFingerprintStore twoPhaseFingerprintStore)
{
Contract.Requires(contentCache != null);
Contract.Requires(twoPhaseFingerprintStore != null);
ArtifactContentCache = contentCache;
TwoPhaseFingerprintStore = twoPhaseFingerprintStore;
}
/// <summary>
/// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache.
/// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>.
/// </summary>
public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>(
this IArtifactContentCache contentCache,
T valueToSerialize,
BoxRef <long> contentSize = null)
{
return(BondExtensions.TrySerializeAndStoreContent(
contentCache,
valueToSerialize,
TryStoreContentAsync,
contentSize));
}
/// <nodoc />
public SinglePhaseFingerprintStoreAdapter(LoggingContext loggingContext, PathTable pathTable, ITwoPhaseFingerprintStore twoPhaseStore, IArtifactContentCache contentCache)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pathTable != null);
Contract.Requires(twoPhaseStore != null);
Contract.Requires(contentCache != null);
m_loggingContext = loggingContext;
m_twoPhaseStore = twoPhaseStore;
m_contentCache = contentCache;
m_pathTable = pathTable;
}
/// <nodoc />
public PipTwoPhaseCache(LoggingContext loggingContext, EngineCache cache, PipExecutionContext context, PathExpander pathExpander)
{
Contract.Requires(loggingContext != null);
Contract.Requires(cache != null);
Contract.Requires(context != null);
LoggingContext = loggingContext;
ArtifactContentCache = cache.ArtifactContentCache;
TwoPhaseFingerprintStore = cache.TwoPhaseFingerprintStore;
Context = context;
Counters = new CounterCollection <PipCachingCounter>();
m_pathExpander = pathExpander;
}
/// <summary>
/// Tries to store symlink file to cache.
/// </summary>
public static async Task <Possible <ContentHash> > TryStoreToCacheAsync(
LoggingContext loggingContext,
IArtifactContentCache cache,
ExpandedAbsolutePath symlinkFile)
{
var possibleStore = await cache.TryStoreAsync(FileRealizationMode.HardLinkOrCopy, symlinkFile);
if (!possibleStore.Succeeded)
{
Tracing.Logger.Log.FailedStoreSymlinkFileToCache(loggingContext, symlinkFile.ExpandedPath, possibleStore.Failure.DescribeIncludingInnerFailures());
return(possibleStore.Failure);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file '{symlinkFile}' with hash '{possibleStore.Result}'."));
return(possibleStore.Result);
}
private static async Task <Possible <Unit> > TryStoreContentAsync(
IArtifactContentCache contentCache,
ContentHash valueHash,
ArraySegment <byte> valueBuffer)
{
using (var entryStream = new MemoryStream(
valueBuffer.Array,
valueBuffer.Offset,
valueBuffer.Count,
writable: false))
{
Possible <Unit, Failure> maybeStored = await contentCache.TryStoreAsync(
entryStream,
contentHash : valueHash);
return(maybeStored.WithGenericFailure());
}
}
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash(
IArtifactContentCache contentCache,
ContentHash contentHash,
CancellationToken cancellationToken,
BoxRef <long> contentSize = null)
{
if (!EngineEnvironmentSettings.SkipExtraneousPins)
{
Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable =
await contentCache.TryLoadAvailableContentAsync(new[] { contentHash }, cancellationToken);
if (!maybeAvailable.Succeeded)
{
return(maybeAvailable.Failure);
}
bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable;
if (!contentIsAvailable)
{
return(default(Stream));
}
}
var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash);
if (!maybeStream.Succeeded)
{
if (maybeStream.Failure is NoCasEntryFailure)
{
return(default(Stream));
}
return(maybeStream.Failure);
}
Stream stream = maybeStream.Result;
if (contentSize != null)
{
contentSize.Value = stream.Length;
}
return(stream);
}
/// <summary>
/// This function is only used by Nuget package download which is going to be replaced soon by not using 1-phase cache lookup anymore.
/// </summary>
public override Task<Possible<ContentMaterializationResult, Failure>> TryMaterializeContentAsync(
IArtifactContentCache cache,
FileRealizationMode fileRealizationModes,
AbsolutePath path,
ContentHash contentHash,
bool trackPath = true,
bool recordPathInFileContentTable = true)
{
var request = new MaterializeFileRequest(
cache,
fileRealizationModes,
path,
contentHash,
trackPath,
recordPathInFileContentTable);
m_localDiskContentStoreConcurrencyLimiter.Post(request);
return request.CompletionSource.Task;
}
/// <summary>
/// Runs <see cref="TryLoadAndDeserializeContent{T}(IArtifactContentCache, ContentHash, BoxRef{long})"/> with some retry logic.
/// </summary>
public static async Task <Possible <T, Failure> > TryLoadAndDeserializeContentWithRetry <T>(
this IArtifactContentCache contentCache,
LoggingContext loggingContext,
ContentHash contentHash,
Func <Possible <T, Failure>, bool> shouldRetry,
BoxRef <long> contentSize = null,
int maxRetry = 1)
where T : class
{
Contract.Requires(loggingContext != null);
Contract.Requires(shouldRetry != null);
int retryCount = 0;
Possible <T, Failure> result;
do
{
result = await TryLoadAndDeserializeContent <T>(contentCache, contentHash, contentSize);
if (!shouldRetry(result))
{
if (retryCount > 0)
{
Tracing.Logger.Log.RetryOnLoadingAndDeserializingMetadata(loggingContext, true, retryCount);
}
return(result);
}
++retryCount;
}while (retryCount < maxRetry);
if (retryCount > 1)
{
Tracing.Logger.Log.RetryOnLoadingAndDeserializingMetadata(loggingContext, false, retryCount);
}
return(result);
}
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash(
IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null)
{
Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable =
await contentCache.TryLoadAvailableContentAsync(new[] { contentHash });
if (!maybeAvailable.Succeeded)
{
return(maybeAvailable.Failure);
}
bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable;
if (!contentIsAvailable)
{
return(default(Stream));
}
var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
Stream stream = maybeStream.Result;
if (contentSize != null)
{
contentSize.Value = stream.Length;
}
return(stream);
}
/// <summary>
/// Try load contants.
/// </summary>
public static async Task <Possible <byte[], Failure> > TryLoadContent(
this IArtifactContentCache contentCache,
ContentHash contentHash,
BoxRef <long> contentSize = null,
bool failOnNonSeekableStream = false,
int byteLimit = int.MaxValue)
{
var maybeStream = await TryGetStreamFromContentHash(contentCache, contentHash, contentSize);
if (!maybeStream.Succeeded)
{
return(maybeStream.Failure);
}
var stream = maybeStream.Result;
if (stream == null)
{
return(default(byte[]));
}
try
{
MemoryStream memoryStream;
Stream streamToRead;
if (!stream.CanSeek)
{
if (failOnNonSeekableStream)
{
return(new Failure <string>("Stream is not seekable"));
}
memoryStream = new MemoryStream();
streamToRead = memoryStream;
}
else
{
memoryStream = null;
streamToRead = stream;
}
using (memoryStream)
{
if (memoryStream != null)
{
await stream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
Contract.Assert(streamToRead.CanSeek);
if (streamToRead.Length > byteLimit)
{
return(new Failure <string>(I($"Stream exceeds limit: Length: {streamToRead.Length} byte(s) | Limit: {byteLimit} byte(s)")));
}
var length = (int)streamToRead.Length;
var contentBytesLocal = new byte[length];
int read = 0;
while (read < length)
{
int readThisIteration = await streamToRead.ReadAsync(contentBytesLocal, read, length - read);
if (readThisIteration == 0)
{
return(new Failure <string>("Unexpected end of stream"));
}
read += readThisIteration;
}
return(contentBytesLocal);
}
}
catch (Exception e)
{
return(new Failure <string>(e.GetLogEventMessage()));
}
}
/// <nodoc />
public ElidingArtifactContentCacheWrapper(IArtifactContentCache innerCache)
{
m_innerCache = innerCache;
}