/// <summary> /// Attempts to place content at the specified path. The content should have previously been stored or loaded into <paramref name="cache"/>. /// If not, this operation may fail. This operation may also fail due to I/O-related issues, such as lack of permissions to the destination. /// Note that the containing directory for <paramref name="path"/> is assumed to be created already. /// The file is added to the file content table, and may be tracked by a tracker. /// </summary> public abstract Task <Possible <ContentMaterializationResult, Failure> > TryMaterializeContentAsync( IArtifactContentCache cache, FileRealizationMode fileRealizationModes, AbsolutePath path, ContentHash contentHash, bool trackPath = true, bool recordPathInFileContentTable = true);
/// <summary> /// This method is not implemented. /// </summary> public override Task <Possible <ContentMaterializationResult, Failure> > TryMaterializeContentAsync( IArtifactContentCache cache, FileRealizationMode fileRealizationModes, AbsolutePath path, ContentHash contentHash, bool trackPath = true, bool recordPathInFileContentTable = true) => throw new NotImplementedException();
/// <summary> /// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache. /// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>. /// </summary> public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>( this IArtifactContentCache contentCache, T valueToSerialize, BoxRef <long> contentSize = null, StoreArtifactOptions options = default) { return(BondExtensions.TrySerializeAndStoreContent( valueToSerialize, async(valueHash, valueBuffer) => { using (var entryStream = new MemoryStream( valueBuffer.Array, valueBuffer.Offset, valueBuffer.Count, writable: false)) { Possible <Unit, Failure> maybeStored = await contentCache.TryStoreAsync( entryStream, contentHash: valueHash, options: options); return maybeStored.WithGenericFailure(); } }, contentSize)); }
/// <summary> /// This is a self-contained operation to obtain and deserialize a previously stored structure by its hash. /// First, this tries to load the given content with <see cref="IArtifactContentCache.TryLoadAvailableContentAsync"/>. /// If the content is available, returns a Bond-deserialized <typeparamref name="T"/>. /// If the content is unavailable, returns <c>null</c>. /// To store a structure (the inverse of this method), use <see cref="TrySerializeAndStoreContent{T}"/>. /// </summary> public static async Task <Possible <T, Failure> > TryLoadAndDeserializeContent <T>( this IArtifactContentCache contentCache, ContentHash contentHash, BoxRef <long> contentSize = null) where T : class { var maybeStream = await TryGetStreamFromContentHash(contentCache, contentHash, contentSize); if (!maybeStream.Succeeded) { return(maybeStream.Failure); } var stream = maybeStream.Result; if (stream == null) { return(default(T)); } using (stream) { int streamLength = (int)stream.Length; // Use default bond deserializer return(DeserializeWithInputStream <T>(stream, streamLength)); } }
/// <nodoc /> public SinglePhaseFingerprintStoreAdapter( LoggingContext loggingContext, PipExecutionContext context, ITwoPhaseFingerprintStore twoPhaseStore, IArtifactContentCache contentCache) : this(loggingContext, context.PathTable, twoPhaseStore, contentCache) { Contract.Requires(context != null); }
/// <summary> /// Creates a <see cref="EngineCache"/> which owns the provided facets. /// The facets will be disposed when the new instance is disposed. /// </summary> public EngineCache( IArtifactContentCache contentCache, ITwoPhaseFingerprintStore twoPhaseFingerprintStore) { Contract.Requires(contentCache != null); Contract.Requires(twoPhaseFingerprintStore != null); ArtifactContentCache = contentCache; TwoPhaseFingerprintStore = twoPhaseFingerprintStore; }
/// <summary> /// Bond-serializes a given <typeparamref name="T"/> and stores the result to the content cache. /// The returned content hash can be used to later deserialize the structure with <see cref="TryLoadAndDeserializeContent{T}"/>. /// </summary> public static Task <Possible <ContentHash> > TrySerializeAndStoreContent <T>( this IArtifactContentCache contentCache, T valueToSerialize, BoxRef <long> contentSize = null) { return(BondExtensions.TrySerializeAndStoreContent( contentCache, valueToSerialize, TryStoreContentAsync, contentSize)); }
/// <nodoc /> public SinglePhaseFingerprintStoreAdapter(LoggingContext loggingContext, PathTable pathTable, ITwoPhaseFingerprintStore twoPhaseStore, IArtifactContentCache contentCache) { Contract.Requires(loggingContext != null); Contract.Requires(pathTable != null); Contract.Requires(twoPhaseStore != null); Contract.Requires(contentCache != null); m_loggingContext = loggingContext; m_twoPhaseStore = twoPhaseStore; m_contentCache = contentCache; m_pathTable = pathTable; }
/// <nodoc /> public PipTwoPhaseCache(LoggingContext loggingContext, EngineCache cache, PipExecutionContext context, PathExpander pathExpander) { Contract.Requires(loggingContext != null); Contract.Requires(cache != null); Contract.Requires(context != null); LoggingContext = loggingContext; ArtifactContentCache = cache.ArtifactContentCache; TwoPhaseFingerprintStore = cache.TwoPhaseFingerprintStore; Context = context; Counters = new CounterCollection <PipCachingCounter>(); m_pathExpander = pathExpander; }
/// <summary> /// Tries to store symlink file to cache. /// </summary> public static async Task <Possible <ContentHash> > TryStoreToCacheAsync( LoggingContext loggingContext, IArtifactContentCache cache, ExpandedAbsolutePath symlinkFile) { var possibleStore = await cache.TryStoreAsync(FileRealizationMode.HardLinkOrCopy, symlinkFile); if (!possibleStore.Succeeded) { Tracing.Logger.Log.FailedStoreSymlinkFileToCache(loggingContext, symlinkFile.ExpandedPath, possibleStore.Failure.DescribeIncludingInnerFailures()); return(possibleStore.Failure); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file '{symlinkFile}' with hash '{possibleStore.Result}'.")); return(possibleStore.Result); }
private static async Task <Possible <Unit> > TryStoreContentAsync( IArtifactContentCache contentCache, ContentHash valueHash, ArraySegment <byte> valueBuffer) { using (var entryStream = new MemoryStream( valueBuffer.Array, valueBuffer.Offset, valueBuffer.Count, writable: false)) { Possible <Unit, Failure> maybeStored = await contentCache.TryStoreAsync( entryStream, contentHash : valueHash); return(maybeStored.WithGenericFailure()); } }
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash( IArtifactContentCache contentCache, ContentHash contentHash, CancellationToken cancellationToken, BoxRef <long> contentSize = null) { if (!EngineEnvironmentSettings.SkipExtraneousPins) { Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable = await contentCache.TryLoadAvailableContentAsync(new[] { contentHash }, cancellationToken); if (!maybeAvailable.Succeeded) { return(maybeAvailable.Failure); } bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable; if (!contentIsAvailable) { return(default(Stream)); } } var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash); if (!maybeStream.Succeeded) { if (maybeStream.Failure is NoCasEntryFailure) { return(default(Stream)); } return(maybeStream.Failure); } Stream stream = maybeStream.Result; if (contentSize != null) { contentSize.Value = stream.Length; } return(stream); }
/// <summary> /// This function is only used by Nuget package download which is going to be replaced soon by not using 1-phase cache lookup anymore. /// </summary> public override Task<Possible<ContentMaterializationResult, Failure>> TryMaterializeContentAsync( IArtifactContentCache cache, FileRealizationMode fileRealizationModes, AbsolutePath path, ContentHash contentHash, bool trackPath = true, bool recordPathInFileContentTable = true) { var request = new MaterializeFileRequest( cache, fileRealizationModes, path, contentHash, trackPath, recordPathInFileContentTable); m_localDiskContentStoreConcurrencyLimiter.Post(request); return request.CompletionSource.Task; }
/// <summary> /// Runs <see cref="TryLoadAndDeserializeContent{T}(IArtifactContentCache, ContentHash, BoxRef{long})"/> with some retry logic. /// </summary> public static async Task <Possible <T, Failure> > TryLoadAndDeserializeContentWithRetry <T>( this IArtifactContentCache contentCache, LoggingContext loggingContext, ContentHash contentHash, Func <Possible <T, Failure>, bool> shouldRetry, BoxRef <long> contentSize = null, int maxRetry = 1) where T : class { Contract.Requires(loggingContext != null); Contract.Requires(shouldRetry != null); int retryCount = 0; Possible <T, Failure> result; do { result = await TryLoadAndDeserializeContent <T>(contentCache, contentHash, contentSize); if (!shouldRetry(result)) { if (retryCount > 0) { Tracing.Logger.Log.RetryOnLoadingAndDeserializingMetadata(loggingContext, true, retryCount); } return(result); } ++retryCount; }while (retryCount < maxRetry); if (retryCount > 1) { Tracing.Logger.Log.RetryOnLoadingAndDeserializingMetadata(loggingContext, false, retryCount); } return(result); }
private static async Task <Possible <Stream, Failure> > TryGetStreamFromContentHash( IArtifactContentCache contentCache, ContentHash contentHash, BoxRef <long> contentSize = null) { Possible <ContentAvailabilityBatchResult, Failure> maybeAvailable = await contentCache.TryLoadAvailableContentAsync(new[] { contentHash }); if (!maybeAvailable.Succeeded) { return(maybeAvailable.Failure); } bool contentIsAvailable = maybeAvailable.Result.AllContentAvailable; if (!contentIsAvailable) { return(default(Stream)); } var maybeStream = await contentCache.TryOpenContentStreamAsync(contentHash); if (!maybeStream.Succeeded) { return(maybeStream.Failure); } Stream stream = maybeStream.Result; if (contentSize != null) { contentSize.Value = stream.Length; } return(stream); }
/// <summary> /// Try load contants. /// </summary> public static async Task <Possible <byte[], Failure> > TryLoadContent( this IArtifactContentCache contentCache, ContentHash contentHash, BoxRef <long> contentSize = null, bool failOnNonSeekableStream = false, int byteLimit = int.MaxValue) { var maybeStream = await TryGetStreamFromContentHash(contentCache, contentHash, contentSize); if (!maybeStream.Succeeded) { return(maybeStream.Failure); } var stream = maybeStream.Result; if (stream == null) { return(default(byte[])); } try { MemoryStream memoryStream; Stream streamToRead; if (!stream.CanSeek) { if (failOnNonSeekableStream) { return(new Failure <string>("Stream is not seekable")); } memoryStream = new MemoryStream(); streamToRead = memoryStream; } else { memoryStream = null; streamToRead = stream; } using (memoryStream) { if (memoryStream != null) { await stream.CopyToAsync(memoryStream); memoryStream.Position = 0; } Contract.Assert(streamToRead.CanSeek); if (streamToRead.Length > byteLimit) { return(new Failure <string>(I($"Stream exceeds limit: Length: {streamToRead.Length} byte(s) | Limit: {byteLimit} byte(s)"))); } var length = (int)streamToRead.Length; var contentBytesLocal = new byte[length]; int read = 0; while (read < length) { int readThisIteration = await streamToRead.ReadAsync(contentBytesLocal, read, length - read); if (readThisIteration == 0) { return(new Failure <string>("Unexpected end of stream")); } read += readThisIteration; } return(contentBytesLocal); } } catch (Exception e) { return(new Failure <string>(e.GetLogEventMessage())); } }
/// <nodoc /> public ElidingArtifactContentCacheWrapper(IArtifactContentCache innerCache) { m_innerCache = innerCache; }