/// <inheritdoc /> public async Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode) { var sessionContext = await CreateSessionContextAsync(context); if (!sessionContext) { return(new PlaceFileResult(sessionContext)); } return(await PlaceFileAsync(sessionContext.Value, contentHash, path, accessMode, replacementMode, realizationMode)); }
private static FileState GetFileStateForRealizationMode(FileRealizationMode mode) { switch (mode) { case FileRealizationMode.Copy: return(FileState.Writeable); case FileRealizationMode.HardLink: return(FileState.ReadOnly); case FileRealizationMode.HardLinkOrCopy: return(FileState.ReadOnly); default: throw Contract.AssertFailure("Unhandled FileRealizationMode"); } }
/// <inheritdoc /> protected override Task <PutResult> PutFileCoreAsync( OperationContext operationContext, HashType hashType, AbsolutePath path, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) { return(PerformPutFileGatedOperationAsync(operationContext, () => { return PutCoreAsync( operationContext, (decoratedStreamSession, wrapStream) => decoratedStreamSession.PutFileAsync(operationContext, path, hashType, realizationMode, operationContext.Token, urgencyHint, wrapStream), session => session.PutFileAsync(operationContext, hashType, path, realizationMode, operationContext.Token, urgencyHint), path: path.Path); })); }
/// <inheritdoc /> public async Task <Possible <Unit, Failure> > TryMaterializeAsync( FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, ContentHash contentHash) { // TODO: The cache should be able to do this itself, preferably sharing the same code. // When placing content, we may be replacing output that has been hardlinked elsewhere. // Deleting links requires some care and magic, e.g. if a process has the file mapped. // Correspondingly, IArtifactContentCache prescribes that materialization always produces a 'new' file. var placeResult = await Helpers.RetryOnFailureAsync( async lastAttempt => { return(await TryMaterializeCoreAsync(fileRealizationModes, path, contentHash)); }); return(placeResult); }
/// <inheritdoc /> protected override Task <PutResult> PutFileCoreAsync( OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) { // We are intentionally not gating PutStream operations because we don't expect a high number of them at // the same time. return(PerformPutFileGatedOperationAsync(operationContext, () => { return PutCoreAsync( operationContext, (decoratedStreamSession, wrapStream) => decoratedStreamSession.PutFileAsync(operationContext, path, contentHash, realizationMode, operationContext.Token, urgencyHint, wrapStream), session => session.PutFileAsync(operationContext, contentHash, path, realizationMode, operationContext.Token, urgencyHint)); })); }
/// <inheritdoc /> Task <PutResult> IContentSession.PutFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PutFileCoreAsync(operationContext, contentHash, path, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PutFileRetries]), extraStartMessage: $"({path},{realizationMode},{contentHash.ToShortString()}) trusted=false", extraEndMessage: _ => "trusted=false", counter: _counters[ContentSessionBaseCounters.PutFile]))); }
private async Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, int sessionId) { var startTime = DateTime.UtcNow; PlaceFileResponse response = await RunClientActionAndThrowIfFailedAsync(context, async() => await _client.PlaceFileAsync( new PlaceFileRequest { Header = new RequestHeader(context.Id, sessionId), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Path = path.Path, FileAccessMode = (int)accessMode, FileRealizationMode = (int)realizationMode, FileReplacementMode = (int)replacementMode })); long ticksWaited = response.Header.ServerReceiptTimeUtcTicks - startTime.Ticks; _tracer.TrackClientWaitForServerTicks(ticksWaited); // Workaround: Handle the service returning negative result codes in error cases PlaceFileResult.ResultCode resultCode = response.Header.Result < 0 ? PlaceFileResult.ResultCode.Error : (PlaceFileResult.ResultCode)response.Header.Result; if (!response.Header.Succeeded) { await ResetOnUnknownSessionAsync(context, response.Header, sessionId); var message = string.IsNullOrEmpty(response.Header.ErrorMessage) ? resultCode.ToString() : response.Header.ErrorMessage; return(new PlaceFileResult(resultCode, message, response.Header.Diagnostics)); } else { return(new PlaceFileResult(resultCode, response.ContentSize)); } }
private Task ParallelPutsGets(bool useRandomContent, FileRealizationMode realizationModes) { var context = new Context(Logger); return(TestStore(context, _clock, async store => { using (var tempDirectory = new DisposableDirectory(FileSystem)) { IReadOnlyList <AbsolutePath> pathsToContent = Enumerable.Range(1, ParallelFileCount) .Select(i => tempDirectory.Path / ("tempContent" + i + ".txt")).ToList(); using (var hasher = HashInfoLookup.Find(ContentHashType).CreateContentHasher()) { IReadOnlyList <ContentHash> contentHashes = pathsToContent.Select(pathToContent => { var bytes = useRandomContent ? ThreadSafeRandom.GetBytes(ParallelRandomContentBytes) : new byte[0]; FileSystem.WriteAllBytes(pathToContent, bytes); return hasher.GetContentHash(bytes); }).ToList(); IReadOnlyList <AbsolutePath> outPathsToContent = pathsToContent.Select(pathToContent => tempDirectory.Path / ("out" + pathToContent.FileName)) .ToList(); var puts = Enumerable.Range(0, pathsToContent.Count).Select(i => Task.Run(async() => await store.PutFileAsync(context, pathsToContent[i], realizationModes, ContentHashType, null) )); var gets = Enumerable.Range(0, outPathsToContent.Count).Select(i => Task.Run(async() => await store.PlaceFileAsync( context, contentHashes[i], outPathsToContent[i], FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, realizationModes, null))); await TaskUtilities.SafeWhenAll(puts); await TaskUtilities.SafeWhenAll(gets); } } })); }
/// <inheritdoc /> public Task <IEnumerable <Task <Indexed <PlaceFileResult> > > > PlaceFileAsync( Context context, IReadOnlyList <ContentHashWithPath> hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { return(WithOperationContext( context, token, operationContext => operationContext.PerformNonResultOperationAsync( Tracer, () => PlaceFileCoreAsync(operationContext, hashesWithPaths, accessMode, replacementMode, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PlaceFileBulkRetries]), traceOperationStarted: false, traceOperationFinished: false, counter: _counters[ContentSessionBaseCounters.PlaceFileBulk]))); }
/// <inheritdoc /> public async Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode) { StructResult <int> sessionResult = await _sessionState.GetIdAsync(); if (!sessionResult.Succeeded) { return(new PlaceFileResult(sessionResult)); } int sessionId = sessionResult.Data; return(await PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, sessionId)); }
/// <inheritdoc /> public async Task <IEnumerable <Task <Indexed <PlaceFileResult> > > > PlaceFileAsync( Context context, IReadOnlyList <ContentHashWithPath> hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { // TODO: add support for PerformOperation with Task<IEnumerable<Task<Indexed<ResultBase>>>> using (var cancellableContext = TrackShutdown(context, token)) { using (_counters[ContentSessionBaseCounters.PlaceFileBulk].Start()) { return(await PlaceFileCoreAsync(cancellableContext, hashesWithPaths, accessMode, replacementMode, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PlaceFileBulkRetries])); } } }
/// <summary> /// This function is only used by Nuget package download which is going to be replaced soon by not using 1-phase cache lookup anymore. /// </summary> public override Task<Possible<ContentMaterializationResult, Failure>> TryMaterializeContentAsync( IArtifactContentCache cache, FileRealizationMode fileRealizationModes, AbsolutePath path, ContentHash contentHash, bool trackPath = true, bool recordPathInFileContentTable = true) { var request = new MaterializeFileRequest( cache, fileRealizationModes, path, contentHash, trackPath, recordPathInFileContentTable); m_localDiskContentStoreConcurrencyLimiter.Post(request); return request.CompletionSource.Task; }
private Task <PlaceFileResult> PlaceFileAsync( OperationContext operationContext, SessionContext context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode) { return(PerformOperationAsync( context, _ => Client.PlaceFileAsync( new PlaceFileRequest { Header = context.CreateHeader(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Path = path.Path, FileAccessMode = (int)accessMode, FileRealizationMode = (int)realizationMode, FileReplacementMode = (int)replacementMode }, options: GetCallOptions(Configuration.PlaceDeadline, operationContext.Token)), response => { // Workaround: Handle the service returning negative result codes in error cases PlaceFileResult.ResultCode resultCode = response.Header.Result < 0 ? PlaceFileResult.ResultCode.Error : (PlaceFileResult.ResultCode)response.Header.Result; if (!response.Header.Succeeded) { var message = string.IsNullOrEmpty(response.Header.ErrorMessage) ? resultCode.ToString() : response.Header.ErrorMessage; return new PlaceFileResult(resultCode, message, response.Header.Diagnostics); } else { return new PlaceFileResult(resultCode, response.ContentSize); } })); }
/// <nodoc /> public static Task <PlaceFileResult> PlaceFileAsync <TTracer>( this TTracer tracer, OperationContext context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Func <Task <PlaceFileResult> > func) where TTracer : ContentSessionTracer { return(PlaceFileCall <TTracer> .RunAsync( tracer, context, contentHash, path, accessMode, replacementMode, realizationMode, func)); }
public void FileStateToMemoization() { foreach (FileState fileState in Enum.GetValues(typeof(FileState))) { FileRealizationMode fileRealizationMode = fileState.ToMemoization(); switch (fileState) { case FileState.Writeable: Assert.Equal(FileRealizationMode.CopyNoVerify, fileRealizationMode); break; case FileState.ReadOnly: Assert.Equal(FileRealizationMode.Any, fileRealizationMode); break; default: throw new NotImplementedException(); } } }
/// <inheritdoc /> public Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PlaceFileCoreAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PlaceFileRetries]), extraStartMessage: $"({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})", extraEndMessage: (_) => $"input={contentHash.ToShortString()}", counter: _counters[ContentSessionBaseCounters.PlaceFile]))); }
public Task <IEnumerable <Task <Indexed <PlaceFileResult> > > > PlaceFileAsync( Context context, IReadOnlyList <ContentHashWithPath> hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) { // NOTE: this goes around the FileSystemContentStore's bulk place, rendering it useless. The reason // we do it this way is so that the hardlink logic stays consistent. This means that multiple place // operations from the same request may run at a higher rate than estipulated by the store. IEnumerable <Task <Indexed <PlaceFileResult> > > materializations = hashesWithPaths.Select(async(hashWithPath, index) => { var result = await PlaceFileAsync(context, hashWithPath.Hash, hashWithPath.Path, accessMode, replacementMode, realizationMode, cts, urgencyHint); return(new Indexed <PlaceFileResult>(result, index)); }); return(Task.FromResult(materializations.ToList().AsEnumerable())); }
/// <inheritdoc /> public async Task <Possible <ContentHash, Failure> > TryStoreAsync( FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path) { string expandedPath = GetExpandedPathForCache(path); Possible <ICacheSession, Failure> maybeOpen = m_cache.Get(nameof(TryStoreAsync)); Possible <CasHash, Failure> maybeStored = await maybeOpen.ThenAsync( async cache => { var result = await Helpers.RetryOnFailureAsync( async lastAttempt => { return(await cache.AddToCasAsync(expandedPath, GetFileStateForRealizationMode(fileRealizationModes))); }); return(result); }); return(maybeStored.Then <ContentHash>(c => c.ToContentHash())); }
/// <inheritdoc /> protected override async Task <PutResult> PutFileCoreAsync( OperationContext context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) { if (contentHash.HashType != RequiredHashType) { return(new PutResult( contentHash, $"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{contentHash.HashType}'.")); } try { var fileInfo = new FileInfo(path.Path); var contentSize = fileInfo.Length; using (var streamToPut = FileStreamUtils.OpenFileStreamForAsync( path.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { var putResult = await PutLazyStreamAsync(context, contentHash, streamToPut, urgencyHint).ConfigureAwait(false); if (!putResult.Succeeded) { return(new PutResult( putResult, contentHash, $"Failed to add a BlobStore reference to content with hash=[{contentHash}]")); } } return(new PutResult(contentHash, contentSize)); } catch (Exception e) { return(new PutResult(e, contentHash)); } }
private Task PutFileRecoversAsync( FileRealizationMode fileRealizationMode, Func<TestFileSystemContentStoreInternal, DisposableDirectory, Task<byte[]>> corruptFunc) { var context = new Context(Logger); return TestStore(context, Clock, async store => { using (var tempDirectory = new DisposableDirectory(FileSystem)) { var bytes = await corruptFunc(store, tempDirectory); var contentPath2 = tempDirectory.CreateRandomFileName(); FileSystem.WriteAllBytes(contentPath2, bytes); await store.PutFileAsync( context, contentPath2, fileRealizationMode, ContentHashType, null).ShouldBeSuccess(); FileSystem.GetHardLinkCount(contentPath2) .Should().Be(fileRealizationMode == FileRealizationMode.Copy ? 1 : 2); await store.EnsureHasContent(context, bytes.CalculateHash(ContentHashType), tempDirectory); } }); }
/// <summary> /// Gets the expanded path for the cache to use when putting/placing file. This allows /// the cache to pick a correct CAS root if CAS roots exist on various drives which /// would allow it to use a hardlink rather than copy if the appropriate CAS root is chosen. /// </summary> public string GetExpandedPathForCache(ExpandedAbsolutePath path, FileRealizationMode mode = default) { if (m_rootTranslator == null) { return(path.ExpandedPath); } else { var translatedPath = m_rootTranslator.Translate(path.ExpandedPath); if (translatedPath[0].ToUpperInvariantFast() == path.ExpandedPath[0].ToUpperInvariantFast() && translatedPath.Length > path.ExpandedPath.Length) { // The path root did not change as a result of path translation and its longer // so , just return the original path since the cache only cares about the root // when deciding a particular CAS to hardlink from to avoid MAX_PATH issues return(path.ExpandedPath); } return(translatedPath); } }
private Task PlaceFileRecoversAsync( Context context, Func <TestFileSystemContentStoreInternal, DisposableDirectory, Task <byte[]> > corruptFunc, FileRealizationMode fileRealizationMode) { return(TestStore(context, _clock, async store => { using (var tempDirectory = new DisposableDirectory(FileSystem)) { var bytes = await corruptFunc(store, tempDirectory); var contentHash = bytes.CalculateHash(ContentHashType); var placePath = tempDirectory.CreateRandomFileName(); PlaceFileResult result = null; Func <Task> putFunc = async() => { result = await store.PlaceFileAsync( context, contentHash, placePath, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.HardLink, null); }; putFunc.Should().NotThrow(); Assert.NotNull(result); result.Code.Should() .Be(fileRealizationMode == FileRealizationMode.Copy ? PlaceFileResult.ResultCode.PlacedWithCopy : PlaceFileResult.ResultCode.PlacedWithHardLink); FileSystem.GetHardLinkCount(placePath) .Should() .Be(fileRealizationMode == FileRealizationMode.Copy ? 1 : 2); } })); }
/// <inheritdoc /> public async Task <Possible <ContentHash, Failure> > TryStoreAsync( FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, StoreArtifactOptions options = default) { string expandedPath = GetExpandedPathForCache(path); Possible <ICacheSession, Failure> maybeOpen = m_cache.Get(nameof(TryStoreAsync)); Possible <CasHash, Failure> maybeStored = await PerformArtifactCacheOperationAsync( () => maybeOpen.ThenAsync( async cache => { var result = await Helpers.RetryOnFailureAsync( async lastAttempt => { return(await cache.AddToCasAsync(expandedPath, GetFileStateForRealizationMode(fileRealizationModes), urgencyHint: options.IsCacheEntryContent?UrgencyHint.SkipRegisterContent: default)); }); return(result); }), nameof(TryStoreAsync)); return(maybeStored.Then <ContentHash>(c => c.ToContentHash())); }
/// <inheritdoc /> public Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint) { return(PlaceFileCall <ContentSessionTracer> .RunAsync(Tracer.ContentSessionTracer, new OperationContext(context), contentHash, path, accessMode, replacementMode, realizationMode, async() => { if (WriteThroughContentSession != null) { var result = await WriteThroughContentSession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint).ConfigureAwait(false); if (result.Succeeded || result.Code != PlaceFileResult.ResultCode.NotPlacedContentNotFound) { return result; } } return await BackingContentSession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); })); }
public override void PutFileStop(Context context, PutResult result, bool trusted, AbsolutePath path, FileRealizationMode mode, Severity successSeverity) { if (_eventSource.IsEnabled()) { _eventSource.PutFileStop( context.TraceId, result.Succeeded, result.ErrorMessage, result.ContentHash.ToString()); } base.PutFileStop(context, result, trusted, path, mode, successSeverity: DiagnosticLevelSeverity(result.Duration)); }
public override void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Severity successSeverity) { if (_eventSource.IsEnabled()) { _eventSource.PlaceFileStop(context.TraceId, (int)result.Code, result.ErrorMessage); } base.PlaceFileStop(context, contentHash, result, path, accessMode, replacementMode, realizationMode, successSeverity: DiagnosticLevelSeverity(result.Duration)); }
/// <inheritdoc /> protected override Task <IEnumerable <Task <Indexed <PlaceFileResult> > > > PlaceFileCoreAsync(OperationContext operationContext, IReadOnlyList <ContentHashWithPath> hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) { throw new NotImplementedException(); }
private async Task PutFileAcrossDrivesAsync( FileRealizationMode allowedFileRealizationMode, bool contentAlreadyCached, bool contentShouldBeCached, Func <PutResult, bool> checkResult) { // This only works when we have multiple drives. if (FileSystem is not MemoryFileSystem) { return; } using (var testDirectory = new DisposableDirectory(FileSystem)) { var context = new Context(Logger); try { using (var store = Create(testDirectory.Path, Clock)) { await store.StartupAsync(context).ShouldBeSuccess(); byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize); ContentHash contentHash = bytes.CalculateHash(ContentHashType); // Verify content doesn't exist yet in store Assert.False(await store.ContainsAsync(context, contentHash, null)); var pathToContentDifferentVolume = new AbsolutePath(PathGeneratorUtilities.GetAbsolutePath("D", "foo.txt")); try { FileSystem.WriteAllBytes(pathToContentDifferentVolume, bytes); if (contentAlreadyCached) { await store.PutFileAsync( context, pathToContentDifferentVolume, FileRealizationMode.Copy, ContentHashType, null).ShouldBeSuccess(); } var result = await store.PutFileAsync( context, pathToContentDifferentVolume, allowedFileRealizationMode, ContentHashType, null); Assert.True(checkResult(result)); (await store.ContainsAsync(context, contentHash, null)).Should() .Be(contentShouldBeCached); } finally { FileSystem.DeleteFile(pathToContentDifferentVolume); } await store.ShutdownAsync(context).ShouldBeSuccess(); } } finally { FileSystem.DeleteDirectory(testDirectory.Path, DeleteOptions.All); } } }
public Task <Possible <ContentHash, Failure> > TryStoreAsync(FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path) { return(this.m_cache.TryStoreAsync(fileRealizationModes, path)); }
public Task <Possible <Unit, Failure> > TryMaterializeAsync(FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, ContentHash contentHash) { return(this.m_cache.TryMaterializeAsync(fileRealizationModes, path, contentHash)); }