public Task <Result <TState> > ReadModifyWriteAsync <TState>( OperationContext context, BlobName fileName, Func <TState, TState> transform) where TState : new() { return(ReadModifyWriteAsync <TState, Unit>(context, fileName, current => (transform(current), Unit.Void)).SelectAsync(r => r.Value.NextState)); }
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { await _container.CreateIfNotExistsAsync( accessType : BlobContainerPublicAccessType.Off, options : DefaultBlobStorageRequestOptions, operationContext : null, cancellationToken : context.Token); return(await base.StartupCoreAsync(context)); }
/// <inheritdoc /> protected override async Task <PinResult> PinCoreAsync( OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) { try { var bulkResults = await PinAsync(context, new[] { contentHash }, context.Token, urgencyHint); return(await bulkResults.SingleAwaitIndexed()); } catch (Exception e) { return(new PinResult(e)); } }
/// <inheritdoc /> protected override async Task <PlaceFileResult> PlaceFileCoreAsync( OperationContext context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) { try { if (replacementMode != FileReplacementMode.ReplaceExisting && File.Exists(path.Path)) { return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists)); } if (ImplicitPin == ImplicitPin.PutAndGet) { var pinResult = await PinAsync(context, contentHash, context.Token, urgencyHint).ConfigureAwait(false); if (!pinResult.Succeeded) { return(pinResult.Code == PinResult.ResultCode.ContentNotFound ? new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedContentNotFound) : new PlaceFileResult(pinResult)); } } var fileMode = replacementMode == FileReplacementMode.ReplaceExisting ? FileMode.Create : FileMode.CreateNew; var possibleLength = await PlaceFileInternalAsync(context, contentHash, path.Path, fileMode).ConfigureAwait(false); return(possibleLength.HasValue ? new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy, possibleLength.Value) : new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedContentNotFound)); } catch (IOException e) when(IsErrorFileExists(e)) { return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists)); } catch (Exception e) { return(new PlaceFileResult(e)); } }
private async Task <Stream> GetStreamInternalAsync(OperationContext context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes) { if (_downloadBlobsThroughBlobStore) { return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternalThroughBlobStore", innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts), context.Token).ConfigureAwait(false)); } else { if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri)) { _blobCounters[BlobContentSessionCounters.VstsDownloadUriFetchedFromRemote].Increment(); var blobId = contentHash.ToBlobIdentifier(); var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( new[] { ToVstsBlobIdentifier(blobId) }, EdgeCache.NotAllowed, cancellationToken: innerCts), context.Token).ConfigureAwait(false); if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri)) { return(null); } DownloadUriCache.Instance.AddDownloadUri(contentHash, uri); } else { _blobCounters[BlobContentSessionCounters.VstsDownloadUriFetchedInMemory].Increment(); } return(await GetStreamThroughAzureBlobs( uri.NotNullUri, overrideStreamMinimumReadSizeInBytes, _parallelSegmentDownloadConfig.SegmentDownloadTimeout, context.Token).ConfigureAwait(false)); } }
private Task <long?> PlaceFileInternalAsync( OperationContext context, ContentHash contentHash, string path, FileMode fileMode) { return(AsyncHttpRetryHelper <long?> .InvokeAsync( async() => { Stream httpStream = null; try { httpStream = await GetStreamInternalAsync(context, contentHash, null).ConfigureAwait(false); if (httpStream == null) { return null; } try { var success = DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var preauthUri); var uri = success ? preauthUri.NotNullUri : new Uri("http://empty.com"); Directory.CreateDirectory(Directory.GetParent(path).FullName); // TODO: Investigate using ManagedParallelBlobDownloader instead (bug 1365340) await ParallelHttpDownload.Download( _parallelSegmentDownloadConfig, uri, httpStream, null, path, fileMode, context.Token, (destinationPath, offset, endOffset) => Tracer.Debug(context, $"Download {destinationPath} [{offset}, {endOffset}) start."), (destinationPath, offset, endOffset) => Tracer.Debug(context, $"Download {destinationPath} [{offset}, {endOffset}) end."), (destinationPath, offset, endOffset, message) => Tracer.Debug(context, $"Download {destinationPath} [{offset}, {endOffset}) failed. (message: {message})"), async(offset, token) => { var offsetStream = await GetStreamInternalAsync( context, contentHash, _parallelSegmentDownloadConfig.SegmentSizeInBytes).ConfigureAwait(false); offsetStream.Position = offset; return offsetStream; }, () => BufferPool.Get()).ConfigureAwait(false); } catch (Exception e) when(fileMode == FileMode.CreateNew && !IsErrorFileExists(e)) { try { // Need to delete here so that a partial download doesn't run afoul of FileReplacementMode.FailIfExists upon retry // Don't do this if the error itself was that the file already existed File.Delete(path); } catch (Exception ex) { Tracer.Warning(context, $"Error deleting file at {path}: {ex}"); } throw; } return httpStream.Length; } catch (StorageException storageEx) when(storageEx.InnerException is WebException) { var webEx = (WebException)storageEx.InnerException; if (((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.NotFound) { return null; } throw; } finally { httpStream?.Dispose(); } }, maxRetries : 5, tracer : new AppTraceSourceContextAdapter(context, "BlobReadOnlyContentSession", SourceLevels.All), canRetryDelegate : exception => { // HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM. // Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not. // Once the next release is completed, these retries can be removed. if (exception is HttpRequestException && exception.InnerException is WebException) { return true; } while (exception != null) { if (exception is TimeoutException) { return true; } exception = exception.InnerException; } return false; }, cancellationToken : context.Token, continueOnCapturedContext : false, context : context.TracingContext.Id.ToString())); }
private async Task <IEnumerable <Task <Indexed <PinResult> > > > UpdateBlobStoreAsync(OperationContext context, IReadOnlyList <ContentHash> contentHashes, DateTime endDateTime) { // Convert missing content hashes to blob Ids var blobIds = contentHashes.Select(c => ToVstsBlobIdentifier(c.ToBlobIdentifier())).ToList(); // Call TryReference on the blob ids var references = blobIds.Distinct().ToDictionary( blobIdentifier => blobIdentifier, _ => (IEnumerable <BlobReference>) new[] { new BlobReference(endDateTime) }); // TODO: In groups of 1000 (bug 1365340) var referenceResults = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "UpdateBlobStore", innerCts => BlobStoreHttpClient.TryReferenceAsync(references, cancellationToken: innerCts), context.Token).ConfigureAwait(false); _counters[BackingContentStore.SessionCounters.PinSatisfiedFromRemote].Increment(); // There's 1-1 mapping between given content hashes and blob ids var remoteResults = blobIds .Select((blobId, i) => { var pinResult = referenceResults.ContainsKey(blobId) ? PinResult.ContentNotFound : PinResult.Success; if (pinResult.Succeeded) { BackingContentStoreExpiryCache.Instance.AddExpiry(contentHashes[i], endDateTime); } return(pinResult.WithIndex(i)); }); return(remoteResults.AsTasks()); }
/// <inheritdoc /> protected override Task <IEnumerable <Task <Indexed <PlaceFileResult> > > > PlaceFileCoreAsync(OperationContext context, IReadOnlyList <ContentHashWithPath> hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) => throw new NotImplementedException();
protected override async Task <IEnumerable <Task <Indexed <PinResult> > > > PinCoreAsync(OperationContext context, IReadOnlyList <ContentHash> contentHashes, UrgencyHint urgencyHint, Counter retryCounter, Counter fileCounter) { try { var endDateTime = DateTime.UtcNow + TimeToKeepContent; return(await Workflows.RunWithFallback( contentHashes, hashes => CheckInMemoryCaches(hashes, endDateTime), hashes => UpdateBlobStoreAsync(context, hashes, endDateTime), result => result.Succeeded)); } catch (Exception ex) { context.TracingContext.Warning($"Exception when querying pins against the VSTS services {ex}"); return(contentHashes.Select((_, index) => Task.FromResult(new PinResult(ex).WithIndex(index)))); } }
/// <inheritdoc /> protected override async Task <OpenStreamResult> OpenStreamCoreAsync( OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) { if (contentHash.HashType != RequiredHashType) { return(new OpenStreamResult( $"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{contentHash.HashType}'.")); } string tempFile = null; try { if (ImplicitPin == ImplicitPin.PutAndGet) { var pinResult = await PinAsync(context, contentHash, context.Token, urgencyHint).ConfigureAwait(false); if (!pinResult.Succeeded) { if (pinResult.Code == PinResult.ResultCode.ContentNotFound) { return(new OpenStreamResult(null)); } return(new OpenStreamResult(pinResult)); } } tempFile = TempDirectory.CreateRandomFileName().Path; var possibleLength = await PlaceFileInternalAsync(context, contentHash, tempFile, FileMode.Create).ConfigureAwait(false); if (possibleLength.HasValue) { return(new OpenStreamResult(new FileStream( tempFile, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, StreamBufferSize, FileOptions.DeleteOnClose))); } return(new OpenStreamResult(null)); } catch (Exception e) { return(new OpenStreamResult(e)); } finally { if (tempFile != null) { try { File.Delete(tempFile); } catch (Exception e) { Tracer.Warning(context, $"Error deleting temporary file at {tempFile}: {e}"); } } } }