public Task <Result <TState> > ReadModifyWriteAsync <TState>(
OperationContext context,
BlobName fileName,
Func <TState, TState> transform)
where TState : new()
{
return(ReadModifyWriteAsync <TState, Unit>(context, fileName, current => (transform(current), Unit.Void)).SelectAsync(r => r.Value.NextState));
}
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
await _container.CreateIfNotExistsAsync(
accessType : BlobContainerPublicAccessType.Off,
options : DefaultBlobStorageRequestOptions,
operationContext : null,
cancellationToken : context.Token);
return(await base.StartupCoreAsync(context));
}
/// <inheritdoc />
protected override async Task <PinResult> PinCoreAsync(
OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter)
{
try
{
var bulkResults = await PinAsync(context, new[] { contentHash }, context.Token, urgencyHint);
return(await bulkResults.SingleAwaitIndexed());
}
catch (Exception e)
{
return(new PinResult(e));
}
}
/// <inheritdoc />
protected override async Task <PlaceFileResult> PlaceFileCoreAsync(
OperationContext context,
ContentHash contentHash,
AbsolutePath path,
FileAccessMode accessMode,
FileReplacementMode replacementMode,
FileRealizationMode realizationMode,
UrgencyHint urgencyHint,
Counter retryCounter)
{
try
{
if (replacementMode != FileReplacementMode.ReplaceExisting && File.Exists(path.Path))
{
return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists));
}
if (ImplicitPin == ImplicitPin.PutAndGet)
{
var pinResult = await PinAsync(context, contentHash, context.Token, urgencyHint).ConfigureAwait(false);
if (!pinResult.Succeeded)
{
return(pinResult.Code == PinResult.ResultCode.ContentNotFound
? new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedContentNotFound)
: new PlaceFileResult(pinResult));
}
}
var fileMode = replacementMode == FileReplacementMode.ReplaceExisting
? FileMode.Create
: FileMode.CreateNew;
var possibleLength =
await PlaceFileInternalAsync(context, contentHash, path.Path, fileMode).ConfigureAwait(false);
return(possibleLength.HasValue
? new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy, possibleLength.Value)
: new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedContentNotFound));
}
catch (IOException e) when(IsErrorFileExists(e))
{
return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists));
}
catch (Exception e)
{
return(new PlaceFileResult(e));
}
}
private async Task <Stream> GetStreamInternalAsync(OperationContext context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes)
{
if (_downloadBlobsThroughBlobStore)
{
return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternalThroughBlobStore",
innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts),
context.Token).ConfigureAwait(false));
}
else
{
if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri))
{
_blobCounters[BlobContentSessionCounters.VstsDownloadUriFetchedFromRemote].Increment();
var blobId = contentHash.ToBlobIdentifier();
var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternal",
innerCts => BlobStoreHttpClient.GetDownloadUrisAsync(
new[] { ToVstsBlobIdentifier(blobId) },
EdgeCache.NotAllowed,
cancellationToken: innerCts),
context.Token).ConfigureAwait(false);
if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri))
{
return(null);
}
DownloadUriCache.Instance.AddDownloadUri(contentHash, uri);
}
else
{
_blobCounters[BlobContentSessionCounters.VstsDownloadUriFetchedInMemory].Increment();
}
return(await GetStreamThroughAzureBlobs(
uri.NotNullUri,
overrideStreamMinimumReadSizeInBytes,
_parallelSegmentDownloadConfig.SegmentDownloadTimeout,
context.Token).ConfigureAwait(false));
}
}
private Task <long?> PlaceFileInternalAsync(
OperationContext context, ContentHash contentHash, string path, FileMode fileMode)
{
return(AsyncHttpRetryHelper <long?> .InvokeAsync(
async() =>
{
Stream httpStream = null;
try
{
httpStream = await GetStreamInternalAsync(context, contentHash, null).ConfigureAwait(false);
if (httpStream == null)
{
return null;
}
try
{
var success = DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var preauthUri);
var uri = success ? preauthUri.NotNullUri : new Uri("http://empty.com");
Directory.CreateDirectory(Directory.GetParent(path).FullName);
// TODO: Investigate using ManagedParallelBlobDownloader instead (bug 1365340)
await ParallelHttpDownload.Download(
_parallelSegmentDownloadConfig,
uri,
httpStream,
null,
path,
fileMode,
context.Token,
(destinationPath, offset, endOffset) =>
Tracer.Debug(context, $"Download {destinationPath} [{offset}, {endOffset}) start."),
(destinationPath, offset, endOffset) =>
Tracer.Debug(context, $"Download {destinationPath} [{offset}, {endOffset}) end."),
(destinationPath, offset, endOffset, message) =>
Tracer.Debug(context, $"Download {destinationPath} [{offset}, {endOffset}) failed. (message: {message})"),
async(offset, token) =>
{
var offsetStream = await GetStreamInternalAsync(
context,
contentHash,
_parallelSegmentDownloadConfig.SegmentSizeInBytes).ConfigureAwait(false);
offsetStream.Position = offset;
return offsetStream;
},
() => BufferPool.Get()).ConfigureAwait(false);
}
catch (Exception e) when(fileMode == FileMode.CreateNew && !IsErrorFileExists(e))
{
try
{
// Need to delete here so that a partial download doesn't run afoul of FileReplacementMode.FailIfExists upon retry
// Don't do this if the error itself was that the file already existed
File.Delete(path);
}
catch (Exception ex)
{
Tracer.Warning(context, $"Error deleting file at {path}: {ex}");
}
throw;
}
return httpStream.Length;
}
catch (StorageException storageEx) when(storageEx.InnerException is WebException)
{
var webEx = (WebException)storageEx.InnerException;
if (((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.NotFound)
{
return null;
}
throw;
}
finally
{
httpStream?.Dispose();
}
},
maxRetries : 5,
tracer : new AppTraceSourceContextAdapter(context, "BlobReadOnlyContentSession", SourceLevels.All),
canRetryDelegate : exception =>
{
// HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM.
// Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not.
// Once the next release is completed, these retries can be removed.
if (exception is HttpRequestException && exception.InnerException is WebException)
{
return true;
}
while (exception != null)
{
if (exception is TimeoutException)
{
return true;
}
exception = exception.InnerException;
}
return false;
},
cancellationToken : context.Token,
continueOnCapturedContext : false,
context : context.TracingContext.Id.ToString()));
}
private async Task <IEnumerable <Task <Indexed <PinResult> > > > UpdateBlobStoreAsync(OperationContext context, IReadOnlyList <ContentHash> contentHashes, DateTime endDateTime)
{
// Convert missing content hashes to blob Ids
var blobIds = contentHashes.Select(c => ToVstsBlobIdentifier(c.ToBlobIdentifier())).ToList();
// Call TryReference on the blob ids
var references = blobIds.Distinct().ToDictionary(
blobIdentifier => blobIdentifier,
_ => (IEnumerable <BlobReference>) new[] { new BlobReference(endDateTime) });
// TODO: In groups of 1000 (bug 1365340)
var referenceResults = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"UpdateBlobStore",
innerCts => BlobStoreHttpClient.TryReferenceAsync(references, cancellationToken: innerCts),
context.Token).ConfigureAwait(false);
_counters[BackingContentStore.SessionCounters.PinSatisfiedFromRemote].Increment();
// There's 1-1 mapping between given content hashes and blob ids
var remoteResults = blobIds
.Select((blobId, i) =>
{
var pinResult = referenceResults.ContainsKey(blobId)
? PinResult.ContentNotFound
: PinResult.Success;
if (pinResult.Succeeded)
{
BackingContentStoreExpiryCache.Instance.AddExpiry(contentHashes[i], endDateTime);
}
return(pinResult.WithIndex(i));
});
return(remoteResults.AsTasks());
}
/// <inheritdoc /> protected override Task <IEnumerable <Task <Indexed <PlaceFileResult> > > > PlaceFileCoreAsync(OperationContext context, IReadOnlyList <ContentHashWithPath> hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) => throw new NotImplementedException();
protected override async Task <IEnumerable <Task <Indexed <PinResult> > > > PinCoreAsync(OperationContext context, IReadOnlyList <ContentHash> contentHashes, UrgencyHint urgencyHint, Counter retryCounter, Counter fileCounter)
{
try
{
var endDateTime = DateTime.UtcNow + TimeToKeepContent;
return(await Workflows.RunWithFallback(
contentHashes,
hashes => CheckInMemoryCaches(hashes, endDateTime),
hashes => UpdateBlobStoreAsync(context, hashes, endDateTime),
result => result.Succeeded));
}
catch (Exception ex)
{
context.TracingContext.Warning($"Exception when querying pins against the VSTS services {ex}");
return(contentHashes.Select((_, index) => Task.FromResult(new PinResult(ex).WithIndex(index))));
}
}
/// <inheritdoc />
protected override async Task <OpenStreamResult> OpenStreamCoreAsync(
OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter)
{
if (contentHash.HashType != RequiredHashType)
{
return(new OpenStreamResult(
$"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{contentHash.HashType}'."));
}
string tempFile = null;
try
{
if (ImplicitPin == ImplicitPin.PutAndGet)
{
var pinResult = await PinAsync(context, contentHash, context.Token, urgencyHint).ConfigureAwait(false);
if (!pinResult.Succeeded)
{
if (pinResult.Code == PinResult.ResultCode.ContentNotFound)
{
return(new OpenStreamResult(null));
}
return(new OpenStreamResult(pinResult));
}
}
tempFile = TempDirectory.CreateRandomFileName().Path;
var possibleLength =
await PlaceFileInternalAsync(context, contentHash, tempFile, FileMode.Create).ConfigureAwait(false);
if (possibleLength.HasValue)
{
return(new OpenStreamResult(new FileStream(
tempFile,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete,
StreamBufferSize,
FileOptions.DeleteOnClose)));
}
return(new OpenStreamResult(null));
}
catch (Exception e)
{
return(new OpenStreamResult(e));
}
finally
{
if (tempFile != null)
{
try
{
File.Delete(tempFile);
}
catch (Exception e)
{
Tracer.Warning(context, $"Error deleting temporary file at {tempFile}: {e}");
}
}
}
}
