/// <inheritdoc />
public Task <PlaceFileResult> PlaceFileAsync(
Context context,
ContentHash contentHash,
AbsolutePath path,
FileAccessMode accessMode,
FileReplacementMode replacementMode,
FileRealizationMode realizationMode,
CancellationToken token,
UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PlaceFileCoreAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint, BaseCounters[ContentSessionBaseCounters.PlaceFileRetries]),
extraStartMessage: $"({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})",
traceOperationStarted: TraceOperationStarted,
extraEndMessage: result =>
{
var message = $"input=({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})";
if (result.Metadata == null)
{
return message;
}
return message + $" Gate.OccupiedCount={result.Metadata.GateOccupiedCount} Gate.Wait={result.Metadata.GateWaitTime.TotalMilliseconds}ms";
},
traceErrorsOnly: TraceErrorsOnlyForPlaceFile(path),
counter: BaseCounters[ContentSessionBaseCounters.PlaceFile])));
}
private async Task <Result <(long newCapacity, string key)> > TryReserveAsync(OperationContext context, long byteCount, ContentHash hash)
{
var operationStart = _clock.UtcNow;
var time = new DateTime(ticks: operationStart.Ticks / _blobExpiryTime.Ticks * _blobExpiryTime.Ticks);
var key = $"BlobCapacity@{time.ToString("yyyyMMdd:hhmmss.fff")}";
if (key == _lastFailedReservationKey)
{
string message = $"Skipping reservation for blob [{hash.ToShortString()}] because key [{key}] ran out of capacity.";
return(Result.FromErrorMessage <(long newCapacity, string key)>(message));
}
var newUsedCapacity = await _redis.ExecuteBatchAsync(context, async batch =>
{
var stringSetTask = batch.StringSetAsync(key, 0, _capacityExpiryTime, StackExchange.Redis.When.NotExists);
var incrementTask = batch.StringIncrementAsync(key, byValue: byteCount);
await Task.WhenAll(stringSetTask, incrementTask);
return(await incrementTask);
}, RedisOperation.StringIncrement);
var couldReserve = newUsedCapacity <= _maxCapacityPerTimeBox;
if (!couldReserve)
{
_lastFailedReservationKey = key;
string error = $"Could not reserve {byteCount} for {hash.ToShortString()} because key [{key}] ran out of capacity. Expected new capacity={newUsedCapacity} bytes, Max capacity={_maxCapacityPerTimeBox} bytes.";
return(Result.FromErrorMessage <(long newCapacity, string key)>(error));
}
return(Result.Success((newUsedCapacity, key)));
}
private RejectionReason CanHandlePushRequest(Context cacheContext, ContentHash hash, IPushFileHandler store)
{
if (store == null)
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because no stores implement {nameof(IPushFileHandler)}.");
return(RejectionReason.NotSupported);
}
if (!store.CanAcceptContent(cacheContext, hash, out var rejectionReason))
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped: {rejectionReason}");
return(rejectionReason);
}
lock (_pushesLock)
{
if (_ongoingPushes.Count >= _ongoingPushCountLimit)
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because the max number of proactive pushes of '{_ongoingPushCountLimit}' is reached. OngoingPushes.Count={_ongoingPushes.Count}.");
return(RejectionReason.CopyLimitReached);
}
if (!_ongoingPushes.Add(hash))
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because another request to push it is already being handled.");
return(RejectionReason.OngoingCopy);
}
}
return(RejectionReason.Accepted);
}
private bool CanHandlePushRequest(Context cacheContext, ContentHash hash, [NotNullWhen(true)] IPushFileHandler store)
{
if (store == null)
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because no stores implement {nameof(IPushFileHandler)}.");
return(false);
}
if (!store.CanAcceptContent(cacheContext, hash, out var rejectionReason))
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped: {rejectionReason}");
return(false);
}
var count = _ongoingPushes.Count;
if (count >= _ongoingPushCountLimit)
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because the max number of proactive pushes of '{_ongoingPushCountLimit}' is reached. OngoingPushes.Count={count}.");
return(false);
}
if (!_ongoingPushes.Add(hash))
{
Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because another request to push it is already being handled.");
return(false);
}
return(true);
}
private async Task <ExistenceResponse> CheckFileExistsAsync(ExistenceRequest request, CancellationToken token)
{
OperationStarted();
DateTime startTime = DateTime.UtcNow;
Context cacheContext = new Context(new Guid(request.TraceId), Logger);
HashType type = (HashType)request.HashType;
ContentHash hash = request.ContentHash.ToContentHash((HashType)request.HashType);
// Iterate through all known stores, looking for content in each.
// In most of our configurations there is just one store anyway,
// and doing this means both we can callers don't have
// to deal with cache roots and drive letters.
foreach (KeyValuePair <string, IContentStore> entry in _contentStoreByCacheName)
{
if (entry.Value is IStreamStore store)
{
FileExistenceResult result = await store.CheckFileExistsAsync(cacheContext, hash);
if (result.Succeeded)
{
return(new ExistenceResponse {
Header = ResponseHeader.Success(startTime)
});
}
}
}
return(new ExistenceResponse {
Header = ResponseHeader.Failure(startTime, $"{hash.ToShortString()} not found in the cache")
});
}
private async Task <(bool isValid, string error)> ValidateFileAsync(Context context, ContentHash expectedHash, FileInfo fileInfo)
{
try
{
var path = fileInfo.FullPath;
// The cache entry is invalid if the size in content directory doesn't mach an actual size
if (_contentStoreInternal.TryGetFileInfo(expectedHash, out var contentFileInfo) && contentFileInfo.FileSize != fileInfo.Length)
{
return(isValid : false, error : $"File size mismatch. Expected size is {contentFileInfo.FileSize} and size on disk is {fileInfo.Length}.");
}
// Or if the content doesn't match the hash.
var actualHashAndSize = await _contentStoreInternal.TryHashFileAsync(context, path, expectedHash.HashType);
if (actualHashAndSize != null && actualHashAndSize.Value.Hash != expectedHash)
{
// Don't need to add an expected hash into the error string because the client code will always put it into the final error message.
return(isValid : false, error : $"Hash mismatch. Actual hash is {actualHashAndSize.Value.Hash.ToShortString()}");
}
return(isValid : true, error : string.Empty);
}
catch (Exception e)
{
_tracer.Warning(context, $"SelfCheck: Content hash is invalid. Hash={expectedHash.ToShortString()}, Error={e}");
return(isValid : true, error : string.Empty);
}
}
/// <summary>
/// The reservation strategy consists of timeboxes of 30 minutes, where each box only has half the max permitted
/// capacity. This is to account for Redis not deleting files exactly when their TTL expires.
/// Under this scheme, each blob will try to add its length to its box's capacity and fail if max capacity has
/// been exceeded.
/// </summary>
private async Task <bool> TryReserveAsync(OperationContext context, long byteCount, ContentHash hash)
{
var operationStart = _clock.UtcNow;
var time = new DateTime(ticks: operationStart.Ticks / _blobExpiryTime.Ticks * _blobExpiryTime.Ticks);
var key = $"BlobCapacity@{time.ToString("yyyyMMdd:hhmmss.fff")}";
if (key == _lastFailedReservationKey)
{
context.TraceDebug($"Skipping reservation for blob [{hash.ToShortString()}] because key [{key}] has already been used in a previous failed reservation");
return(false);
}
var newUsedCapacity = await _redis.ExecuteBatchAsync(context, async batch =>
{
var stringSetTask = batch.StringSetAsync(key, 0, _capacityExpiryTime, StackExchange.Redis.When.NotExists);
var incrementTask = batch.StringIncrementAsync(key, byValue: byteCount);
await Task.WhenAll(stringSetTask, incrementTask);
return(await incrementTask);
}, RedisOperation.StringIncrement);
var couldReserve = newUsedCapacity <= _maxCapacityPerTimeBox;
context.TraceDebug($"{(couldReserve ? "Successfully reserved" : "Could not reserve")} {byteCount} bytes in {key} for {hash.ToShortString()}. New used capacity: {newUsedCapacity} bytes");
if (!couldReserve)
{
_lastFailedReservationKey = key;
}
return(couldReserve);
}
/// <summary>
/// Puts a blob into Redis. Will fail only if capacity cannot be reserved or if Redis fails in some way.
/// </summary>
public Task <BoolResult> PutBlobAsync(OperationContext context, ContentHash hash, byte[] blob)
{
return(context.PerformOperationAsync(
_tracer,
async() =>
{
var key = GetBlobKey(hash);
if (await _redis.KeyExistsAsync(context, key, context.Token))
{
context.TraceDebug($"Hash=[{hash.ToShortString()}] is already in Redis. PutBlob skipped.");
_counters[RedisBlobAdapterCounters.SkippedBlobs].Increment();
return BoolResult.Success;
}
if (!await TryReserveAsync(context, blob.Length, hash))
{
_counters[RedisBlobAdapterCounters.FailedReservations].Increment();
return new BoolResult("Failed to reserve space for blob.");
}
var success = await _redis.StringSetAsync(context, key, blob, _blobExpiryTime, StackExchange.Redis.When.Always, context.Token);
return success ? BoolResult.Success : new BoolResult("Redis value could not be updated to upload blob.");
},
extraStartMessage: $"Size=[{blob.Length}]",
counter: _counters[RedisBlobAdapterCounters.PutBlob]));
}
/// <inheritdoc />
Task <PutResult> IContentSession.PutFileAsync(
Context context,
ContentHash contentHash,
AbsolutePath path,
FileRealizationMode realizationMode,
CancellationToken token,
UrgencyHint urgencyHint)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PutFileCoreAsync(operationContext, contentHash, path, realizationMode, urgencyHint, BaseCounters[ContentSessionBaseCounters.PutFileRetries]),
extraStartMessage: $"({path},{realizationMode},{contentHash.ToShortString()}) trusted=false",
traceOperationStarted: TraceOperationStarted,
extraEndMessage: result =>
{
var message = $"({path},{realizationMode}) trusted=false";
if (result.MetaData == null)
{
return message;
}
return message + $" Gate.OccupiedCount={result.MetaData.GateOccupiedCount} Gate.Wait={result.MetaData.GateWaitTime.TotalMilliseconds}ms";
},
traceErrorsOnly: TraceErrorsOnly,
counter: BaseCounters[ContentSessionBaseCounters.PutFile])));
}
/// <summary>
/// Pushes content to another machine.
/// </summary>
public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Result <Stream> > > source)
{
try
{
var pushRequest = new PushRequest(hash, context.TracingContext.Id);
var headers = pushRequest.GetMetadata();
var call = _client.PushFile(headers, cancellationToken: context.Token);
var requestStream = call.RequestStream;
var responseHeaders = await call.ResponseHeadersAsync;
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. To avoid an exception, exit early instead.
if (responseHeaders.Count == 0)
{
return(PushFileResult.ServerUnavailable());
}
var pushResponse = PushResponse.FromMetadata(responseHeaders);
if (!pushResponse.ShouldCopy)
{
context.TraceDebug($"{nameof(PushFileAsync)}: copy of {hash.ToShortString()} was skipped.");
return(PushFileResult.Rejected());
}
var streamResult = await source();
if (!streamResult)
{
await requestStream.CompleteAsync();
return(new PushFileResult(streamResult, "Failed to retrieve source stream."));
}
using (var stream = streamResult.Value)
{
await StreamContentAsync(stream, new byte[_bufferSize], requestStream, context.Token);
}
await requestStream.CompleteAsync();
var responseStream = call.ResponseStream;
await responseStream.MoveNext(context.Token);
var response = responseStream.Current;
return(response.Header.Succeeded
? PushFileResult.PushSucceeded()
: new PushFileResult(response.Header.ErrorMessage));
}
catch (RpcException r)
{
return(new PushFileResult(r));
}
}
public override string ToString()
{
string baseResult = $"Hash=[{_hash.ToShortString()}], BlobSize=[{_blobSize}], RedisKey=[{_redisKey}]";
if (Succeeded)
{
return($"{baseResult}. AlreadyInRedis=[{_alreadyInRedis}], SkipReason=[{_skipReason}] NewCapacity=[{_newRedisCapacity}]. {_extraMsg}");
}
return($"{baseResult}. {ErrorMessage}");
}
/// <inheritdoc />
public Task <PlaceFileResult> PlaceFileAsync(
Context context,
ContentHash contentHash,
AbsolutePath path,
FileAccessMode accessMode,
FileReplacementMode replacementMode,
FileRealizationMode realizationMode,
CancellationToken token,
UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PlaceFileCoreAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PlaceFileRetries]),
extraStartMessage: $"({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})",
extraEndMessage: (_) => $"input={contentHash.ToShortString()}",
counter: _counters[ContentSessionBaseCounters.PlaceFile])));
}
/// <summary>
/// Tries to get a blob from Redis.
/// </summary>
public Task <Result <byte[]> > GetBlobAsync(OperationContext context, ContentHash hash)
{
return(context.PerformOperationAsync(
_tracer,
async() =>
{
byte[] result = await _redis.StringGetAsync(context, GetBlobKey(hash), context.Token);
if (result == null)
{
return new Result <byte[]>($"Blob for hash=[{hash.ToShortString()}] was not found.");
}
_counters[RedisBlobAdapterCounters.DownloadedBytes].Add(result.Length);
_counters[RedisBlobAdapterCounters.DownloadedBlobs].Increment();
return new Result <byte[]>(result);
},
traceOperationStarted: false,
extraEndMessage: result => result.Succeeded ? $"Hash=[{hash.ToShortString()}], Size=[{result.Value.Length}]" : $"Hash=[{hash.ToShortString()}]",
counter: _counters[RedisBlobAdapterCounters.GetBlob]));
}
public virtual void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Severity successSeverity = Severity.Debug)
{
if (context.IsEnabled)
{
TracerOperationFinished(
context,
result,
$"{Name}.{PlaceFileCallName}({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode}) stop {result.DurationMs}ms result=[{result}]",
successSeverity);
}
_placeFileCallCounter.Completed(result.Duration.Ticks);
}
public virtual void OpenStreamStop(Context context, ContentHash contentHash, OpenStreamResult result, Severity successSeverity = Severity.Debug)
{
if (context.IsEnabled)
{
TracerOperationFinished(
context,
result,
$"{Name}.{OpenStreamCallName} stop {result.DurationMs}ms input=[{contentHash.ToShortString()}] result=[{result}]",
successSeverity);
}
_openStreamCallCounter.Completed(result.Duration.Ticks);
}
/// <summary>
/// Pushes content to another machine. Failure to open the source stream should return a null stream.
/// </summary>
public async Task <BoolResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Stream> > source)
{
try
{
var pushRequest = new PushRequest(hash, context.TracingContext.Id);
var headers = pushRequest.GetMetadata();
var call = _client.PushFile(headers, cancellationToken: context.Token);
var requestStream = call.RequestStream;
var responseHeaders = await call.ResponseHeadersAsync;
var pushResponse = PushResponse.FromMetadata(responseHeaders);
if (!pushResponse.ShouldCopy)
{
context.TraceDebug($"{nameof(PushFileAsync)}: copy of {hash.ToShortString()} was skipped.");
return(BoolResult.Success);
}
var stream = await source();
if (stream == null)
{
await requestStream.CompleteAsync();
return(new BoolResult("Failed to retrieve source stream."));
}
using (stream)
{
await StreamContentAsync(stream, new byte[_bufferSize], requestStream, context.Token);
}
await requestStream.CompleteAsync();
var responseStream = call.ResponseStream;
await responseStream.MoveNext(context.Token);
var response = responseStream.Current;
return(response.Header.Succeeded
? BoolResult.Success
: new BoolResult(response.Header.ErrorMessage));
}
catch (RpcException r)
{
return(new BoolResult(r));
}
}
/// <inheritdoc />
Task <PutResult> IContentSession.PutStreamAsync(
Context context,
ContentHash contentHash,
Stream stream,
CancellationToken token,
UrgencyHint urgencyHint)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PutStreamCoreAsync(operationContext, contentHash, stream, urgencyHint, _counters[ContentSessionBaseCounters.PutStreamRetries]),
extraStartMessage: $"({contentHash.ToShortString()})",
counter: _counters[ContentSessionBaseCounters.PutStream])));
}
/// <inheritdoc />
public Task <PinResult> PinAsync(
Context context,
ContentHash contentHash,
CancellationToken token,
UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PinCoreAsync(operationContext, contentHash, urgencyHint, _counters[ContentSessionBaseCounters.PinRetries]),
traceOperationStarted: false,
extraEndMessage: _ => $"input=[{contentHash.ToShortString()}]",
counter: _counters[ContentSessionBaseCounters.Pin])));
}
/// <inheritdoc />
public Task <PinResult> PinAsync(
Context context,
ContentHash contentHash,
CancellationToken token,
UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PinCoreAsync(operationContext, contentHash, urgencyHint, BaseCounters[ContentSessionBaseCounters.PinRetries]),
traceOperationStarted: TraceOperationStarted,
traceOperationFinished: TracePinFinished,
traceErrorsOnly: TraceErrorsOnly,
extraEndMessage: r => $"input=[{contentHash.ToShortString()}] size=[{r.ContentSize}]",
counter: BaseCounters[ContentSessionBaseCounters.Pin])));
}
public void PlaceFileCopy(Context context, AbsolutePath path, ContentHash contentHash, TimeSpan duration)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileCopy(context.Id.ToString(), path.Path, contentHash.ToString());
}
_placeFileCopyCallCounter.Completed(duration.Ticks);
if (!context.IsEnabled)
{
return;
}
var ms = (long)duration.TotalMilliseconds;
DebugCore(context, $"{Name}.PlaceFileCopy({path},{contentHash.ToShortString()}) {ms}ms");
}
public void PlaceFileCopy(Context context, AbsolutePath path, ContentHash contentHash, TimeSpan duration)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileCopy(context.TraceId, path.Path, contentHash.ToString());
}
_placeFileCopyCallCounter.Completed(duration.Ticks);
if (!context.IsEnabled)
{
return;
}
var ms = (long)duration.TotalMilliseconds;
TraceDiagnostic(context, $"{Name}.PlaceFileCopy({path},{contentHash.ToShortString()}) {ms}ms", TimeSpan.FromMilliseconds(ms), operation: "PlaceFileCopy");
}
/// <inheritdoc />
Task <PutResult> IContentSession.PutFileAsync(
Context context,
ContentHash contentHash,
AbsolutePath path,
FileRealizationMode realizationMode,
CancellationToken token,
UrgencyHint urgencyHint)
{
return(WithOperationContext(
context,
token,
operationContext => operationContext.PerformOperationAsync(
Tracer,
() => PutFileCoreAsync(operationContext, contentHash, path, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PutFileRetries]),
extraStartMessage: $"({path},{realizationMode},{contentHash.ToShortString()}) trusted=false",
extraEndMessage: _ => "trusted=false",
counter: _counters[ContentSessionBaseCounters.PutFile])));
}
public async Task <GetBulkLocationsResult> GetBulkAsync(OperationContext context, ContentHash hash)
{
var startedCopyHash = ComputeStartedCopyHash(hash);
await RegisterLocalLocationAsync(context, new[] { new ContentHashWithSize(startedCopyHash, -1) }).ThrowIfFailure();
for (int i = 0; i < Configuration.PropagationIterations; i++)
{
// If initial place fails, try to copy the content from remote locations
var(hashInfo, pendingCopyCount) = await GetFileLocationsAsync(context, hash, startedCopyHash);
var machineId = ClusterState.PrimaryMachineId.Index;
int machineNumber = GetMachineNumber();
var requiredReplicas = ComputeRequiredReplicas(machineNumber);
var actualReplicas = hashInfo.Locations?.Count ?? 0;
// Copy from peers if:
// The number of pending copies is known to be less that the max allowed copies
// OR the number replicas exceeds the number of required replicas computed based on the machine index
bool shouldCopy = pendingCopyCount < Configuration.MaxSimultaneousCopies || actualReplicas >= requiredReplicas;
Tracer.Debug(context, $"{i} (ShouldCopy={shouldCopy}): Hash={hash.ToShortString()}, Id={machineId}" +
$", Replicas={actualReplicas}, RequiredReplicas={requiredReplicas}, Pending={pendingCopyCount}, Max={Configuration.MaxSimultaneousCopies}");
if (shouldCopy)
{
return(new GetBulkLocationsResult(new[] { hashInfo }));
}
// Wait for content to propagate to more machines
await Task.Delay(Configuration.PropagationDelay, context.Token);
}
return(new GetBulkLocationsResult("Insufficient replicas"));
}
/// <inheritdoc />
public override string ToString()
{
return($"[ContentHash={ContentHash.ToShortString()} Exists={Exists}]");
}
public virtual void PinStop(Context context, ContentHash input, PinResult result)
{
if (context.IsEnabled)
{
TracerOperationFinished(context, result, $"{Name}.{PinCallName} stop {result.DurationMs}ms input=[{input.ToShortString()}] result=[{result}]");
}
_pinCallCounter.Completed(result.Duration.Ticks);
}
/// <inheritdoc />
public async Task <FileExistenceResult> CheckFileExistsAsync(Context context, ContentHash contentHash)
{
if (await Store.ContainsAsync(context, contentHash, null))
{
return(new FileExistenceResult());
}
else
{
return(new FileExistenceResult(FileExistenceResult.ResultCode.FileNotFound, $"{contentHash.ToShortString()} wasn't found in the cache"));
}
}
protected async Task AssertDoesNotContain(FileSystemContentStoreInternal store, ContentHash contentHash)
{
var contains = await store.ContainsAsync(Context, contentHash);
contains.Should().BeFalse($"Expected hash={contentHash.ToShortString()} to not be present but was");
}
private void TraceException(OperationContext context, ContentHash hash, Uri?azureBlobUri, Exception e, [CallerMemberName] string?operation = null)
{
string errorMessage = $"{operation} failed. ContentHash=[{hash.ToShortString()}], BaseAddress=[{BlobStoreHttpClient.BaseAddress}], BlobUri=[{getBlobUri(azureBlobUri)}]";
// Explicitly trace all the failures here to simplify errors analysis.
Tracer.Debug(context, $"{errorMessage}. Error=[{e}]");
private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash, string path = null)
{
if (!_pendingProactivePuts.Add(hash))
{
return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult));
}
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
try
{
var hashArray = _buildIdHash != null
? new[] { hash, _buildIdHash.Value }
: new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
return ProactiveCopyResult.CopyNotRequiredResult;
}
else
{
getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure();
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return ProactiveCopyResult.CopyNotRequiredResult;
}
IReadOnlyList <MachineLocation> buildRingMachines = null;
// Get random machine inside build ring
Task <BoolResult> insideRingCopyTask;
if ((Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) != 0)
{
if (_buildIdHash != null)
{
buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations;
var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray();
if (candidates.Length > 0)
{
var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)];
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines).");
insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring."));
}
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found."));
}
}
else
{
insideRingCopyTask = BoolResult.SuccessTask;
}
buildRingMachines ??= new[] { LocalCacheRootMachineLocation };
Task <BoolResult> outsideRingCopyTask;
if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) != 0)
{
var fromPredictionStore = true;
Result <MachineLocation> getLocationResult = null;
if (_predictionStore != null && path != null)
{
var machines = _predictionStore.GetTargetMachines(context, path);
if (machines?.Count > 0)
{
var index = ThreadSafeRandom.Generator.Next(0, machines.Count);
getLocationResult = new Result <MachineLocation>(new MachineLocation(machines[index]));
}
}
if (getLocationResult == null)
{
getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines);
fromPredictionStore = false;
}
if (getLocationResult.Succeeded)
{
var candidate = getLocationResult.Value;
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring. Candidate gotten from {(fromPredictionStore ? nameof(RocksDbContentPlacementPredictionStore) : nameof(ContentLocationStore))}");
outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult));
}
}
else
{
outsideRingCopyTask = BoolResult.SuccessTask;
}
return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask);
}
finally
{
_pendingProactivePuts.Remove(hash);
}
}));
public override string ToString()
{
return($"[ContentHash={ContentHash.ToShortString()} Size={Size} LocationCount={Locations?.Count}]");
}
