/// <inheritdoc /> public Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PlaceFileCoreAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint, BaseCounters[ContentSessionBaseCounters.PlaceFileRetries]), extraStartMessage: $"({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})", traceOperationStarted: TraceOperationStarted, extraEndMessage: result => { var message = $"input=({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})"; if (result.Metadata == null) { return message; } return message + $" Gate.OccupiedCount={result.Metadata.GateOccupiedCount} Gate.Wait={result.Metadata.GateWaitTime.TotalMilliseconds}ms"; }, traceErrorsOnly: TraceErrorsOnlyForPlaceFile(path), counter: BaseCounters[ContentSessionBaseCounters.PlaceFile]))); }
private async Task <Result <(long newCapacity, string key)> > TryReserveAsync(OperationContext context, long byteCount, ContentHash hash) { var operationStart = _clock.UtcNow; var time = new DateTime(ticks: operationStart.Ticks / _blobExpiryTime.Ticks * _blobExpiryTime.Ticks); var key = $"BlobCapacity@{time.ToString("yyyyMMdd:hhmmss.fff")}"; if (key == _lastFailedReservationKey) { string message = $"Skipping reservation for blob [{hash.ToShortString()}] because key [{key}] ran out of capacity."; return(Result.FromErrorMessage <(long newCapacity, string key)>(message)); } var newUsedCapacity = await _redis.ExecuteBatchAsync(context, async batch => { var stringSetTask = batch.StringSetAsync(key, 0, _capacityExpiryTime, StackExchange.Redis.When.NotExists); var incrementTask = batch.StringIncrementAsync(key, byValue: byteCount); await Task.WhenAll(stringSetTask, incrementTask); return(await incrementTask); }, RedisOperation.StringIncrement); var couldReserve = newUsedCapacity <= _maxCapacityPerTimeBox; if (!couldReserve) { _lastFailedReservationKey = key; string error = $"Could not reserve {byteCount} for {hash.ToShortString()} because key [{key}] ran out of capacity. Expected new capacity={newUsedCapacity} bytes, Max capacity={_maxCapacityPerTimeBox} bytes."; return(Result.FromErrorMessage <(long newCapacity, string key)>(error)); } return(Result.Success((newUsedCapacity, key))); }
private RejectionReason CanHandlePushRequest(Context cacheContext, ContentHash hash, IPushFileHandler store) { if (store == null) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because no stores implement {nameof(IPushFileHandler)}."); return(RejectionReason.NotSupported); } if (!store.CanAcceptContent(cacheContext, hash, out var rejectionReason)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped: {rejectionReason}"); return(rejectionReason); } lock (_pushesLock) { if (_ongoingPushes.Count >= _ongoingPushCountLimit) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because the max number of proactive pushes of '{_ongoingPushCountLimit}' is reached. OngoingPushes.Count={_ongoingPushes.Count}."); return(RejectionReason.CopyLimitReached); } if (!_ongoingPushes.Add(hash)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because another request to push it is already being handled."); return(RejectionReason.OngoingCopy); } } return(RejectionReason.Accepted); }
private bool CanHandlePushRequest(Context cacheContext, ContentHash hash, [NotNullWhen(true)] IPushFileHandler store) { if (store == null) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because no stores implement {nameof(IPushFileHandler)}."); return(false); } if (!store.CanAcceptContent(cacheContext, hash, out var rejectionReason)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped: {rejectionReason}"); return(false); } var count = _ongoingPushes.Count; if (count >= _ongoingPushCountLimit) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because the max number of proactive pushes of '{_ongoingPushCountLimit}' is reached. OngoingPushes.Count={count}."); return(false); } if (!_ongoingPushes.Add(hash)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because another request to push it is already being handled."); return(false); } return(true); }
private async Task <ExistenceResponse> CheckFileExistsAsync(ExistenceRequest request, CancellationToken token) { OperationStarted(); DateTime startTime = DateTime.UtcNow; Context cacheContext = new Context(new Guid(request.TraceId), Logger); HashType type = (HashType)request.HashType; ContentHash hash = request.ContentHash.ToContentHash((HashType)request.HashType); // Iterate through all known stores, looking for content in each. // In most of our configurations there is just one store anyway, // and doing this means both we can callers don't have // to deal with cache roots and drive letters. foreach (KeyValuePair <string, IContentStore> entry in _contentStoreByCacheName) { if (entry.Value is IStreamStore store) { FileExistenceResult result = await store.CheckFileExistsAsync(cacheContext, hash); if (result.Succeeded) { return(new ExistenceResponse { Header = ResponseHeader.Success(startTime) }); } } } return(new ExistenceResponse { Header = ResponseHeader.Failure(startTime, $"{hash.ToShortString()} not found in the cache") }); }
private async Task <(bool isValid, string error)> ValidateFileAsync(Context context, ContentHash expectedHash, FileInfo fileInfo) { try { var path = fileInfo.FullPath; // The cache entry is invalid if the size in content directory doesn't mach an actual size if (_contentStoreInternal.TryGetFileInfo(expectedHash, out var contentFileInfo) && contentFileInfo.FileSize != fileInfo.Length) { return(isValid : false, error : $"File size mismatch. Expected size is {contentFileInfo.FileSize} and size on disk is {fileInfo.Length}."); } // Or if the content doesn't match the hash. var actualHashAndSize = await _contentStoreInternal.TryHashFileAsync(context, path, expectedHash.HashType); if (actualHashAndSize != null && actualHashAndSize.Value.Hash != expectedHash) { // Don't need to add an expected hash into the error string because the client code will always put it into the final error message. return(isValid : false, error : $"Hash mismatch. Actual hash is {actualHashAndSize.Value.Hash.ToShortString()}"); } return(isValid : true, error : string.Empty); } catch (Exception e) { _tracer.Warning(context, $"SelfCheck: Content hash is invalid. Hash={expectedHash.ToShortString()}, Error={e}"); return(isValid : true, error : string.Empty); } }
/// <summary> /// The reservation strategy consists of timeboxes of 30 minutes, where each box only has half the max permitted /// capacity. This is to account for Redis not deleting files exactly when their TTL expires. /// Under this scheme, each blob will try to add its length to its box's capacity and fail if max capacity has /// been exceeded. /// </summary> private async Task <bool> TryReserveAsync(OperationContext context, long byteCount, ContentHash hash) { var operationStart = _clock.UtcNow; var time = new DateTime(ticks: operationStart.Ticks / _blobExpiryTime.Ticks * _blobExpiryTime.Ticks); var key = $"BlobCapacity@{time.ToString("yyyyMMdd:hhmmss.fff")}"; if (key == _lastFailedReservationKey) { context.TraceDebug($"Skipping reservation for blob [{hash.ToShortString()}] because key [{key}] has already been used in a previous failed reservation"); return(false); } var newUsedCapacity = await _redis.ExecuteBatchAsync(context, async batch => { var stringSetTask = batch.StringSetAsync(key, 0, _capacityExpiryTime, StackExchange.Redis.When.NotExists); var incrementTask = batch.StringIncrementAsync(key, byValue: byteCount); await Task.WhenAll(stringSetTask, incrementTask); return(await incrementTask); }, RedisOperation.StringIncrement); var couldReserve = newUsedCapacity <= _maxCapacityPerTimeBox; context.TraceDebug($"{(couldReserve ? "Successfully reserved" : "Could not reserve")} {byteCount} bytes in {key} for {hash.ToShortString()}. New used capacity: {newUsedCapacity} bytes"); if (!couldReserve) { _lastFailedReservationKey = key; } return(couldReserve); }
/// <summary> /// Puts a blob into Redis. Will fail only if capacity cannot be reserved or if Redis fails in some way. /// </summary> public Task <BoolResult> PutBlobAsync(OperationContext context, ContentHash hash, byte[] blob) { return(context.PerformOperationAsync( _tracer, async() => { var key = GetBlobKey(hash); if (await _redis.KeyExistsAsync(context, key, context.Token)) { context.TraceDebug($"Hash=[{hash.ToShortString()}] is already in Redis. PutBlob skipped."); _counters[RedisBlobAdapterCounters.SkippedBlobs].Increment(); return BoolResult.Success; } if (!await TryReserveAsync(context, blob.Length, hash)) { _counters[RedisBlobAdapterCounters.FailedReservations].Increment(); return new BoolResult("Failed to reserve space for blob."); } var success = await _redis.StringSetAsync(context, key, blob, _blobExpiryTime, StackExchange.Redis.When.Always, context.Token); return success ? BoolResult.Success : new BoolResult("Redis value could not be updated to upload blob."); }, extraStartMessage: $"Size=[{blob.Length}]", counter: _counters[RedisBlobAdapterCounters.PutBlob])); }
/// <inheritdoc /> Task <PutResult> IContentSession.PutFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PutFileCoreAsync(operationContext, contentHash, path, realizationMode, urgencyHint, BaseCounters[ContentSessionBaseCounters.PutFileRetries]), extraStartMessage: $"({path},{realizationMode},{contentHash.ToShortString()}) trusted=false", traceOperationStarted: TraceOperationStarted, extraEndMessage: result => { var message = $"({path},{realizationMode}) trusted=false"; if (result.MetaData == null) { return message; } return message + $" Gate.OccupiedCount={result.MetaData.GateOccupiedCount} Gate.Wait={result.MetaData.GateWaitTime.TotalMilliseconds}ms"; }, traceErrorsOnly: TraceErrorsOnly, counter: BaseCounters[ContentSessionBaseCounters.PutFile]))); }
/// <summary> /// Pushes content to another machine. /// </summary> public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Result <Stream> > > source) { try { var pushRequest = new PushRequest(hash, context.TracingContext.Id); var headers = pushRequest.GetMetadata(); var call = _client.PushFile(headers, cancellationToken: context.Token); var requestStream = call.RequestStream; var responseHeaders = await call.ResponseHeadersAsync; // If the remote machine couldn't be contacted, GRPC returns an empty // header collection. To avoid an exception, exit early instead. if (responseHeaders.Count == 0) { return(PushFileResult.ServerUnavailable()); } var pushResponse = PushResponse.FromMetadata(responseHeaders); if (!pushResponse.ShouldCopy) { context.TraceDebug($"{nameof(PushFileAsync)}: copy of {hash.ToShortString()} was skipped."); return(PushFileResult.Rejected()); } var streamResult = await source(); if (!streamResult) { await requestStream.CompleteAsync(); return(new PushFileResult(streamResult, "Failed to retrieve source stream.")); } using (var stream = streamResult.Value) { await StreamContentAsync(stream, new byte[_bufferSize], requestStream, context.Token); } await requestStream.CompleteAsync(); var responseStream = call.ResponseStream; await responseStream.MoveNext(context.Token); var response = responseStream.Current; return(response.Header.Succeeded ? PushFileResult.PushSucceeded() : new PushFileResult(response.Header.ErrorMessage)); } catch (RpcException r) { return(new PushFileResult(r)); } }
public override string ToString() { string baseResult = $"Hash=[{_hash.ToShortString()}], BlobSize=[{_blobSize}], RedisKey=[{_redisKey}]"; if (Succeeded) { return($"{baseResult}. AlreadyInRedis=[{_alreadyInRedis}], SkipReason=[{_skipReason}] NewCapacity=[{_newRedisCapacity}]. {_extraMsg}"); } return($"{baseResult}. {ErrorMessage}"); }
/// <inheritdoc /> public Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PlaceFileCoreAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PlaceFileRetries]), extraStartMessage: $"({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode})", extraEndMessage: (_) => $"input={contentHash.ToShortString()}", counter: _counters[ContentSessionBaseCounters.PlaceFile]))); }
/// <summary> /// Tries to get a blob from Redis. /// </summary> public Task <Result <byte[]> > GetBlobAsync(OperationContext context, ContentHash hash) { return(context.PerformOperationAsync( _tracer, async() => { byte[] result = await _redis.StringGetAsync(context, GetBlobKey(hash), context.Token); if (result == null) { return new Result <byte[]>($"Blob for hash=[{hash.ToShortString()}] was not found."); } _counters[RedisBlobAdapterCounters.DownloadedBytes].Add(result.Length); _counters[RedisBlobAdapterCounters.DownloadedBlobs].Increment(); return new Result <byte[]>(result); }, traceOperationStarted: false, extraEndMessage: result => result.Succeeded ? $"Hash=[{hash.ToShortString()}], Size=[{result.Value.Length}]" : $"Hash=[{hash.ToShortString()}]", counter: _counters[RedisBlobAdapterCounters.GetBlob])); }
public virtual void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Severity successSeverity = Severity.Debug) { if (context.IsEnabled) { TracerOperationFinished( context, result, $"{Name}.{PlaceFileCallName}({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode}) stop {result.DurationMs}ms result=[{result}]", successSeverity); } _placeFileCallCounter.Completed(result.Duration.Ticks); }
public virtual void OpenStreamStop(Context context, ContentHash contentHash, OpenStreamResult result, Severity successSeverity = Severity.Debug) { if (context.IsEnabled) { TracerOperationFinished( context, result, $"{Name}.{OpenStreamCallName} stop {result.DurationMs}ms input=[{contentHash.ToShortString()}] result=[{result}]", successSeverity); } _openStreamCallCounter.Completed(result.Duration.Ticks); }
/// <summary> /// Pushes content to another machine. Failure to open the source stream should return a null stream. /// </summary> public async Task <BoolResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Stream> > source) { try { var pushRequest = new PushRequest(hash, context.TracingContext.Id); var headers = pushRequest.GetMetadata(); var call = _client.PushFile(headers, cancellationToken: context.Token); var requestStream = call.RequestStream; var responseHeaders = await call.ResponseHeadersAsync; var pushResponse = PushResponse.FromMetadata(responseHeaders); if (!pushResponse.ShouldCopy) { context.TraceDebug($"{nameof(PushFileAsync)}: copy of {hash.ToShortString()} was skipped."); return(BoolResult.Success); } var stream = await source(); if (stream == null) { await requestStream.CompleteAsync(); return(new BoolResult("Failed to retrieve source stream.")); } using (stream) { await StreamContentAsync(stream, new byte[_bufferSize], requestStream, context.Token); } await requestStream.CompleteAsync(); var responseStream = call.ResponseStream; await responseStream.MoveNext(context.Token); var response = responseStream.Current; return(response.Header.Succeeded ? BoolResult.Success : new BoolResult(response.Header.ErrorMessage)); } catch (RpcException r) { return(new BoolResult(r)); } }
/// <inheritdoc /> Task <PutResult> IContentSession.PutStreamAsync( Context context, ContentHash contentHash, Stream stream, CancellationToken token, UrgencyHint urgencyHint) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PutStreamCoreAsync(operationContext, contentHash, stream, urgencyHint, _counters[ContentSessionBaseCounters.PutStreamRetries]), extraStartMessage: $"({contentHash.ToShortString()})", counter: _counters[ContentSessionBaseCounters.PutStream]))); }
/// <inheritdoc /> public Task <PinResult> PinAsync( Context context, ContentHash contentHash, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PinCoreAsync(operationContext, contentHash, urgencyHint, _counters[ContentSessionBaseCounters.PinRetries]), traceOperationStarted: false, extraEndMessage: _ => $"input=[{contentHash.ToShortString()}]", counter: _counters[ContentSessionBaseCounters.Pin]))); }
/// <inheritdoc /> public Task <PinResult> PinAsync( Context context, ContentHash contentHash, CancellationToken token, UrgencyHint urgencyHint = UrgencyHint.Nominal) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PinCoreAsync(operationContext, contentHash, urgencyHint, BaseCounters[ContentSessionBaseCounters.PinRetries]), traceOperationStarted: TraceOperationStarted, traceOperationFinished: TracePinFinished, traceErrorsOnly: TraceErrorsOnly, extraEndMessage: r => $"input=[{contentHash.ToShortString()}] size=[{r.ContentSize}]", counter: BaseCounters[ContentSessionBaseCounters.Pin]))); }
public void PlaceFileCopy(Context context, AbsolutePath path, ContentHash contentHash, TimeSpan duration) { if (_eventSource.IsEnabled()) { _eventSource.PlaceFileCopy(context.Id.ToString(), path.Path, contentHash.ToString()); } _placeFileCopyCallCounter.Completed(duration.Ticks); if (!context.IsEnabled) { return; } var ms = (long)duration.TotalMilliseconds; DebugCore(context, $"{Name}.PlaceFileCopy({path},{contentHash.ToShortString()}) {ms}ms"); }
public void PlaceFileCopy(Context context, AbsolutePath path, ContentHash contentHash, TimeSpan duration) { if (_eventSource.IsEnabled()) { _eventSource.PlaceFileCopy(context.TraceId, path.Path, contentHash.ToString()); } _placeFileCopyCallCounter.Completed(duration.Ticks); if (!context.IsEnabled) { return; } var ms = (long)duration.TotalMilliseconds; TraceDiagnostic(context, $"{Name}.PlaceFileCopy({path},{contentHash.ToShortString()}) {ms}ms", TimeSpan.FromMilliseconds(ms), operation: "PlaceFileCopy"); }
/// <inheritdoc /> Task <PutResult> IContentSession.PutFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, CancellationToken token, UrgencyHint urgencyHint) { return(WithOperationContext( context, token, operationContext => operationContext.PerformOperationAsync( Tracer, () => PutFileCoreAsync(operationContext, contentHash, path, realizationMode, urgencyHint, _counters[ContentSessionBaseCounters.PutFileRetries]), extraStartMessage: $"({path},{realizationMode},{contentHash.ToShortString()}) trusted=false", extraEndMessage: _ => "trusted=false", counter: _counters[ContentSessionBaseCounters.PutFile]))); }
public async Task <GetBulkLocationsResult> GetBulkAsync(OperationContext context, ContentHash hash) { var startedCopyHash = ComputeStartedCopyHash(hash); await RegisterLocalLocationAsync(context, new[] { new ContentHashWithSize(startedCopyHash, -1) }).ThrowIfFailure(); for (int i = 0; i < Configuration.PropagationIterations; i++) { // If initial place fails, try to copy the content from remote locations var(hashInfo, pendingCopyCount) = await GetFileLocationsAsync(context, hash, startedCopyHash); var machineId = ClusterState.PrimaryMachineId.Index; int machineNumber = GetMachineNumber(); var requiredReplicas = ComputeRequiredReplicas(machineNumber); var actualReplicas = hashInfo.Locations?.Count ?? 0; // Copy from peers if: // The number of pending copies is known to be less that the max allowed copies // OR the number replicas exceeds the number of required replicas computed based on the machine index bool shouldCopy = pendingCopyCount < Configuration.MaxSimultaneousCopies || actualReplicas >= requiredReplicas; Tracer.Debug(context, $"{i} (ShouldCopy={shouldCopy}): Hash={hash.ToShortString()}, Id={machineId}" + $", Replicas={actualReplicas}, RequiredReplicas={requiredReplicas}, Pending={pendingCopyCount}, Max={Configuration.MaxSimultaneousCopies}"); if (shouldCopy) { return(new GetBulkLocationsResult(new[] { hashInfo })); } // Wait for content to propagate to more machines await Task.Delay(Configuration.PropagationDelay, context.Token); } return(new GetBulkLocationsResult("Insufficient replicas")); }
/// <inheritdoc /> public override string ToString() { return($"[ContentHash={ContentHash.ToShortString()} Exists={Exists}]"); }
public virtual void PinStop(Context context, ContentHash input, PinResult result) { if (context.IsEnabled) { TracerOperationFinished(context, result, $"{Name}.{PinCallName} stop {result.DurationMs}ms input=[{input.ToShortString()}] result=[{result}]"); } _pinCallCounter.Completed(result.Duration.Ticks); }
/// <inheritdoc /> public async Task <FileExistenceResult> CheckFileExistsAsync(Context context, ContentHash contentHash) { if (await Store.ContainsAsync(context, contentHash, null)) { return(new FileExistenceResult()); } else { return(new FileExistenceResult(FileExistenceResult.ResultCode.FileNotFound, $"{contentHash.ToShortString()} wasn't found in the cache")); } }
protected async Task AssertDoesNotContain(FileSystemContentStoreInternal store, ContentHash contentHash) { var contains = await store.ContainsAsync(Context, contentHash); contains.Should().BeFalse($"Expected hash={contentHash.ToShortString()} to not be present but was"); }
private void TraceException(OperationContext context, ContentHash hash, Uri?azureBlobUri, Exception e, [CallerMemberName] string?operation = null) { string errorMessage = $"{operation} failed. ContentHash=[{hash.ToShortString()}], BaseAddress=[{BlobStoreHttpClient.BaseAddress}], BlobUri=[{getBlobUri(azureBlobUri)}]"; // Explicitly trace all the failures here to simplify errors analysis. Tracer.Debug(context, $"{errorMessage}. Error=[{e}]");
private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash, string path = null) { if (!_pendingProactivePuts.Add(hash)) { return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult)); } return(context.PerformOperationAsync( Tracer, traceErrorsOnly: true, operation: async() => { try { var hashArray = _buildIdHash != null ? new[] { hash, _buildIdHash.Value } : new[] { hash }; // First check in local location store, then global if failed. var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local); if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { _counters[Counters.GetLocationsSatisfiedFromLocal].Increment(); return ProactiveCopyResult.CopyNotRequiredResult; } else { getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure(); _counters[Counters.GetLocationsSatisfiedFromRemote].Increment(); } if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { return ProactiveCopyResult.CopyNotRequiredResult; } IReadOnlyList <MachineLocation> buildRingMachines = null; // Get random machine inside build ring Task <BoolResult> insideRingCopyTask; if ((Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) != 0) { if (_buildIdHash != null) { buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations; var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray(); if (candidates.Length > 0) { var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines)."); insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } else { insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring.")); } } else { insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found.")); } } else { insideRingCopyTask = BoolResult.SuccessTask; } buildRingMachines ??= new[] { LocalCacheRootMachineLocation }; Task <BoolResult> outsideRingCopyTask; if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) != 0) { var fromPredictionStore = true; Result <MachineLocation> getLocationResult = null; if (_predictionStore != null && path != null) { var machines = _predictionStore.GetTargetMachines(context, path); if (machines?.Count > 0) { var index = ThreadSafeRandom.Generator.Next(0, machines.Count); getLocationResult = new Result <MachineLocation>(new MachineLocation(machines[index])); } } if (getLocationResult == null) { getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines); fromPredictionStore = false; } if (getLocationResult.Succeeded) { var candidate = getLocationResult.Value; Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring. Candidate gotten from {(fromPredictionStore ? nameof(RocksDbContentPlacementPredictionStore) : nameof(ContentLocationStore))}"); outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } else { outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult)); } } else { outsideRingCopyTask = BoolResult.SuccessTask; } return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask); } finally { _pendingProactivePuts.Remove(hash); } }));
public override string ToString() { return($"[ContentHash={ContentHash.ToShortString()} Size={Size} LocationCount={Locations?.Count}]"); }