private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash, string path = null) { if (!_pendingProactivePuts.Add(hash)) { return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult)); } return(context.PerformOperationAsync( Tracer, traceErrorsOnly: true, operation: async() => { try { var hashArray = _buildIdHash != null ? new[] { hash, _buildIdHash.Value } : new[] { hash }; // First check in local location store, then global if failed. var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local); if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { _counters[Counters.GetLocationsSatisfiedFromLocal].Increment(); return ProactiveCopyResult.CopyNotRequiredResult; } else { getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure(); _counters[Counters.GetLocationsSatisfiedFromRemote].Increment(); } if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { return ProactiveCopyResult.CopyNotRequiredResult; } IReadOnlyList <MachineLocation> buildRingMachines = null; // Get random machine inside build ring Task <BoolResult> insideRingCopyTask; if ((Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) != 0) { if (_buildIdHash != null) { buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations; var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray(); if (candidates.Length > 0) { var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines)."); insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } else { insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring.")); } } else { insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found.")); } } else { insideRingCopyTask = BoolResult.SuccessTask; } buildRingMachines ??= new[] { LocalCacheRootMachineLocation }; Task <BoolResult> outsideRingCopyTask; if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) != 0) { var fromPredictionStore = true; Result <MachineLocation> getLocationResult = null; if (_predictionStore != null && path != null) { var machines = _predictionStore.GetTargetMachines(context, path); if (machines?.Count > 0) { var index = ThreadSafeRandom.Generator.Next(0, machines.Count); getLocationResult = new Result <MachineLocation>(new MachineLocation(machines[index])); } } if (getLocationResult == null) { getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines); fromPredictionStore = false; } if (getLocationResult.Succeeded) { var candidate = getLocationResult.Value; Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring. Candidate gotten from {(fromPredictionStore ? nameof(RocksDbContentPlacementPredictionStore) : nameof(ContentLocationStore))}"); outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } else { outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult)); } } else { outsideRingCopyTask = BoolResult.SuccessTask; } return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask); } finally { _pendingProactivePuts.Remove(hash); } }));
private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash) { if (!_pendingProactivePuts.Add(hash)) { return(BoolResult.SuccessTask); } return(context.PerformOperationAsync( Tracer, traceErrorsOnly: true, operation: async() => { try { var hashArray = _buildIdHash != null ? new[] { hash, _buildIdHash.Value } : new[] { hash }; // First check in local location store, then global if failed. var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local); if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { _counters[Counters.GetLocationsSatisfiedFromLocal].Increment(); return BoolResult.Success; } else { getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure(); _counters[Counters.GetLocationsSatisfiedFromRemote].Increment(); } if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { return BoolResult.Success; } IReadOnlyList <MachineLocation> buildRingMachines; Task <BoolResult> copyToBuildRingMachineTask = BoolResult.SuccessTask; // Get random machine inside build ring if (_buildIdHash != null) { buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations; var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray(); if (candidates.Length > 0) { var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines)."); copyToBuildRingMachineTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } } else { buildRingMachines = new[] { LocalCacheRootMachineLocation }; } BoolResult result = BoolResult.Success; var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines); if (getLocationResult.Succeeded) { var candidate = getLocationResult.Value; Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' outside build ring."); result &= await DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } return result & await copyToBuildRingMachineTask; } finally { _pendingProactivePuts.Remove(hash); } })); }