/// <summary> /// Executes a put operation, while providing the logic to retrieve the bytes that were put through a RecordingStream. /// RecordingStream makes it possible to see the actual bytes that are being read by the inner ContentSession. /// </summary> private async Task <PutResult> PutCoreAsync( OperationContext context, Func <IDecoratedStreamContentSession, Func <Stream, Stream>, Task <PutResult> > putRecordedAsync, Func <IContentSession, Task <PutResult> > putAsync) { PutResult result; if (ContentLocationStore.AreBlobsSupported && Inner is IDecoratedStreamContentSession decoratedStreamSession) { RecordingStream recorder = null; result = await putRecordedAsync(decoratedStreamSession, stream => { if (stream.CanSeek && stream.Length <= ContentLocationStore.MaxBlobSize) { recorder = new RecordingStream(inner: stream, size: stream.Length); return(recorder); } return(stream); }); if (result && recorder != null) { // Fire and forget since this step is optional. await ContentLocationStore.PutBlobAsync(context, result.ContentHash, recorder.RecordedBytes).FireAndForgetAndReturnTask(context); } } else { result = await putAsync(Inner); } return(await RegisterPutAsync(context, context.Token, UrgencyHint.Nominal, result)); }
/// <summary> /// Executes a put operation, while providing the logic to retrieve the bytes that were put through a RecordingStream. /// RecordingStream makes it possible to see the actual bytes that are being read by the inner ContentSession. /// </summary> private async Task <PutResult> PutCoreAsync( OperationContext context, Func <IDecoratedStreamContentSession, Func <Stream, Stream>, Task <PutResult> > putRecordedAsync, Func <IContentSession, Task <PutResult> > putAsync, string path = null) { PutResult result; bool putBlob = false; if (ContentLocationStore.AreBlobsSupported && Inner is IDecoratedStreamContentSession decoratedStreamSession) { RecordingStream recorder = null; result = await putRecordedAsync(decoratedStreamSession, stream => { if (stream.CanSeek && stream.Length <= ContentLocationStore.MaxBlobSize) { recorder = new RecordingStream(inner: stream, size: stream.Length); return(recorder); } return(stream); }); if (result && recorder != null) { // Fire and forget since this step is optional. var putBlobResult = await ContentLocationStore.PutBlobAsync(context, result.ContentHash, recorder.RecordedBytes); putBlob = putBlobResult.Succeeded; } } else { result = await putAsync(Inner); } if (!result) { return(result); } // It is important to register location before requesting the proactive copy; otherwise, we can fail the proactive copy. var registerResult = await RegisterPutAsync(context, UrgencyHint.Nominal, result); // Only perform proactive copy to other machines if we didn't put the blob into Redis and we succeeded in registering our location. if (!putBlob && registerResult && Settings.ProactiveCopyMode != ProactiveCopyMode.Disabled) { // Since the rest of the operation is done asynchronously, create new context to stop cancelling operation prematurely. WithOperationContext( context, CancellationToken.None, operationContext => RequestProactiveCopyIfNeededAsync(operationContext, result.ContentHash, path) ).FireAndForget(context); } return(registerResult); }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { await base.StartupCoreAsync(context).ThrowIfFailure(); if (Constants.TryExtractBuildId(Name, out _buildId) && Guid.TryParse(_buildId, out var buildIdGuid)) { // Generate a fake hash for the build and register a content entry in the location store to represent // machines in the build ring _buildIdHash = new ContentHash(HashType.MD5, buildIdGuid.ToByteArray()); Tracer.Info(context, $"Registering machine with build {_buildId} (build id hash: {_buildIdHash.Value.ToShortString()}"); await ContentLocationStore.RegisterLocalLocationAsync(context, new[] { new ContentHashWithSize(_buildIdHash.Value, _buildId.Length) }, context.Token, UrgencyHint.Nominal).ThrowIfFailure(); } return(BoolResult.Success); }
/// <summary> /// Executes a put operation, while providing the logic to retrieve the bytes that were put through a RecordingStream. /// RecordingStream makes it possible to see the actual bytes that are being read by the inner ContentSession. /// </summary> private async Task <PutResult> PutCoreAsync( OperationContext context, Func <IDecoratedStreamContentSession, Func <Stream, Stream>, Task <PutResult> > putRecordedAsync, Func <IContentSession, Task <PutResult> > putAsync) { PutResult result; if (ContentLocationStore.AreBlobsSupported && Inner is IDecoratedStreamContentSession decoratedStreamSession) { RecordingStream recorder = null; result = await putRecordedAsync(decoratedStreamSession, stream => { if (stream.CanSeek && stream.Length <= ContentLocationStore.MaxBlobSize) { recorder = new RecordingStream(inner: stream, size: stream.Length); return(recorder); } return(stream); }); if (result && recorder != null) { // Fire and forget since this step is optional. await ContentLocationStore.PutBlobAsync(context, result.ContentHash, recorder.RecordedBytes).FireAndForgetAndReturnTask(context); } } else { result = await putAsync(Inner); } var putResult = await RegisterPutAsync(context, UrgencyHint.Nominal, result); if (putResult.Succeeded && Settings.EnableProactiveCopy) { // Since the rest of the operation is done asynchronously, create new context to stop cancelling operation prematurely. WithOperationContext( context, CancellationToken.None, operationContext => RequestProactiveCopyIfNeededAsync(operationContext, putResult.ContentHash) ).FireAndForget(context); } return(putResult); }
private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash) { return(context.PerformOperationAsync( Tracer, traceErrorsOnly: true, operation: async() => { var hashArray = new[] { hash }; // First check in local location store, then global if failed. var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local); if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { _counters[Counters.GetLocationsSatisfiedFromLocal].Increment(); } else { getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global); if (getLocationsResult.Succeeded) { _counters[Counters.GetLocationsSatisfiedFromRemote].Increment(); } } if (!getLocationsResult.Succeeded) { return new BoolResult(getLocationsResult); } if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { return BoolResult.Success; } var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: LocalCacheRootMachineLocation); if (!getLocationResult.Succeeded) { return new BoolResult(getLocationResult); } return await DistributedCopier.RequestCopyFileAsync(context, hash, getLocationResult.Value); })); }
private async Task <PutResult> RegisterPutAsync(OperationContext context, UrgencyHint urgencyHint, PutResult putResult) { if (putResult.Succeeded) { var updateResult = await ContentLocationStore.RegisterLocalLocationAsync( context, new[] { new ContentHashWithSize(putResult.ContentHash, putResult.ContentSize) }, context.Token, urgencyHint); if (!updateResult.Succeeded) { return(new PutResult(updateResult, putResult.ContentHash)); } } return(putResult); }
private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash, string path = null) { if (!_pendingProactivePuts.Add(hash)) { return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult)); } return(context.PerformOperationAsync( Tracer, traceErrorsOnly: true, operation: async() => { try { var hashArray = _buildIdHash != null ? new[] { hash, _buildIdHash.Value } : new[] { hash }; // First check in local location store, then global if failed. var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local); if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { _counters[Counters.GetLocationsSatisfiedFromLocal].Increment(); return ProactiveCopyResult.CopyNotRequiredResult; } else { getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure(); _counters[Counters.GetLocationsSatisfiedFromRemote].Increment(); } if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { return ProactiveCopyResult.CopyNotRequiredResult; } IReadOnlyList <MachineLocation> buildRingMachines = null; // Get random machine inside build ring Task <BoolResult> insideRingCopyTask; if ((Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) != 0) { if (_buildIdHash != null) { buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations; var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray(); if (candidates.Length > 0) { var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines)."); insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } else { insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring.")); } } else { insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found.")); } } else { insideRingCopyTask = BoolResult.SuccessTask; } buildRingMachines ??= new[] { LocalCacheRootMachineLocation }; Task <BoolResult> outsideRingCopyTask; if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) != 0) { var fromPredictionStore = true; Result <MachineLocation> getLocationResult = null; if (_predictionStore != null && path != null) { var machines = _predictionStore.GetTargetMachines(context, path); if (machines?.Count > 0) { var index = ThreadSafeRandom.Generator.Next(0, machines.Count); getLocationResult = new Result <MachineLocation>(new MachineLocation(machines[index])); } } if (getLocationResult == null) { getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines); fromPredictionStore = false; } if (getLocationResult.Succeeded) { var candidate = getLocationResult.Value; Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring. Candidate gotten from {(fromPredictionStore ? nameof(RocksDbContentPlacementPredictionStore) : nameof(ContentLocationStore))}"); outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } else { outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult)); } } else { outsideRingCopyTask = BoolResult.SuccessTask; } return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask); } finally { _pendingProactivePuts.Remove(hash); } }));
private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash) { if (!_pendingProactivePuts.Add(hash)) { return(BoolResult.SuccessTask); } return(context.PerformOperationAsync( Tracer, traceErrorsOnly: true, operation: async() => { try { var hashArray = _buildIdHash != null ? new[] { hash, _buildIdHash.Value } : new[] { hash }; // First check in local location store, then global if failed. var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local); if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { _counters[Counters.GetLocationsSatisfiedFromLocal].Increment(); return BoolResult.Success; } else { getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure(); _counters[Counters.GetLocationsSatisfiedFromRemote].Increment(); } if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold) { return BoolResult.Success; } IReadOnlyList <MachineLocation> buildRingMachines; Task <BoolResult> copyToBuildRingMachineTask = BoolResult.SuccessTask; // Get random machine inside build ring if (_buildIdHash != null) { buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations; var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray(); if (candidates.Length > 0) { var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines)."); copyToBuildRingMachineTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } } else { buildRingMachines = new[] { LocalCacheRootMachineLocation }; } BoolResult result = BoolResult.Success; var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines); if (getLocationResult.Succeeded) { var candidate = getLocationResult.Value; Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' outside build ring."); result &= await DistributedCopier.RequestCopyFileAsync(context, hash, candidate); } return result & await copyToBuildRingMachineTask; } finally { _pendingProactivePuts.Remove(hash); } })); }