/// <summary>
/// Executes a put operation, while providing the logic to retrieve the bytes that were put through a RecordingStream.
/// RecordingStream makes it possible to see the actual bytes that are being read by the inner ContentSession.
/// </summary>
private async Task <PutResult> PutCoreAsync(
OperationContext context,
Func <IDecoratedStreamContentSession, Func <Stream, Stream>, Task <PutResult> > putRecordedAsync,
Func <IContentSession, Task <PutResult> > putAsync)
{
PutResult result;
if (ContentLocationStore.AreBlobsSupported && Inner is IDecoratedStreamContentSession decoratedStreamSession)
{
RecordingStream recorder = null;
result = await putRecordedAsync(decoratedStreamSession, stream =>
{
if (stream.CanSeek && stream.Length <= ContentLocationStore.MaxBlobSize)
{
recorder = new RecordingStream(inner: stream, size: stream.Length);
return(recorder);
}
return(stream);
});
if (result && recorder != null)
{
// Fire and forget since this step is optional.
await ContentLocationStore.PutBlobAsync(context, result.ContentHash, recorder.RecordedBytes).FireAndForgetAndReturnTask(context);
}
}
else
{
result = await putAsync(Inner);
}
return(await RegisterPutAsync(context, context.Token, UrgencyHint.Nominal, result));
}
/// <summary>
/// Executes a put operation, while providing the logic to retrieve the bytes that were put through a RecordingStream.
/// RecordingStream makes it possible to see the actual bytes that are being read by the inner ContentSession.
/// </summary>
private async Task <PutResult> PutCoreAsync(
OperationContext context,
Func <IDecoratedStreamContentSession, Func <Stream, Stream>, Task <PutResult> > putRecordedAsync,
Func <IContentSession, Task <PutResult> > putAsync,
string path = null)
{
PutResult result;
bool putBlob = false;
if (ContentLocationStore.AreBlobsSupported && Inner is IDecoratedStreamContentSession decoratedStreamSession)
{
RecordingStream recorder = null;
result = await putRecordedAsync(decoratedStreamSession, stream =>
{
if (stream.CanSeek && stream.Length <= ContentLocationStore.MaxBlobSize)
{
recorder = new RecordingStream(inner: stream, size: stream.Length);
return(recorder);
}
return(stream);
});
if (result && recorder != null)
{
// Fire and forget since this step is optional.
var putBlobResult = await ContentLocationStore.PutBlobAsync(context, result.ContentHash, recorder.RecordedBytes);
putBlob = putBlobResult.Succeeded;
}
}
else
{
result = await putAsync(Inner);
}
if (!result)
{
return(result);
}
// It is important to register location before requesting the proactive copy; otherwise, we can fail the proactive copy.
var registerResult = await RegisterPutAsync(context, UrgencyHint.Nominal, result);
// Only perform proactive copy to other machines if we didn't put the blob into Redis and we succeeded in registering our location.
if (!putBlob && registerResult && Settings.ProactiveCopyMode != ProactiveCopyMode.Disabled)
{
// Since the rest of the operation is done asynchronously, create new context to stop cancelling operation prematurely.
WithOperationContext(
context,
CancellationToken.None,
operationContext => RequestProactiveCopyIfNeededAsync(operationContext, result.ContentHash, path)
).FireAndForget(context);
}
return(registerResult);
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
await base.StartupCoreAsync(context).ThrowIfFailure();
if (Constants.TryExtractBuildId(Name, out _buildId) && Guid.TryParse(_buildId, out var buildIdGuid))
{
// Generate a fake hash for the build and register a content entry in the location store to represent
// machines in the build ring
_buildIdHash = new ContentHash(HashType.MD5, buildIdGuid.ToByteArray());
Tracer.Info(context, $"Registering machine with build {_buildId} (build id hash: {_buildIdHash.Value.ToShortString()}");
await ContentLocationStore.RegisterLocalLocationAsync(context, new[] { new ContentHashWithSize(_buildIdHash.Value, _buildId.Length) }, context.Token, UrgencyHint.Nominal).ThrowIfFailure();
}
return(BoolResult.Success);
}
/// <summary>
/// Executes a put operation, while providing the logic to retrieve the bytes that were put through a RecordingStream.
/// RecordingStream makes it possible to see the actual bytes that are being read by the inner ContentSession.
/// </summary>
private async Task <PutResult> PutCoreAsync(
OperationContext context,
Func <IDecoratedStreamContentSession, Func <Stream, Stream>, Task <PutResult> > putRecordedAsync,
Func <IContentSession, Task <PutResult> > putAsync)
{
PutResult result;
if (ContentLocationStore.AreBlobsSupported && Inner is IDecoratedStreamContentSession decoratedStreamSession)
{
RecordingStream recorder = null;
result = await putRecordedAsync(decoratedStreamSession, stream =>
{
if (stream.CanSeek && stream.Length <= ContentLocationStore.MaxBlobSize)
{
recorder = new RecordingStream(inner: stream, size: stream.Length);
return(recorder);
}
return(stream);
});
if (result && recorder != null)
{
// Fire and forget since this step is optional.
await ContentLocationStore.PutBlobAsync(context, result.ContentHash, recorder.RecordedBytes).FireAndForgetAndReturnTask(context);
}
}
else
{
result = await putAsync(Inner);
}
var putResult = await RegisterPutAsync(context, UrgencyHint.Nominal, result);
if (putResult.Succeeded && Settings.EnableProactiveCopy)
{
// Since the rest of the operation is done asynchronously, create new context to stop cancelling operation prematurely.
WithOperationContext(
context,
CancellationToken.None,
operationContext => RequestProactiveCopyIfNeededAsync(operationContext, putResult.ContentHash)
).FireAndForget(context);
}
return(putResult);
}
private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash)
{
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
var hashArray = new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
}
else
{
getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global);
if (getLocationsResult.Succeeded)
{
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
}
if (!getLocationsResult.Succeeded)
{
return new BoolResult(getLocationsResult);
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return BoolResult.Success;
}
var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: LocalCacheRootMachineLocation);
if (!getLocationResult.Succeeded)
{
return new BoolResult(getLocationResult);
}
return await DistributedCopier.RequestCopyFileAsync(context, hash, getLocationResult.Value);
}));
}
private async Task <PutResult> RegisterPutAsync(OperationContext context, UrgencyHint urgencyHint, PutResult putResult)
{
if (putResult.Succeeded)
{
var updateResult = await ContentLocationStore.RegisterLocalLocationAsync(
context,
new[] { new ContentHashWithSize(putResult.ContentHash, putResult.ContentSize) },
context.Token,
urgencyHint);
if (!updateResult.Succeeded)
{
return(new PutResult(updateResult, putResult.ContentHash));
}
}
return(putResult);
}
private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash, string path = null)
{
if (!_pendingProactivePuts.Add(hash))
{
return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult));
}
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
try
{
var hashArray = _buildIdHash != null
? new[] { hash, _buildIdHash.Value }
: new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
return ProactiveCopyResult.CopyNotRequiredResult;
}
else
{
getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure();
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return ProactiveCopyResult.CopyNotRequiredResult;
}
IReadOnlyList <MachineLocation> buildRingMachines = null;
// Get random machine inside build ring
Task <BoolResult> insideRingCopyTask;
if ((Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) != 0)
{
if (_buildIdHash != null)
{
buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations;
var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray();
if (candidates.Length > 0)
{
var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)];
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines).");
insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring."));
}
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found."));
}
}
else
{
insideRingCopyTask = BoolResult.SuccessTask;
}
buildRingMachines ??= new[] { LocalCacheRootMachineLocation };
Task <BoolResult> outsideRingCopyTask;
if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) != 0)
{
var fromPredictionStore = true;
Result <MachineLocation> getLocationResult = null;
if (_predictionStore != null && path != null)
{
var machines = _predictionStore.GetTargetMachines(context, path);
if (machines?.Count > 0)
{
var index = ThreadSafeRandom.Generator.Next(0, machines.Count);
getLocationResult = new Result <MachineLocation>(new MachineLocation(machines[index]));
}
}
if (getLocationResult == null)
{
getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines);
fromPredictionStore = false;
}
if (getLocationResult.Succeeded)
{
var candidate = getLocationResult.Value;
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring. Candidate gotten from {(fromPredictionStore ? nameof(RocksDbContentPlacementPredictionStore) : nameof(ContentLocationStore))}");
outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult));
}
}
else
{
outsideRingCopyTask = BoolResult.SuccessTask;
}
return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask);
}
finally
{
_pendingProactivePuts.Remove(hash);
}
}));
private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash)
{
if (!_pendingProactivePuts.Add(hash))
{
return(BoolResult.SuccessTask);
}
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
try
{
var hashArray = _buildIdHash != null
? new[] { hash, _buildIdHash.Value }
: new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
return BoolResult.Success;
}
else
{
getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure();
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return BoolResult.Success;
}
IReadOnlyList <MachineLocation> buildRingMachines;
Task <BoolResult> copyToBuildRingMachineTask = BoolResult.SuccessTask;
// Get random machine inside build ring
if (_buildIdHash != null)
{
buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations;
var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray();
if (candidates.Length > 0)
{
var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)];
Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines).");
copyToBuildRingMachineTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
}
else
{
buildRingMachines = new[] { LocalCacheRootMachineLocation };
}
BoolResult result = BoolResult.Success;
var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines);
if (getLocationResult.Succeeded)
{
var candidate = getLocationResult.Value;
Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' outside build ring.");
result &= await DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
return result & await copyToBuildRingMachineTask;
}
finally
{
_pendingProactivePuts.Remove(hash);
}
}));
}
