private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash)
{
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
var hashArray = new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
}
else
{
getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global);
if (getLocationsResult.Succeeded)
{
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
}
if (!getLocationsResult.Succeeded)
{
return new BoolResult(getLocationsResult);
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return BoolResult.Success;
}
var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: LocalCacheRootMachineLocation);
if (!getLocationResult.Succeeded)
{
return new BoolResult(getLocationResult);
}
return await DistributedCopier.RequestCopyFileAsync(context, hash, getLocationResult.Value);
}));
}
internal void DistributedService
(
[Description("Cache name")] string cacheName,
[Description("Cache root path")] string cachePath,
[DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(GrpcPortDescription)] uint grpcPort,
[DefaultValue(null), Description("Writable directory for service operations (use CWD if null)")] string dataRootPath,
[DefaultValue(null), Description("Identifier for the stamp this service will run as")] string stampId,
[DefaultValue(null), Description("Identifier for the ring this service will run as")] string ringId,
[DefaultValue(Constants.OneMB), Description("Max size quota in MB")] int maxSizeQuotaMB
)
{
Initialize();
try
{
var cancellationTokenSource = new CancellationTokenSource();
Console.CancelKeyPress += (sender, args) =>
{
cancellationTokenSource.Cancel();
};
var copier = new DistributedCopier();
var pathTransformer = new DistributedPathTransformer();
var host = new HostInfo(stampId, ringId, new List <string>());
var arguments = CreateDistributedCacheServiceArguments(
copier: copier,
pathTransformer: pathTransformer,
host: host,
cacheName: cacheName,
cacheRootPath: cachePath,
grpcPort: grpcPort,
maxSizeQuotaMB: maxSizeQuotaMB,
dataRootPath: dataRootPath,
ct: cancellationTokenSource.Token);
DistributedCacheServiceFacade.RunAsync(arguments).GetAwaiter().GetResult();
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
private Task <ProactiveCopyResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash, string path = null)
{
if (!_pendingProactivePuts.Add(hash))
{
return(Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult));
}
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
try
{
var hashArray = _buildIdHash != null
? new[] { hash, _buildIdHash.Value }
: new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
return ProactiveCopyResult.CopyNotRequiredResult;
}
else
{
getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure();
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return ProactiveCopyResult.CopyNotRequiredResult;
}
IReadOnlyList <MachineLocation> buildRingMachines = null;
// Get random machine inside build ring
Task <BoolResult> insideRingCopyTask;
if ((Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) != 0)
{
if (_buildIdHash != null)
{
buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations;
var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray();
if (candidates.Length > 0)
{
var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)];
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines).");
insideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("Could not find any machines belonging to the build ring."));
}
}
else
{
insideRingCopyTask = Task.FromResult(new BoolResult("BuildId was not specified, so machines in the build ring cannot be found."));
}
}
else
{
insideRingCopyTask = BoolResult.SuccessTask;
}
buildRingMachines ??= new[] { LocalCacheRootMachineLocation };
Task <BoolResult> outsideRingCopyTask;
if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) != 0)
{
var fromPredictionStore = true;
Result <MachineLocation> getLocationResult = null;
if (_predictionStore != null && path != null)
{
var machines = _predictionStore.GetTargetMachines(context, path);
if (machines?.Count > 0)
{
var index = ThreadSafeRandom.Generator.Next(0, machines.Count);
getLocationResult = new Result <MachineLocation>(new MachineLocation(machines[index]));
}
}
if (getLocationResult == null)
{
getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines);
fromPredictionStore = false;
}
if (getLocationResult.Succeeded)
{
var candidate = getLocationResult.Value;
Tracer.Info(context, $"{nameof(RequestProactiveCopyIfNeededAsync)}: Copying {hash.ToShortString()} to machine '{candidate}' outside build ring. Candidate gotten from {(fromPredictionStore ? nameof(RocksDbContentPlacementPredictionStore) : nameof(ContentLocationStore))}");
outsideRingCopyTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
else
{
outsideRingCopyTask = Task.FromResult(new BoolResult(getLocationResult));
}
}
else
{
outsideRingCopyTask = BoolResult.SuccessTask;
}
return new ProactiveCopyResult(await insideRingCopyTask, await outsideRingCopyTask);
}
finally
{
_pendingProactivePuts.Remove(hash);
}
}));
private Task <BoolResult> RequestProactiveCopyIfNeededAsync(OperationContext context, ContentHash hash)
{
if (!_pendingProactivePuts.Add(hash))
{
return(BoolResult.SuccessTask);
}
return(context.PerformOperationAsync(
Tracer,
traceErrorsOnly: true,
operation: async() =>
{
try
{
var hashArray = _buildIdHash != null
? new[] { hash, _buildIdHash.Value }
: new[] { hash };
// First check in local location store, then global if failed.
var getLocationsResult = await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local);
if (getLocationsResult.Succeeded && getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
_counters[Counters.GetLocationsSatisfiedFromLocal].Increment();
return BoolResult.Success;
}
else
{
getLocationsResult += await ContentLocationStore.GetBulkAsync(context, hashArray, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailure();
_counters[Counters.GetLocationsSatisfiedFromRemote].Increment();
}
if (getLocationsResult.ContentHashesInfo[0].Locations.Count > Settings.ProactiveCopyLocationsThreshold)
{
return BoolResult.Success;
}
IReadOnlyList <MachineLocation> buildRingMachines;
Task <BoolResult> copyToBuildRingMachineTask = BoolResult.SuccessTask;
// Get random machine inside build ring
if (_buildIdHash != null)
{
buildRingMachines = getLocationsResult.ContentHashesInfo[getLocationsResult.ContentHashesInfo.Count - 1].Locations;
var candidates = buildRingMachines.Where(m => !m.Equals(LocalCacheRootMachineLocation)).ToArray();
if (candidates.Length > 0)
{
var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)];
Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' in build ring (of {candidates.Length} machines).");
copyToBuildRingMachineTask = DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
}
else
{
buildRingMachines = new[] { LocalCacheRootMachineLocation };
}
BoolResult result = BoolResult.Success;
var getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: buildRingMachines);
if (getLocationResult.Succeeded)
{
var candidate = getLocationResult.Value;
Tracer.Info(context, $"Copying {hash.ToShortString()} to machine '{candidate}' outside build ring.");
result &= await DistributedCopier.RequestCopyFileAsync(context, hash, candidate);
}
return result & await copyToBuildRingMachineTask;
}
finally
{
_pendingProactivePuts.Remove(hash);
}
}));
}
