/// <inheritdoc/>
public override Task <RpcResponse> AttachCompleted(AttachCompletionInfo message, ServerCallContext context)
{
var bondMessage = message.ToOpenBond();
m_masterService.AttachCompleted(bondMessage);
return(Task.FromResult(new RpcResponse()));
}
public static OpenBond.AttachCompletionInfo ToOpenBond(this AttachCompletionInfo message)
{
return(new OpenBond.AttachCompletionInfo()
{
WorkerId = message.WorkerId,
AvailableRamMb = message.AvailableRamMb,
MaxConcurrency = message.MaxConcurrency,
WorkerCacheValidationContentHash = message.WorkerCacheValidationContentHash.ToBondContentHash(),
});
}
private async Task <bool> SendAttachCompletedAfterProcessBuildRequestStartedAsync()
{
if (!m_isGrpcEnabled)
{
#if !DISABLE_FEATURE_BOND_RPC
m_bondMasterClient.Start(m_services, OnConnectionTimeOutAsync);
#endif
}
var cacheValidationContent = Guid.NewGuid().ToByteArray();
var cacheValidationContentHash = ContentHashingUtilities.HashBytes(cacheValidationContent);
var possiblyStored = await m_environment.Cache.ArtifactContentCache.TryStoreAsync(
new MemoryStream(cacheValidationContent),
cacheValidationContentHash);
if (!possiblyStored.Succeeded)
{
Logger.Log.DistributionFailedToStoreValidationContentToWorkerCacheWithException(
m_appLoggingContext,
cacheValidationContentHash.ToHex(),
possiblyStored.Failure.DescribeIncludingInnerFailures());
Exit(timedOut: true, "Failed to validate retrieve content from master via cache");
return(false);
}
var attachCompletionInfo = new AttachCompletionInfo
{
WorkerId = WorkerId,
MaxConcurrency = m_maxProcesses,
AvailableRamMb = m_scheduler.LocalWorker.TotalMemoryMb,
WorkerCacheValidationContentHash = cacheValidationContentHash.ToBondContentHash(),
};
Contract.Assert(attachCompletionInfo.WorkerCacheValidationContentHash != null, "worker cache validation content hash is null");
var attachCompletionResult = await m_masterClient.AttachCompletedAsync(attachCompletionInfo);
if (!attachCompletionResult.Succeeded)
{
Logger.Log.DistributionInactiveMaster(m_appLoggingContext, (int)attachCompletionResult.Duration.TotalMinutes);
Exit(timedOut: true, "Failed to attach to master");
return(true);
}
else
{
m_notifyMasterExecutionLogTarget = new NotifyMasterExecutionLogTarget(WorkerId, m_masterClient, m_environment.Context, m_scheduler.PipGraph.GraphId, m_scheduler.PipGraph.MaxAbsolutePathIndex, m_services);
m_scheduler.AddExecutionLogTarget(m_notifyMasterExecutionLogTarget);
m_sendThread.Start();
}
return(true);
}
public static OpenBond.AttachCompletionInfo ToOpenBond(this AttachCompletionInfo message)
{
return(new OpenBond.AttachCompletionInfo()
{
WorkerId = message.WorkerId,
AvailableRamMb = message.AvailableRamMb,
AvailableCommitMb = message.AvailableCommitMb,
MaxProcesses = message.MaxProcesses,
MaxMaterialize = message.MaxMaterialize,
WorkerCacheValidationContentHash = message.WorkerCacheValidationContentHash.ToBondContentHash(),
});
}
private async Task <bool> SendAttachCompletedAfterProcessBuildRequestStartedAsync()
{
var cacheValidationContent = Guid.NewGuid().ToByteArray();
var cacheValidationContentHash = ContentHashingUtilities.HashBytes(cacheValidationContent);
var possiblyStored = await m_environment.Cache.ArtifactContentCache.TryStoreAsync(
new MemoryStream(cacheValidationContent),
cacheValidationContentHash);
if (!possiblyStored.Succeeded)
{
Logger.Log.DistributionFailedToStoreValidationContentToWorkerCacheWithException(
m_appLoggingContext,
cacheValidationContentHash.ToHex(),
possiblyStored.Failure.DescribeIncludingInnerFailures());
Exit("Failed to validate retrieve content from master via cache", isUnexpected: true);
return(false);
}
var attachCompletionInfo = new AttachCompletionInfo
{
WorkerId = WorkerId,
MaxProcesses = m_config.Schedule.MaxProcesses,
MaxMaterialize = m_config.Schedule.MaxMaterialize,
AvailableRamMb = m_scheduler.LocalWorker.TotalRamMb,
AvailableCommitMb = m_scheduler.LocalWorker.TotalCommitMb,
WorkerCacheValidationContentHash = cacheValidationContentHash.ToBondContentHash(),
};
Contract.Assert(attachCompletionInfo.WorkerCacheValidationContentHash != null, "worker cache validation content hash is null");
var attachCompletionResult = await m_masterClient.AttachCompletedAsync(attachCompletionInfo);
if (!attachCompletionResult.Succeeded)
{
Exit($"Failed to attach to master. Duration: {(int)attachCompletionResult.Duration.TotalMinutes}", isUnexpected: true);
return(true);
}
else
{
m_notifyMasterExecutionLogTarget = new NotifyMasterExecutionLogTarget(WorkerId, m_masterClient, m_environment.Context, m_scheduler.PipGraph.GraphId, m_scheduler.PipGraph.MaxAbsolutePathIndex, m_services);
m_scheduler.AddExecutionLogTarget(m_notifyMasterExecutionLogTarget);
m_sendThread.Start();
}
return(true);
}
public async void AttachCompletedAsync(AttachCompletionInfo attachCompletionInfo)
{
Contract.Requires(attachCompletionInfo != null);
// There is a nearly impossible race condition where the node may still be
// in the Starting state (i.e. waiting for ACK of Attach call) so we try to transition
// from Starting AND Started
var isStatusUpdated = ChangeStatus(WorkerNodeStatus.Starting, WorkerNodeStatus.Attached);
isStatusUpdated |= ChangeStatus(WorkerNodeStatus.Started, WorkerNodeStatus.Attached);
if (!isStatusUpdated || m_workerClient == null)
{
// If the status is not changed to Attached due to the current status,
// then no need to validate cache connection.
// The worker might have already validated the cache and it is running.
// Or the worker might have been stopped due to the master termination.
return;
}
m_cacheValidationContentHash = attachCompletionInfo.WorkerCacheValidationContentHash;
TotalProcessSlots = attachCompletionInfo.MaxConcurrency;
TotalMemoryMb = attachCompletionInfo.AvailableRamMb;
var validateCacheSuccess = await ValidateCacheConnection();
if (validateCacheSuccess)
{
ChangeStatus(WorkerNodeStatus.Attached, WorkerNodeStatus.Running);
Volatile.Write(ref m_everAvailable, true);
m_sendThread.Start();
}
else
{
await FinishAsync("ValidateCacheConnection failed");
}
}
/// <summary>
/// Completes the attachment of a worker.
/// </summary>
public void AttachCompleted(AttachCompletionInfo attachCompletionInfo)
{
var worker = GetWorkerById(attachCompletionInfo.WorkerId);
worker.AttachCompletedAsync(attachCompletionInfo);
}
/// <summary>
/// Completes the attachment of a worker.
/// </summary>
void IOrchestratorService.AttachCompleted(AttachCompletionInfo attachCompletionInfo)
{
var worker = GetWorkerById(attachCompletionInfo.WorkerId);
worker.AttachCompletedAsync(attachCompletionInfo);
}