/// <inheritdoc />
public override async Task FinishAsync(string buildFailure)
{
m_buildRequests.CompleteAdding();
if (m_sendThread.IsAlive)
{
m_sendThread.Join();
}
bool initiatedStop = false;
while (true)
{
WorkerNodeStatus status = Status;
if (status == WorkerNodeStatus.Stopping || status == WorkerNodeStatus.Stopped)
{
break;
}
if (ChangeStatus(status, WorkerNodeStatus.Stopping))
{
initiatedStop = true;
break;
}
}
if (initiatedStop)
{
CancellationTokenSource exitCancellation = new CancellationTokenSource();
// Only wait a short amount of time for exit (15 seconds) if worker is not successfully attached.
if (m_attachCompletion.Task.Status != TaskStatus.RanToCompletion || !await m_attachCompletion.Task)
{
exitCancellation.CancelAfter(TimeSpan.FromSeconds(15));
}
var buildEndData = new BuildEndData()
{
Failure = buildFailure ?? m_exitFailure
};
await m_workerClient.ExitAsync(buildEndData, exitCancellation.Token);
m_executionBlobQueue.CompleteAdding();
using (m_masterService.Environment.Counters.StartStopwatch(PipExecutorCounter.RemoteWorker_AwaitExecutionBlobCompletionDuration))
{
if (!m_executionBlobQueue.IsCompleted)
{
// Wait for execution blobs to be processed.
await m_executionBlobCompletion.Task;
}
}
ChangeStatus(WorkerNodeStatus.Stopping, WorkerNodeStatus.Stopped);
}
}
public override async void Finish(string buildFailure)
{
m_buildRequests.CompleteAdding();
if (m_sendThread.IsAlive)
{
m_sendThread.Join();
}
bool initiatedStop = false;
while (true)
{
WorkerNodeStatus status = Status;
if (status == WorkerNodeStatus.Stopping || status == WorkerNodeStatus.Stopped)
{
break;
}
if (ChangeStatus(status, WorkerNodeStatus.Stopping))
{
initiatedStop = true;
break;
}
}
if (initiatedStop)
{
CancellationTokenSource exitCancellation = new CancellationTokenSource();
// Only wait a short amount of time for exit (15 seconds) if worker is not successfully attached.
if (m_attachCompletion.Task.Status != TaskStatus.RanToCompletion || !await m_attachCompletion.Task)
{
exitCancellation.CancelAfter(TimeSpan.FromSeconds(15));
}
var buildEndData = new BuildEndData()
{
Failure = buildFailure ?? m_exitFailure
};
await m_workerClient.ExitAsync(buildEndData, exitCancellation.Token);
ChangeStatus(WorkerNodeStatus.Stopping, WorkerNodeStatus.Stopped);
}
}
private bool ChangeStatus(WorkerNodeStatus fromStatus, WorkerNodeStatus toStatus, [CallerMemberName] string callerName = null)
{
if (fromStatus == toStatus)
{
return(false);
}
if (Interlocked.CompareExchange(ref m_status, (int)toStatus, (int)fromStatus) != (int)fromStatus)
{
return(false);
}
if (toStatus == WorkerNodeStatus.Stopped || toStatus == WorkerNodeStatus.Stopping)
{
m_attachCompletion.TrySetResult(false);
}
else if (toStatus == WorkerNodeStatus.Attached)
{
m_attachCompletion.TrySetResult(true);
}
Logger.Log.DistributionWorkerChangedState(
m_appLoggingContext,
m_serviceLocation.IpAddress,
m_serviceLocation.Port,
fromStatus.ToString(),
toStatus.ToString(),
callerName);
// In the case of a stop fail all pending pips and stop the timer.
if (toStatus == WorkerNodeStatus.Stopped)
{
// The worker goes in a stopped state either by a scheduler request or because it lost connection with the
// remote machine. Check which one applies.
bool isLostConnection = fromStatus != WorkerNodeStatus.Stopping;
Stop(isLostConnection);
}
OnStatusChanged();
return(true);
}
