private void Kill(OperationContext context)
{
context
.WithoutCancellationToken() // Not using the cancellation token from the context.
.PerformOperation(
Tracer,
() =>
{
// Using Result<string> for tracing purposes.
if (HasExited)
{
OnExited(context, "TerminateServiceAlreadyExited");
return(Result.Success("AlreadyExited"));
}
_process.Kill(context);
return(Result.Success("ProcessKilled"));
},
extraStartMessage: $"ProcessId={ProcessId}, ServiceId={ServiceId}",
messageFactory: r => $"ProcessId={ProcessId}, ServiceId={ServiceId}, {r}")
.IgnoreFailure();
// Intentionally trying to set the result that indicates the cancellation after PerformOperation call that will never throw.
_processExitSource.TrySetResult(-1);
}
/// <nodoc />
public Task <PluginResponseResult <bool> > StartAsync()
{
var res = PluginClient.StartAsync();
m_startCompletionTaskSoure.TrySetResult(Unit.Void);
return(res);
}
/// <summary>
/// Creates an instance of <see cref="AsyncProcessExecutor"/>.
/// </summary>
public AsyncProcessExecutor(
Process process,
TimeSpan timeout,
Action <string> outputBuilder = null,
Action <string> errorBuilder = null,
SandboxedProcessInfo sandboxedProcessInfo = null)
{
Contract.Requires(process != null);
Process = process;
Process.Exited += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
if (outputBuilder != null)
{
process.OutputDataReceived += (sender, e) => FeedOutputBuilder(m_stdoutFlushedTcs, e.Data, outputBuilder);
}
if (errorBuilder != null)
{
process.ErrorDataReceived += (sender, e) => FeedOutputBuilder(m_stderrFlushedTcs, e.Data, errorBuilder);
}
m_timeout = timeout;
m_sandboxedProcessInfo = sandboxedProcessInfo;
}
private static async Task FeedStandardInputAsync(DetouredProcess detouredProcess, TextReader reader, TaskSourceSlim <bool> stdInTcs)
{
try
{
// We always have a redirected handle, and we always close it eventually (even if we have no actual input to feed)
if (reader != null)
{
while (true)
{
string line = await reader.ReadLineAsync();
if (line == null)
{
break;
}
await detouredProcess.WriteStandardInputLineAsync(line);
}
}
detouredProcess.CloseStandardInput();
stdInTcs.TrySetResult(true);
}
catch (Exception e)
{
stdInTcs.TrySetException(e);
}
}
/// <summary>
/// Creates an instance of <see cref="AsyncProcessExecutor"/>.
/// </summary>
public AsyncProcessExecutor(
Process process,
TimeSpan timeout,
Action <string> outputBuilder = null,
Action <string> errorBuilder = null,
string provenance = null,
Action <string> logger = null)
{
Contract.RequiresNotNull(process);
m_logger = logger;
Process = process;
Process.Exited += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
if (outputBuilder != null)
{
process.OutputDataReceived += (sender, e) => FeedOutputBuilder(m_stdoutFlushedTcs, e.Data, outputBuilder);
}
if (errorBuilder != null)
{
process.ErrorDataReceived += (sender, e) => FeedOutputBuilder(m_stderrFlushedTcs, e.Data, errorBuilder);
}
m_timeout = timeout;
m_provenance = provenance;
}
/// <summary>
/// Waits for the process' standard output and error to get flushed.
/// </summary>
/// <remarks>
/// Note that this task completes as soon as <see cref="Process"/> exits.
/// After <see cref="Process"/> exits, however, any of its child processes might still be running,
/// and might still be using their parent's stdout and stderr, which is why this task is not
/// going to necessarily complete right after <see cref="WaitForExitAsync"/> completes.
///
/// Note also that no timeout is applied here, i.e., if those child processes never exit,
/// this task never completes.
/// </remarks>
public Task WaitForStdOutAndStdErrAsync()
{
Log($"waiting for stderr and stdout to flush");
if (m_outputBuilder == null)
{
m_stdoutFlushedTcs.TrySetResult(Unit.Void);
}
if (m_errorBuilder == null)
{
m_stderrFlushedTcs.TrySetResult(Unit.Void);
}
return(Task.WhenAll(m_stdoutFlushedTcs.Task, m_stderrFlushedTcs.Task));
}
/// <summary>
/// Completes this block. The completion is propagated to the <see cref="TargetBlock"/>,
/// if so has been requested in the <see cref="LinkTo"/> method.
/// </summary>
public void Complete()
{
if (m_stopTask.TrySetResult(Unit.Void))
{
Verbose("Setting completion result");
}
}
/// <summary>
/// Mutates <see cref="Process"/>.
/// </summary>
protected Process CreateAndSetUpProcess()
{
Contract.Requires(Process == null);
#if !PLATFORM_OSX
if (!FileUtilities.FileExistsNoFollow(ProcessInfo.FileName))
{
ThrowFileDoesNotExist();
}
#else
var mode = GetFilePermissionsForFilePath(ProcessInfo.FileName, followSymlink: false);
if (mode < 0)
{
ThrowFileDoesNotExist();
}
var filePermissions = checked ((FilePermissions)mode);
FilePermissions exePermission = FilePermissions.S_IXUSR;
if (!filePermissions.HasFlag(exePermission))
{
SetFilePermissionsForFilePath(ProcessInfo.FileName, exePermission);
}
#endif
Process = new Process();
Process.StartInfo = new ProcessStartInfo
{
FileName = ProcessInfo.FileName,
Arguments = ProcessInfo.Arguments,
WorkingDirectory = ProcessInfo.WorkingDirectory,
StandardErrorEncoding = m_output.Encoding,
StandardOutputEncoding = m_error.Encoding,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
Process.StartInfo.EnvironmentVariables.Clear();
if (ProcessInfo.EnvironmentVariables != null)
{
foreach (var envKvp in ProcessInfo.EnvironmentVariables.ToDictionary())
{
Process.StartInfo.EnvironmentVariables[envKvp.Key] = envKvp.Value;
}
}
Process.EnableRaisingEvents = true;
Process.OutputDataReceived += (sender, e) => FeedStdOut(m_output, m_stdoutFlushedTcs, e.Data);
Process.ErrorDataReceived += (sender, e) => FeedStdErr(m_error, m_stderrFlushedTcs, e.Data);
Process.Exited += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
return(Process);
void ThrowFileDoesNotExist()
{
ThrowCouldNotStartProcess(I($"File '{ProcessInfo.FileName}' not found"), new Win32Exception(0x2));
}
}
private static void FeedOutputBuilder(SandboxedProcessOutputBuilder output, TaskSourceSlim <Unit> signalCompletion, string line)
{
output.AppendLine(line);
if (line == null)
{
signalCompletion.TrySetResult(Unit.Void);
}
}
public void Run()
{
// Flush the base stream as a part of running this action
m_binaryLogger.m_logStreamWriter.BaseStream.Flush();
// Then signal completion
m_taskSource.TrySetResult(Unit.Void);
}
public async Task SetFinalResult()
{
var taskResult = await BatchExecutionTask;
// "Yielding" execution to another thread to avoid deadlocks.
// SetResult causes a task's continuation to run in the same thread that can cause issues
// because a continuation of BatchExecutionTask could be a synchronous continuation as well.
await Task.Yield();
FinalTaskResult.TrySetResult(taskResult);
}
private Task StopAllPlugins()
{
return(Task.WhenAll(m_plugins.Values.Where(pluginInfoTask => pluginInfoTask.Result.Succeeded)
.Select(pluginInfoTask =>
{
Tracing.Logger.Log.PluginManagerLogMessage(m_loggingContext, $"Stop plugin {pluginInfoTask.Result.Result.Name}-{pluginInfoTask.Result.Result.Id}");
return pluginInfoTask.Result.Result.StopAsync();
}))
.ContinueWith(t => m_pluginStopTask.TrySetResult(Unit.Void)));
}
internal static void FeedOutputBuilder(SandboxedProcessOutputBuilder output, TaskSourceSlim <Unit> signalCompletion, string line)
{
if (signalCompletion.Task.IsCompleted)
{
return;
}
output.AppendLine(line);
if (line == null)
{
signalCompletion.TrySetResult(Unit.Void);
}
}
private Possible <TaskSourceSlim <IIpcResult> > TrySetResult(TaskSourceSlim <IIpcResult> taskSource, Response response)
{
response.Result.Timestamp.Response_AfterSetTime = DateTime.UtcNow;
if (taskSource.TrySetResult(response.Result))
{
Logger.Verbose("Setting response for request id {0} succeeded", response.RequestId);
return(taskSource);
}
else
{
Logger.Verbose("Setting response for request id {0} failed", response.RequestId);
return(new Failure <string>("Result was already set for request ID = " + response.RequestId));
}
}
private static void FeedOutputBuilder(TaskSourceSlim <Unit> signalCompletion, string line, Action <string> eat)
{
if (signalCompletion.Task.IsCompleted)
{
return;
}
eat(line);
if (line == null)
{
signalCompletion.TrySetResult(Unit.Void);
}
}
/// <summary>
/// Mutates <see cref="Process"/>.
/// </summary>
protected Process CreateAndSetUpProcess()
{
Contract.Requires(Process == null);
if (!FileUtilities.FileExistsNoFollow(ProcessInfo.FileName))
{
ThrowCouldNotStartProcess(I($"File '{ProcessInfo.FileName}' not found"), new Win32Exception(0x2));
}
#if PLATFORM_OSX
// TODO: TASK 1488150
// When targeting macOS and runnung on VSTS agents, we make sure the 'execute bit' has been set on the binary about to be started
// as VSTS VMs currently have issues around file permission preservance
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_NAME")))
{
SetFilePermissionsForFilePath(ProcessInfo.FileName, FilePermissions.S_IRWXU);
}
#endif
Process = new Process();
Process.StartInfo = new ProcessStartInfo
{
FileName = ProcessInfo.FileName,
Arguments = ProcessInfo.Arguments,
WorkingDirectory = ProcessInfo.WorkingDirectory,
StandardErrorEncoding = m_output.Encoding,
StandardOutputEncoding = m_error.Encoding,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
Process.StartInfo.EnvironmentVariables.Clear();
if (ProcessInfo.EnvironmentVariables != null)
{
foreach (var envKvp in ProcessInfo.EnvironmentVariables.ToDictionary())
{
Process.StartInfo.EnvironmentVariables[envKvp.Key] = envKvp.Value;
}
}
Process.EnableRaisingEvents = true;
Process.OutputDataReceived += (sender, e) => FeedStdOut(m_output, m_stdoutFlushedTcs, e.Data);
Process.ErrorDataReceived += (sender, e) => FeedStdErr(m_error, m_stderrFlushedTcs, e.Data);
Process.Exited += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
return(Process);
}
/// <summary>
/// Release pip's resources after worker is done with the task
/// </summary>
public void ReleaseResources(RunnablePip runnablePip)
{
Contract.Assert(runnablePip.AcquiredResourceWorker == this);
runnablePip.AcquiredResourceWorker = null;
var processRunnablePip = runnablePip as ProcessRunnablePip;
if (processRunnablePip != null)
{
if (runnablePip.Step == PipExecutionStep.CacheLookup)
{
Interlocked.Decrement(ref m_acquiredCacheLookupSlots);
OnWorkerResourcesChanged(WorkerResource.AvailableCacheLookupSlots, increased: true);
runnablePip.SetWorker(null);
}
else
{
Contract.Assert(processRunnablePip.Resources.HasValue);
Interlocked.Add(ref m_acquiredProcessSlots, -processRunnablePip.Weight);
var resources = processRunnablePip.Resources.Value;
m_workerSemaphores.ReleaseResources(resources);
OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: true);
}
}
if (runnablePip.PipType == PipType.Ipc)
{
Interlocked.Decrement(ref m_acquiredIpcSlots);
}
if (AcquiredSlots == 0 && Status == WorkerNodeStatus.Stopping)
{
DrainCompletion.TrySetResult(true);
}
}
/// <summary>
/// Kills process.
/// </summary>
public Task KillAsync()
{
Contract.Requires(Process != null);
try
{
if (!Process.HasExited)
{
Process.Kill();
}
}
catch (Exception e) when(e is Win32Exception || e is InvalidOperationException)
{
// thrown if the process doesn't exist (e.g., because it has already completed on its own)
}
m_stdoutFlushedTcs.TrySetResult(Unit.Void);
m_stderrFlushedTcs.TrySetResult(Unit.Void);
Killed = true;
return(WhenExited);
}
/// <summary>
/// Mutates <see cref="Process"/>.
/// </summary>
protected Process CreateAndSetUpProcess()
{
Contract.Requires(Process == null);
if (!FileUtilities.FileExistsNoFollow(ProcessInfo.FileName))
{
ThrowCouldNotStartProcess(I($"File '{ProcessInfo.FileName}' not found"), new Win32Exception(0x2));
}
Process = new Process();
Process.StartInfo = new ProcessStartInfo
{
FileName = ProcessInfo.FileName,
Arguments = ProcessInfo.Arguments,
WorkingDirectory = ProcessInfo.WorkingDirectory,
StandardErrorEncoding = m_output.Encoding,
StandardOutputEncoding = m_error.Encoding,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
Process.StartInfo.EnvironmentVariables.Clear();
if (ProcessInfo.EnvironmentVariables != null)
{
foreach (var envKvp in ProcessInfo.EnvironmentVariables.ToDictionary())
{
Process.StartInfo.EnvironmentVariables[envKvp.Key] = envKvp.Value;
}
}
Process.EnableRaisingEvents = true;
Process.OutputDataReceived += (sender, e) => FeedOutputBuilder(m_output, m_stdoutFlushedTcs, e.Data);
Process.ErrorDataReceived += (sender, e) => FeedOutputBuilder(m_error, m_stderrFlushedTcs, e.Data);
Process.Exited += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
return(Process);
}
/// <inheritdoc />
public virtual Task KillAsync()
{
Contract.Requires(Started);
ProcessDumper.TryDumpProcessAndChildren(ProcessId, ProcessInfo.TimeoutDumpDirectory, out m_dumpCreationException);
try
{
if (!Process.HasExited)
{
Process.Kill();
}
}
catch (Exception e) when(e is Win32Exception || e is InvalidOperationException)
{
// thrown if the process doesn't exist (e.g., because it has already completed on its own)
}
m_stdoutFlushedTcs.TrySetResult(Unit.Void);
m_stderrFlushedTcs.TrySetResult(Unit.Void);
m_killed = true;
return(WhenExited);
}
public bool Set(Func <ExecutionResult> executionResultFactory)
{
return(Completion.TrySetResult(new Lazy <ExecutionResult>(executionResultFactory)));
}
/// <inheritdoc />
public void PostInitializationCompleted(Context context, BoolResult result)
{
context.Debug($"Setting result for post-initialization completion task to '{result}'.");
_postInitializationCompletion.TrySetResult(result);
}
/// <summary> /// Signals the Request as being complete with the given result value. /// </summary> public void Complete(TResult result) => _tcs.TrySetResult(result);
/// <summary>
/// Release pip's resources after worker is done with the task
/// </summary>
public void ReleaseResources(RunnablePip runnablePip, PipExecutionStep nextStep)
{
Contract.Assert(runnablePip.AcquiredResourceWorker == this);
var stepCompleted = runnablePip.Step;
bool isCancelledOrFailed = nextStep == PipExecutionStep.ChooseWorkerCpu || nextStep == PipExecutionStep.HandleResult;
var processRunnablePip = runnablePip as ProcessRunnablePip;
if (processRunnablePip != null)
{
switch (stepCompleted)
{
case PipExecutionStep.CacheLookup:
{
Interlocked.Decrement(ref m_acquiredCacheLookupSlots);
OnWorkerResourcesChanged(WorkerResource.AvailableCacheLookupSlots, increased: true);
runnablePip.SetWorker(null);
runnablePip.AcquiredResourceWorker = null;
break;
}
case PipExecutionStep.MaterializeInputs:
{
Interlocked.Decrement(ref m_acquiredMaterializeInputSlots);
OnWorkerResourcesChanged(WorkerResource.AvailableMaterializeInputSlots, increased: true);
if (isCancelledOrFailed)
{
releaseExecuteProcessSlots();
releasePostProcessSlots();
}
break;
}
case PipExecutionStep.ExecuteProcess:
{
releaseExecuteProcessSlots();
if (isCancelledOrFailed)
{
releasePostProcessSlots();
}
break;
}
case PipExecutionStep.PostProcess:
{
releasePostProcessSlots();
break;
}
}
}
if (runnablePip.PipType == PipType.Ipc)
{
if (stepCompleted == PipExecutionStep.ExecuteNonProcessPip || isCancelledOrFailed)
{
Interlocked.Decrement(ref m_acquiredLightSlots);
runnablePip.SetWorker(null);
runnablePip.AcquiredResourceWorker = null;
}
}
if (AcquiredSlots == 0 && Status == WorkerNodeStatus.Stopping)
{
DrainCompletion.TrySetResult(true);
}
void releaseExecuteProcessSlots()
{
Contract.Assert(processRunnablePip.Resources.HasValue);
if (processRunnablePip.Process.IsLight)
{
Interlocked.Decrement(ref m_acquiredLightSlots);
}
else
{
Interlocked.Add(ref m_acquiredProcessSlots, -processRunnablePip.Weight);
OnWorkerResourcesChanged(WorkerResource.AvailableProcessSlots, increased: true);
}
var resources = processRunnablePip.Resources.Value;
m_workerSemaphores.ReleaseResources(resources);
}
void releasePostProcessSlots()
{
Interlocked.Decrement(ref m_acquiredPostProcessSlots);
runnablePip.SetWorker(null);
runnablePip.AcquiredResourceWorker = null;
}
}
private async Task <PipResult> StartServiceOrShutdownServiceAsync(
IPipExecutionEnvironment environment,
PipId pipId,
TaskSourceSlim <bool> serviceLaunchCompletion,
bool isStartup)
{
// Ensure task does not block current thread
await Task.Yield();
var loggingContext = m_executePhaseLoggingContext;
try
{
var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId);
if (isStartup)
{
Logger.Log.ScheduleServicePipStarting(
loggingContext,
serviceProcess.GetDescription(m_context));
Interlocked.Increment(ref m_runningServicesCount);
}
using (var operationContext = m_operationTracker.StartOperation(
isStartup
? PipExecutorCounter.ExecuteServiceDuration
: PipExecutorCounter.ExecuteServiceShutdownDuration,
pipId,
PipType.Process,
loggingContext))
{
var serviceStartTask = PipExecutor.ExecuteServiceStartOrShutdownAsync(
#pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call
// Bug #1155822: There is a race condition where service start/shutdown can
// cause crash in the operation tracker because the parent operation is already completed
// this is not fully understood, but the tracking of details of services operations is not
// important so this disables it
OperationContext.CreateUntracked(loggingContext),
#pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call
environment,
serviceProcess,
processId =>
{
serviceLaunchCompletion.TrySetResult(isStartup);
});
using (
operationContext.StartAsyncOperation(
isStartup
? PipExecutorCounter.ExecuteServiceStartupLaunchDuration
: PipExecutorCounter.ExecuteServiceShutdownLaunchDuration))
{
Analysis.IgnoreResult(
await Task.WhenAny(serviceLaunchCompletion.Task, serviceStartTask),
justification: "Task<Task> doesn't contain any data."
);
}
var result = await serviceStartTask;
return(PipResult.CreateWithPointPerformanceInfo(result.Result));
}
}
finally
{
// Attempt to set the result to false to indicate service startup failure.
// This will not succeed if the service is already started
// and the result is set to true.
if (serviceLaunchCompletion.TrySetResult(false))
{
var serviceProcess = HydrateServiceStartOrShutdownProcess(pipId);
Logger.Log.ScheduleServiceTerminatedBeforeStartupWasSignaled(
loggingContext,
serviceProcess.GetDescription(m_context));
}
}
}
