public void SerializeSandboxedProcessInfo()
{
var pt = new PathTable();
var fam =
new FileAccessManifest(pt, CreateDirectoryTranslator())
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false
};
var vac = new ValidationDataCreator(fam, pt);
vac.AddScope(A("C", "Users", "AppData"), FileAccessPolicy.AllowAll);
vac.AddPath(A("C", "Source", "source.txt"), FileAccessPolicy.AllowReadAlways);
vac.AddPath(A("C", "Out", "out.txt"), FileAccessPolicy.AllowAll);
var standardFiles = new SandboxedProcessStandardFiles(A("C", "pip", "pip.out"), A("C", "pip", "pip.err"));
var envVars = new Dictionary <string, string>()
{
["Var1"] = "Val1",
["Var2"] = "Val2",
};
IBuildParameters buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars);
var sidebandLogFile = A("C", "engine-cache", "sideband-logs", "log-1");
var loggerRootDirs = new[] { A("C", "out", "dir1"), A("C", "out", "dir2") };
var sharedOpaqueOutputLogger = new SidebandWriter(DefaultSidebandMetadata, sidebandLogFile, loggerRootDirs);
SandboxedProcessInfo info = new SandboxedProcessInfo(
pt,
new StandardFileStorage(standardFiles),
A("C", "tool", "tool.exe"),
fam,
true,
null,
sidebandWriter: sharedOpaqueOutputLogger)
{
Arguments = @"/arg1:val1 /arg2:val2",
WorkingDirectory = A("C", "Source"),
EnvironmentVariables = buildParameters,
Timeout = TimeSpan.FromMinutes(15),
PipSemiStableHash = 0x12345678,
PipDescription = nameof(SerializeSandboxedProcessInfo),
ProcessIdListener = null,
TimeoutDumpDirectory = A("C", "Timeout"),
SandboxKind = global::BuildXL.Utilities.Configuration.SandboxKind.Default,
AllowedSurvivingChildProcessNames = new[] { "conhost.exe", "mspdbsrv.exe" },
NestedProcessTerminationTimeout = SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
StandardInputSourceInfo = StandardInputInfo.CreateForData("Data"),
StandardObserverDescriptor = new SandboxObserverDescriptor()
{
WarningRegex = new ExpandedRegexDescriptor("*warn", System.Text.RegularExpressions.RegexOptions.Compiled)
},
};
// Serialize and deserialize.
SandboxedProcessInfo readInfo = null;
using (var stream = new MemoryStream())
{
info.Serialize(stream);
stream.Position = 0;
readInfo = SandboxedProcessInfo.Deserialize(
stream,
new global::BuildXL.Utilities.Instrumentation.Common.LoggingContext("Test"),
null);
}
using (readInfo.SidebandWriter)
{
// Verify.
XAssert.AreEqual(info.FileName, readInfo.FileName);
XAssert.AreEqual(info.Arguments, readInfo.Arguments);
XAssert.AreEqual(info.WorkingDirectory, readInfo.WorkingDirectory);
var readEnvVars = readInfo.EnvironmentVariables.ToDictionary();
XAssert.AreEqual(envVars.Count, readEnvVars.Count);
foreach (var kvp in envVars)
{
XAssert.AreEqual(kvp.Value, readEnvVars[kvp.Key]);
}
XAssert.AreEqual(info.Timeout, readInfo.Timeout);
XAssert.AreEqual(info.PipSemiStableHash, readInfo.PipSemiStableHash);
XAssert.AreEqual(info.PipDescription, readInfo.PipDescription);
XAssert.AreEqual(info.ProcessIdListener, readInfo.ProcessIdListener);
XAssert.AreEqual(info.TimeoutDumpDirectory, readInfo.TimeoutDumpDirectory);
XAssert.AreEqual(info.SandboxKind, readInfo.SandboxKind);
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames.Length, readInfo.AllowedSurvivingChildProcessNames.Length);
for (int i = 0; i < info.AllowedSurvivingChildProcessNames.Length; ++i)
{
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames[i], readInfo.AllowedSurvivingChildProcessNames[i]);
}
XAssert.AreEqual(info.NestedProcessTerminationTimeout, readInfo.NestedProcessTerminationTimeout);
XAssert.AreEqual(info.StandardInputSourceInfo, readInfo.StandardInputSourceInfo);
XAssert.IsNotNull(readInfo.SandboxedProcessStandardFiles);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.SandboxedProcessStandardFiles.StandardOutput);
XAssert.AreEqual(standardFiles.StandardError, readInfo.SandboxedProcessStandardFiles.StandardError);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
XAssert.AreEqual(standardFiles.StandardError, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardError));
XAssert.IsFalse(readInfo.ContainerConfiguration.IsIsolationEnabled);
XAssert.AreEqual(sidebandLogFile, readInfo.SidebandWriter.SidebandLogFile);
XAssert.ArrayEqual(loggerRootDirs, readInfo.SidebandWriter.RootDirectories.ToArray());
ValidationDataCreator.TestManifestRetrieval(vac.DataItems, readInfo.FileAccessManifest, false);
}
}
protected static Task <ISandboxedProcess> StartProcessAsync(SandboxedProcessInfo info, bool forceSandboxing = true)
{
return(SandboxedProcessFactory.StartAsync(info, forceSandboxing));
}
/// <summary>
/// Runs the a tool in a sandboxed process and returns the result.
/// These optional callback Actions can be provided:
/// beforeLaunch is invoked right before the process is launched
/// onResult is invoked after getting a successful result
/// </summary>>
public static async Task <SandboxedProcessResult> RunSandboxedToolAsync(FrontEndContext context,
string pathToTool,
string buildStorageDirectory,
FileAccessManifest fileAccessManifest,
string arguments,
string workingDirectory,
string description,
BuildParameters.IBuildParameters buildParameters,
Action beforeLaunch = null, // Invoked right before the process starts
Action onResult = null // Action to be taken after getting a successful result
)
{
var info =
new SandboxedProcessInfo(
context.PathTable,
new ToolBuildStorage(buildStorageDirectory),
pathToTool,
fileAccessManifest,
disableConHostSharing: true,
ContainerConfiguration.DisabledIsolation,
loggingContext: context.LoggingContext)
{
Arguments = arguments,
WorkingDirectory = workingDirectory,
PipSemiStableHash = 0,
PipDescription = "CMakeRunner - Ninja specs and input files generator",
EnvironmentVariables = buildParameters
};
var process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing : true);
var registration = context.CancellationToken.Register(
() =>
{
try
{
process.KillAsync().GetAwaiter().GetResult();
}
catch (TaskCanceledException)
{
// If the process has already terminated or doesn't exist, an TaskCanceledException is raised.
// In either case, we swallow the exception, cancellation is already requested by the user
}
});
beforeLaunch?.Invoke();
var result = process.GetResultAsync().ContinueWith(
r =>
{
// Dispose the registration for the cancellation token once the process is done
#pragma warning disable AsyncFixer02
registration.Dispose();
#pragma warning restore AsyncFixer02
//
onResult?.Invoke();
return(r.GetAwaiter().GetResult());
});
return(await result);
}
private async Task <(ExitCode, SandboxedProcessResult)> ExecuteAsync(SandboxedProcessInfo info)
{
m_logger.LogInfo($"Start execution: {info.GetCommandLine()}");
FileAccessManifest fam = info.FileAccessManifest;
try
{
if (fam.CheckDetoursMessageCount && !OperatingSystemHelper.IsUnixOS)
{
string semaphoreName = !string.IsNullOrEmpty(info.DetoursFailureFile)
? info.DetoursFailureFile.Replace('\\', '_')
: "Detours_" + info.PipSemiStableHash.ToString("X16", CultureInfo.InvariantCulture) + "-" + Guid.NewGuid().ToString();
int maxRetry = 3;
// We check this first due to bug#1873910 when executing in a VM on Cloudbuild.
// Creating this Semaphore may fail due to a semaphore with the same name already existing.
// The reason for this is currently unknown, however since the name for the failures file is
// created in SandboxedProcessPipExecutor.GetDetoursInternalErrorFilePath with a unique guid
// along with the pip hash it should be safe to dispose because the name is unique to this pip.
if (System.Threading.Semaphore.TryOpenExisting(semaphoreName, out var existingSemaphore))
{
m_logger.LogInfo($"Disposing existing semaphore with name '{semaphoreName}'.");
// Calling dispose on this will allow us to create a new semaphore with the same name
existingSemaphore.Dispose();
}
while (!fam.SetMessageCountSemaphore(semaphoreName))
{
m_logger.LogInfo($"Semaphore '{semaphoreName}' for counting Detours messages is already opened");
fam.UnsetMessageCountSemaphore();
--maxRetry;
if (maxRetry == 0)
{
break;
}
semaphoreName += $"_{maxRetry}";
}
if (maxRetry == 0)
{
m_logger.LogError($"Semaphore for counting Detours messages cannot be newly created");
return(ExitCode.FailedSandboxPreparation, null);
}
}
using (Stream standardInputStream = TryOpenStandardInputStream(info, out bool succeedInOpeningStdIn))
{
if (!succeedInOpeningStdIn)
{
return(ExitCode.FailedSandboxPreparation, null);
}
using (StreamReader standardInputReader = standardInputStream == null ? null : new StreamReader(standardInputStream, CharUtilities.Utf8NoBomNoThrow))
{
info.StandardInputReader = standardInputReader;
ISandboxedProcess process = await StartProcessAsync(info);
if (process == null)
{
return(ExitCode.FailedStartProcess, null);
}
SandboxedProcessResult result = await process.GetResultAsync();
// Patch result.
result.WarningCount = m_outputErrorObserver.WarningCount;
result.LastMessageCount = process.GetLastMessageCount();
result.DetoursMaxHeapSize = process.GetDetoursMaxHeapSize();
result.MessageCountSemaphoreCreated = info.FileAccessManifest.MessageCountSemaphore != null;
return(ExitCode.Success, result);
}
}
}
finally
{
fam.UnsetMessageCountSemaphore();
}
}
// TODO: erikmav fails on netcore, need an unmanaged shim to avoid exe/dll [InlineData(false, null)]
// TODO: erikmav [InlineData(true, null)]
// TODO: erikmav [InlineData(true, "cmd.exe")] // Filter should match child
// TODO: erikmav [InlineData(false, "cmd.exe")] // Filter should match child
public async Task CmdWithTestShim(bool useQuotesForChildCmdExe, string processMatch)
{
var context = BuildXLContext.CreateInstanceForTesting();
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = CmdHelper.CmdX64;
string childExecutable = executable;
string quotedExecutable = '"' + executable + '"';
if (useQuotesForChildCmdExe)
{
childExecutable = quotedExecutable;
}
string shimProgram = Path.Combine(currentCodeFolder, "TestSubstituteProcessExecutionShim.exe");
Assert.True(File.Exists(shimProgram), $"Shim test program not found at {shimProgram}");
var shimProgramPath = AbsolutePath.Create(context.PathTable, shimProgram);
var fam =
new FileAccessManifest(context.PathTable)
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false,
SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: true,
processMatch == null ? new ShimProcessMatch[0] : new ShimProcessMatch[] { new ShimProcessMatch(PathAtom.Create(context.StringTable, processMatch), PathAtom.Invalid) })
};
Guid sessionId = Guid.NewGuid();
string sessionIdStr = sessionId.ToString("N");
var loggingContext = new LoggingContext(sessionId, "TestSession", new LoggingContext.SessionInfo(sessionIdStr, "env", sessionId));
string childOutput = "Child cmd that should be shimmed";
string childArgs = $"{childExecutable} /D /C @echo {childOutput}";
// Detours logic should wrap the initial cmd in quotes for easier parsing by shim logic.
string childShimArgs = $"{quotedExecutable} /D /C @echo {childOutput}";
string args = "/D /C echo Top-level cmd. Running child process && " + childArgs;
var stdoutSb = new StringBuilder(128);
var stderrSb = new StringBuilder();
var sandboxedProcessInfo = new SandboxedProcessInfo(
context.PathTable,
new LocalSandboxedFileStorage(),
executable,
disableConHostSharing: true,
loggingContext: loggingContext,
fileAccessManifest: fam)
{
PipDescription = executable,
Arguments = args,
WorkingDirectory = Environment.CurrentDirectory,
StandardOutputEncoding = Encoding.UTF8,
StandardOutputObserver = stdoutStr => stdoutSb.AppendLine(stdoutStr),
StandardErrorEncoding = Encoding.UTF8,
StandardErrorObserver = stderrStr => stderrSb.AppendLine(stderrStr),
EnvironmentVariables = BuildParameters.GetFactory().PopulateFromEnvironment(),
Timeout = TimeSpan.FromMinutes(1),
};
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
string stdout = stdoutSb.ToString();
_output.WriteLine($"stdout: {stdout}");
string stderr = stderrSb.ToString();
_output.WriteLine($"stderr: {stderr}");
Assert.Equal(0, stderr.Length);
string shimOutput = "TestShim: Entered with command line: " + childShimArgs;
int indexOfShim = stdout.IndexOf(shimOutput, StringComparison.Ordinal);
Assert.True(indexOfShim > 0);
}
/// <nodoc />
public Task <SandboxedProcessResult> Run(SandboxOptions option)
{
var pathToProcess = option.args[0];
var workingDir = option.working_dir != AbsolutePath.Invalid ? option.working_dir.ToString(m_pathTable) : Environment.CurrentDirectory;
var fam = CreateManifest(AbsolutePath.Create(m_pathTable, pathToProcess), option, workingDir);
Action <string> stdoutCallback;
if (option.stdout_path != AbsolutePath.Invalid)
{
m_stdoutF = new StreamWriter(option.stdout_path.ToString(m_pathTable));
stdoutCallback = s => m_stdoutF.WriteLine(s);
}
else
{
stdoutCallback = s => Console.WriteLine(s);
}
Action <string> stderrCallback;
if (option.stderr_path != AbsolutePath.Invalid)
{
m_stderrF = new StreamWriter(option.stderr_path.ToString(m_pathTable));
stderrCallback = s => m_stderrF.WriteLine(s);
}
else
{
stderrCallback = s => Console.Error.WriteLine(s);
}
var info = new SandboxedProcessInfo(
m_pathTable,
this,
pathToProcess,
fam,
disableConHostSharing: true,
containerConfiguration: ContainerConfiguration.DisabledIsolation,
loggingContext: m_loggingContext)
{
Arguments = EscapeArgvRest(option.args.Skip(1)),
WorkingDirectory = workingDir,
// PipSemiStableHash = 0,
PipDescription = "BazelSandboxedProcess",
StandardOutputEncoding = Encoding.UTF8,
StandardOutputObserver = stdoutCallback,
StandardErrorEncoding = Encoding.UTF8,
StandardErrorObserver = stderrCallback,
SandboxedKextConnection = OperatingSystemHelper.IsUnixOS ? new KextConnection() : null
};
if (option.timeout_secs != SandboxOptions.kInfiniteTime)
{
info.Timeout = TimeSpan.FromSeconds(option.timeout_secs);
}
if (option.kill_delay_secs != SandboxOptions.kInfiniteTime)
{
info.NestedProcessTerminationTimeout = TimeSpan.FromSeconds(option.kill_delay_secs);
}
var process = SandboxedProcessFactory.StartAsync(info, forceSandboxing: true).GetAwaiter().GetResult();
return(process.GetResultAsync());
}
