public async Task EnumerateFileByHandleWithReadAllowedByScopeAndExplicitReport()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath filePath = WriteEmptyFile(pathTable, @"dir\file");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess | FileAccessPolicy.AllowReadAlways);
},
EnumerateFileOrDirectoryByHandle(@"dir\file")); // Note that we are trying to enumerate a file
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
// Note that there is no directory enumeration reported (this is really a single file probe).
expected: ExpectReport(ReportedFileOperation.CreateFile, RequestedAccess.Read, filePath));
VerifyReportedAccesses(
pathTable,
result.AllUnexpectedFileAccesses,
allowExtraEnumerations: false);
}
private async Task <Possible <Unit> > GenerateBuildDirectoryAsync()
{
Contract.Assert(m_buildDirectory.IsValid);
AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(Name);
AbsolutePath argumentsFile = outputDirectory.Combine(m_context.PathTable, Guid.NewGuid().ToString());
if (!TryRetrieveCMakeSearchLocations(out IEnumerable <AbsolutePath> searchLocations))
{
return(new CMakeGenerationError(m_resolverSettings.ModuleName, m_buildDirectory.ToString(m_context.PathTable)));
}
SandboxedProcessResult result = await ExecuteCMakeRunner(argumentsFile, searchLocations);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
if (!m_context.CancellationToken.IsCancellationRequested)
{
Tracing.Logger.Log.CMakeRunnerInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new CMakeGenerationError(m_resolverSettings.ModuleName, m_buildDirectory.ToString(m_context.PathTable)));
}
FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, Name, result.AllUnexpectedFileAccesses, outputDirectory);
return(Possible.Create(Unit.Void));
}
public async Task EnumerateEmptyDirectoryByHandleWithoutExplicitReport()
{
var pathTable = new PathTable();
AbsolutePath emptyDirPath = CreateDirectory(pathTable, @"emptyDir");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(emptyDirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowReadAlways);
},
EnumerateFileOrDirectoryByHandle(@"emptyDir"));
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
expected: new[]
{
// No CreateFile report, since emptyDir is a directory.
// Note that we get an explicit report for the enumeration, despite not adding ReportAccess (automatic for enumerations).
ExpectReport(ReportedFileOperation.NtQueryDirectoryFile, RequestedAccess.Enumerate, emptyDirPath),
});
VerifyReportedAccesses(
pathTable,
result.AllUnexpectedFileAccesses,
allowExtraEnumerations: false);
}
public async Task TestShimChildProcessWithPluginAndNonEmptyMatchesAsync(bool shimAllProcesses, bool shouldBeShimmed, bool shouldFindMatch)
{
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessResult result = await RunWithPluginAsync(
shimAllProcesses : shimAllProcesses,
shouldBeShimmed : shouldBeShimmed,
shouldFindMatch : shouldFindMatch,
processMatches : shouldFindMatch?new[] { "cmd.exe" } : new[] { "foo.exe" },
stdOutSb : stdOutSb,
stdErrSb : stdErrSb);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
if (shimAllProcesses)
{
AssertShimmedIf(stdOut, !shouldFindMatch || !shouldBeShimmed);
}
else
{
AssertShimmedIf(stdOut, shouldFindMatch && shouldBeShimmed);
}
XAssert.Contains(stdOut, GetChildOutputForPluginTest(shouldBeShimmed));
}
public async Task DeleteViaNtCreateFileIsAllowedWithAllowedWrite()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, "D");
AbsolutePath filePath = WriteEmptyFile(pathTable, @"D\FileToDelete");
AssertFileExists(@"D\FileToDelete");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowWrite | FileAccessPolicy.ReportAccess);
},
DeleteViaNtCreateFile(@"D\FileToDelete"));
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
expected: ExpectReport(ReportedFileOperation.NtCreateFile, RequestedAccess.Write, filePath, FileAccessStatus.Allowed));
AssertFileDoesNotExist(@"D\FileToDelete");
}
public async Task TestShimChildProcessWithPluginWithModifiedArgumentAsync()
{
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessResult result = await RunWithPluginAsync(
shimAllProcesses : false,
shouldBeShimmed : true,
shouldFindMatch : false,
processMatches : new string[0],
stdOutSb : stdOutSb,
stdErrSb : stdErrSb,
shimmedText : "@responseFile");
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
AssertShimmed(stdOut);
// Since shimmedText has '@', it will be replaced by "Content".
// CODESYNC: Public\Src\Engine\UnitTests\Processes.TestPrograms\SubstituteProcessExecutionPlugin\dllmain.cpp
XAssert.Contains(stdOut, GetChildOutputForPluginTest(true, "Content"));
}
public async Task HardlinkCreationIsAllowedWithAllowedWriteAndRead()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, "D");
AbsolutePath srcPath = WriteEmptyFile(pathTable, @"D\Origin");
AbsolutePath targetPath = GetFullPath(pathTable, @"D\Target");
AssertFileExists(@"D\Origin");
AssertFileDoesNotExist(@"D\Target");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess);
manifest.AddPath(srcPath, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing);
manifest.AddScope(targetPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowWrite);
},
CreateHardlink(@"D\Origin", @"D\Target"));
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
expected: new[]
{
ExpectReport(ReportedFileOperation.CreateHardLinkSource, RequestedAccess.Read, srcPath, FileAccessStatus.Allowed),
ExpectReport(ReportedFileOperation.CreateHardLinkDestination, RequestedAccess.Write, targetPath, FileAccessStatus.Allowed)
});
AssertFileExists(@"D\Origin");
AssertFileExists(@"D\Target");
}
public async Task HardlinkCreationDeniedWithoutAllowedWrite()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, "D");
AbsolutePath srcPath = WriteEmptyFile(pathTable, @"D\Origin");
AbsolutePath targetPath = GetFullPath(pathTable, @"D\Target");
AssertFileExists(@"D\Origin");
AssertFileDoesNotExist(@"D\Target");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowReadAlways);
},
CreateHardlink(@"D\Origin", @"D\Target"));
VerifyReportedAccesses(
pathTable,
result.AllUnexpectedFileAccesses,
allowExtraEnumerations: false,
expected: ExpectReport(ReportedFileOperation.CreateHardLinkDestination, RequestedAccess.Write, targetPath, FileAccessStatus.Denied));
AssertFileExists(@"D\Origin");
AssertFileExists(@"D\Target");
}
public async Task EnumerateAndProbeWithFindFirstFileAndFindNextFile()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath fileAPath = WriteEmptyFile(pathTable, @"dir\fileA");
AbsolutePath fileBPath = WriteEmptyFile(pathTable, @"dir\fileB");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess | FileAccessPolicy.AllowReadAlways);
},
EnumerateWithFindFirstFileEx(@"dir\*"));
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
expected: new[]
{
// Note that we assume a deterministic order here; in truth we paper over possible orders in VerifyReportedAccess.
ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.Enumerate, dirPath),
ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.EnumerationProbe, fileAPath),
ExpectReport(ReportedFileOperation.FindNextFile, RequestedAccess.EnumerationProbe, fileBPath),
});
}
public async Task EnumerateFileByHandleWithReadAllowedByScopeWithoutExplicitReport()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath filePath = WriteEmptyFile(pathTable, @"dir\file");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowReadAlways);
},
EnumerateFileOrDirectoryByHandle(@"dir\file")); // Note that we are trying to enumerate a file
// We don't see any reports; the access is just a probe, and we didn't ask for explicit reports.
// Note that this case is very important for enumerations like dir\file\* where dir\file is a static
// (precise file) dependency (not part of a sealed directory) since we only expect reports for sealed
// directory members.
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false);
VerifyReportedAccesses(
pathTable,
result.AllUnexpectedFileAccesses,
allowExtraEnumerations: false);
}
/// <summary>
/// Runs a sequence of RemoteApi commands in a <see cref="SandboxedProcess" />.
/// </summary>
public static async Task <SandboxedProcessResult> RunInSandboxAsync(
LoggingContext loggingContext,
PathTable pathTable,
string workingDirectory,
ISandboxedProcessFileStorage sandboxStorage,
Action <FileAccessManifest> populateManifest,
params Command[] commands)
{
Contract.Requires(!string.IsNullOrEmpty(workingDirectory));
Contract.Requires(populateManifest != null);
if (!File.Exists(ExecutablePath))
{
throw new BuildXLException("Expected to find RemoteApi.exe at " + ExecutablePath);
}
var info =
new SandboxedProcessInfo(pathTable, sandboxStorage, ExecutablePath, disableConHostSharing: false, loggingContext: loggingContext)
{
PipSemiStableHash = 0,
PipDescription = "RemoteApi Test",
Arguments = string.Empty,
WorkingDirectory = workingDirectory,
};
info.FileAccessManifest.ReportFileAccesses = false;
info.FileAccessManifest.ReportUnexpectedFileAccesses = true;
info.FileAccessManifest.FailUnexpectedFileAccesses = false;
info.FileAccessManifest.AddScope(AbsolutePath.Invalid, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportDirectoryEnumerationAccess);
populateManifest(info.FileAccessManifest);
// Allow access to the RemoteApi executable.
AbsolutePath exeDirectory = AbsolutePath.Create(pathTable, Path.GetDirectoryName(ExecutablePath));
info.FileAccessManifest.AddScope(exeDirectory, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowReadAlways);
using (TextReader commandReader = GetCommandReader(commands))
{
info.StandardInputReader = commandReader;
info.StandardInputEncoding = Encoding.ASCII;
// TODO: Maybe watch stdout and validate the command results.
using (SandboxedProcess process = await SandboxedProcess.StartAsync(info))
{
SandboxedProcessResult result = await process.GetResultAsync();
if (result.ExitCode != 0)
{
var stdErr = await result.StandardError.ReadValueAsync();
XAssert.AreEqual(0, result.ExitCode, "RemoteApi.exe failed: " + stdErr);
}
return(result);
}
}
}
private bool TryWriteSandboxedProcessResult(PathTable pathTable, SandboxedProcessResult result)
{
Contract.Requires(result != null);
// When BuildXL serializes SandboxedProcessInfo, it does not serialize the path table used by SandboxedProcessInfo.
// On deserializing that info, a new path table is created; see the Deserialize method of SandboxedProcessInfo.
// Unix sandbox uses the new path table in the deserialized SandboxedProcessInfo to create ManifestPath (AbsolutePath)
// from reported path access (string) in ReportFileAccess. Without special case, the serialization of SandboxedProcessResult
// will serialize the AbsolutePath as is. Then, when SandboxedProcessResult is read by BuildXL, BuildXL will not understand
// the ManifestPath because it is created from a different path table.
//
// In Windows, instead of creating ManifestPath from the reported path access (string), ManifestPath is reported from Detours
// using the AbsolutePath id embedded in the file access manifest. That AbsolutePath id is obtained using the same
// path table used by BuildXL, and thus BuildXL will understand the ManifestPath serialized by this tool.
//
// For Unix, we need to give a special care of path serialization.
bool isWindows = !OperatingSystemHelper.IsUnixOS;
Action <BuildXLWriter, AbsolutePath> writePath = (writer, path) =>
{
if (isWindows)
{
writer.Write(true);
writer.Write(path);
}
else
{
writer.Write(false);
writer.Write(path.ToString(pathTable));
}
};
bool success = false;
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
string sandboxedProcessResultOutputPath = Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile);
m_logger.LogInfo($"Writing sandboxed process result to '{sandboxedProcessResultOutputPath}'");
using (FileStream stream = File.OpenWrite(sandboxedProcessResultOutputPath))
{
result.Serialize(stream, writePath);
}
success = true;
},
ex =>
{
m_logger.LogError(ex.ToString());
success = false;
});
return(success);
}
private async Task <SandboxedProcessResult> RunWithPluginAsync(
bool shimAllProcesses,
bool shouldBeShimmed,
bool shouldFindMatch,
string[] processMatches,
StringBuilder stdOutSb,
StringBuilder stdErrSb,
string shimmedText = null)
{
var context = BuildXLContext.CreateInstanceForTesting();
var shimProgramPath = GetShimProgramPath(context);
var pluginDlls = GetPluginDlls(context);
string executable = CmdHelper.CmdX64;
var fam = CreateCommonFileAccessManifest(context.PathTable);
fam.SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: shimAllProcesses,
processMatches: processMatches.Select(pm => new ShimProcessMatch(PathAtom.Create(context.StringTable, pm), PathAtom.Invalid)).ToArray())
{
SubstituteProcessExecutionPluginDll32Path = pluginDlls.x86Dll,
SubstituteProcessExecutionPluginDll64Path = pluginDlls.x64Dll
};
string childOutput = GetChildOutputForPluginTest(shouldBeShimmed, shimmedText);
string childArgs = $"{executable} /D /C echo {childOutput}";
string args = $"/D /C echo Top-level cmd. Running child process && {childArgs}";
SandboxedProcessInfo sandboxedProcessInfo = CreateCommonSandboxedProcessInfo(
context,
executable,
args,
fam,
stdOutSb,
stdErrSb);
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
if (shouldFindMatch || processMatches.Length == 0)
{
// Plugin should be called when a match is found or no process match is specified.
// When plugin is entered, expect to see "Entering CommandMatches" text in the log.
// CODESYNC: Public\Src\Engine\UnitTests\Processes.TestPrograms\SubstituteProcessExecutionPlugin\dllmain.cpp
AssertLogContains(true, "Entering CommandMatches");
}
return(result);
}
private async Task <Possible <NinjaGraphResult> > ComputeBuildGraphAsync()
{
AbsolutePath outputFile = SerializedGraphPath.Value;
SandboxedProcessResult result = await RunNinjaGraphBuilderAsync(outputFile);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
if (!m_context.CancellationToken.IsCancellationRequested)
{
Tracing.Logger.Log.GraphConstructionInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new NinjaGraphConstructionFailure(m_resolverSettings.ModuleName, ProjectRoot.ToString(m_context.PathTable)));
}
// If the tool exited gracefully, but standard error is not empty, that is interpreted as a warning
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, Name, result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
var serializer = JsonSerializer.Create(GraphSerializationSettings.Settings);
// Add custom deserializer for converting string arrays to AbsolutePath ReadOnlySets
serializer.Converters.Add(new RootAwareAbsolutePathConverter(m_context.PathTable, SpecFile.GetParent(m_context.PathTable)));
serializer.Converters.Add(new ToReadOnlySetJsonConverter <AbsolutePath>());
var outputFileString = outputFile.ToString(m_context.PathTable);
Tracing.Logger.Log.LeftGraphToolOutputAt(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), outputFileString);
NinjaGraphResult projectGraphWithPredictionResult;
using (var sr = new StreamReader(outputFileString))
using (var reader = new JsonTextReader(sr))
{
projectGraphWithPredictionResult = serializer.Deserialize <NinjaGraphResult>(reader);
}
return(projectGraphWithPredictionResult);
}
public async Task CmdWithTestShim_ShimNothingRunsChildProcessWithoutShimAsync(bool shimAllProcess, bool filterMatch)
{
var context = BuildXLContext.CreateInstanceForTesting();
var shimProgramPath = GetShimProgramPath(context);
string executable = CmdHelper.CmdX64;
string processMatch = filterMatch ? null : "foo.exe"; // Filter should never match foo.exe.
var fam = CreateCommonFileAccessManifest(context.PathTable);
fam.SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: shimAllProcess,
processMatches: processMatch == null
? new ShimProcessMatch[0]
: new[] { new ShimProcessMatch(PathAtom.Create(context.StringTable, processMatch), PathAtom.Invalid) });
string predicate = shimAllProcess ? string.Empty : "not ";
string childOutput = $"Child cmd that should {predicate}be shimmed";
string childArgs = $"{executable} /D /C @echo {childOutput}";
string args = "/D /C echo Top-level cmd. Running child process && " + childArgs;
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessInfo sandboxedProcessInfo = CreateCommonSandboxedProcessInfo(
context,
executable,
args,
fam,
stdOutSb,
stdErrSb);
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
AssertShimmedIf(stdOut, shimAllProcess);
XAssert.Contains(stdOut, childOutput);
}
private async Task <Possible <RushGraph> > ComputeBuildGraphAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
SandboxedProcessResult result = await RunRushGraphBuilderAsync(outputFile, buildParameters);
string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
// In case of a cancellation, the tool may have exited with a non-zero
// code, but that's expected
if (!m_context.CancellationToken.IsCancellationRequested)
{
// This should never happen! Report the standard error and exit gracefully
Tracing.Logger.Log.GraphConstructionInternalError(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
return(new RushGraphConstructionFailure(m_resolverSettings, m_context.PathTable));
}
// If the tool exited gracefully, but standard error is not empty, that
// is interpreted as a warning. We propagate that to the BuildXL log
if (!string.IsNullOrEmpty(standardError))
{
Tracing.Logger.Log.GraphConstructionFinishedSuccessfullyButWithWarnings(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
standardError);
}
TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(m_context.PathTable));
JsonSerializer serializer = ConstructProjectGraphSerializer(s_jsonSerializerSettings);
using (var sr = new StreamReader(outputFile.ToString(m_context.PathTable)))
using (var reader = new JsonTextReader(sr))
{
var flattenedRushGraph = serializer.Deserialize <GenericRushGraph <GenericRushProject <string> > >(reader);
RushGraph graph = ResolveDependencies(flattenedRushGraph);
return(graph);
}
}
[InlineData("\"\"", "\"\"")] // Force case where child process is executed with double quotes around it
public async Task CmdWithSingleTokenChildProcessNoArgsAsync(string preCommand, string postCommand)
{
var context = BuildXLContext.CreateInstanceForTesting();
var shimProgramPath = GetShimProgramPath(context);
string executable = CmdHelper.CmdX64;
string childExecutable = executable;
string quotedExecutable = '"' + executable + '"';
childExecutable = quotedExecutable;
var fam = CreateCommonFileAccessManifest(context.PathTable);
// Use 'doskey' (alias manager) built into Windows.
string args = $"/D /C {preCommand}doskey.exe{postCommand}";
fam.SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: false,
processMatches: new[] { new ShimProcessMatch(PathAtom.Create(context.StringTable, "cmd.exe"), PathAtom.Invalid) });
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessInfo sandboxedProcessInfo = CreateCommonSandboxedProcessInfo(
context,
executable,
args,
fam,
stdOutSb,
stdErrSb);
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
}
public async Task CmdWithStartQuoteOnlyFailsToRunFullCommandLineAsync()
{
var context = BuildXLContext.CreateInstanceForTesting();
var shimProgramPath = GetShimProgramPath(context);
var fam = CreateCommonFileAccessManifest(context.PathTable);
fam.SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: true,
processMatches: new ShimProcessMatch[0]);
string executable = CmdHelper.CmdX64;
string childExecutable = '"' + executable; // Only an open quote
string childOutput = "Child cmd that should be shimmed";
string childArgs = $"{childExecutable} /D /C @echo {childOutput}";
string args = "/D /C echo Top-level cmd. Running child process && " + childArgs;
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessInfo sandboxedProcessInfo = CreateCommonSandboxedProcessInfo(
context,
executable,
args,
fam,
stdOutSb,
stdErrSb);
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
XAssert.AreEqual(1, result.ExitCode);
XAssert.AreEqual("The system cannot find the path specified.\r\n", stdErr);
}
public async Task EnumerateEmptyDirectoryWithFindFirstFileAndExplicitReportAndNoMatches()
{
var pathTable = new PathTable();
AbsolutePath emptyDirPath = CreateDirectory(pathTable, @"emptyDir");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(emptyDirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess | FileAccessPolicy.AllowReadAlways);
},
EnumerateWithFindFirstFileEx(@"emptyDir\xxx*")); // xxx* excludes the magic . and .. entries.
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
expected: ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.Enumerate, emptyDirPath));
}
private void PostProcessSandboxedProcessResult(SandboxedProcessInfo info, SandboxedProcessResult result)
{
m_logger.LogInfo("Post processing sandboxed process result");
if (result.FileAccesses != null)
{
if (info.RemoteSandboxedProcessData != null)
{
// TODO: Hack! Hack!
// This changes is done so that AnyBuild does not try to send untracked files as inputs/outputs.
// This changes the file accesses that BuildXL will see.
// Ideally, this filtration should be done in AnyBuild when processing the result of SandboxedProcessResult
// coming from this executor.
HashSet <ReportedFileAccess> trackedAccesses = result
.FileAccesses
.Where(fa => !info.RemoteSandboxedProcessData.IsUntracked(fa.GetPath(info.PathTable))).ToHashSet();
result.FileAccesses = trackedAccesses;
}
}
}
public async Task EnumerateEmptyDirectoryWithFindFirstFileAndNoExplicitReport()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath fileAPath = WriteEmptyFile(pathTable, @"dir\fileA");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest => { manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowReadAlways); },
EnumerateWithFindFirstFileEx(@"dir\*"));
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
// Note that we do not get an EnumerationProbe for fileA since it is not under Report scope.
expected: ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.Enumerate, dirPath));
}
private void PrintObservedAccesses(PathTable pathTable, SandboxedProcessResult result)
{
var accesses = new List <ReportedFileAccess>();
if (result.FileAccesses != null)
{
accesses.AddRange(result.FileAccesses);
}
if (result.AllUnexpectedFileAccesses != null)
{
accesses.AddRange(result.AllUnexpectedFileAccesses);
}
m_logger.LogInfo($"{accesses.Count} observed access(es):");
foreach (var access in accesses)
{
m_logger.LogInfo($"{access.GetPath(pathTable)}: {access.Describe()}");
}
}
public async Task EnumerateNonexistentDirectory()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath fakeDirPath = GetFullPath(pathTable, @"dir\fake");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess | FileAccessPolicy.AllowReadAlways);
},
EnumerateWithFindFirstFileEx(@"dir\fake\*")); // dir\fake doesn't exist so we should expect ERROR_PATH_NOT_FOUND (nonexistent directory to enumerate).
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
expected: ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.Enumerate, fakeDirPath, exists: false));
}
private async Task <(ExitCode, SandboxedProcessResult)> ExecuteAsync(SandboxedProcessInfo info)
{
try
{
using (Stream standardInputStream = TryOpenStandardInputStream(info, out bool succeedInOpeningStdIn))
{
if (!succeedInOpeningStdIn)
{
return(ExitCode.FailedSandboxPreparation, null);
}
using (StreamReader standardInputReader = standardInputStream == null ? null : new StreamReader(standardInputStream, CharUtilities.Utf8NoBomNoThrow))
{
info.StandardInputReader = standardInputReader;
ISandboxedProcess process = await StartProcessAsync(info);
if (process == null)
{
return(ExitCode.FailedStartProcess, null);
}
SandboxedProcessResult result = await process.GetResultAsync();
// Patch result.
result.WarningCount = m_outputErrorObserver.WarningCount;
result.LastMessageCount = process.GetLastMessageCount();
result.DetoursMaxHeapSize = process.GetDetoursMaxHeapSize();
result.MessageCountSemaphoreCreated = info.FileAccessManifest.MessageCountSemaphore != null;
return(ExitCode.Success, result);
}
}
}
finally
{
info.FileAccessManifest.UnsetMessageCountSemaphore();
}
}
public async Task TestShimChildProcessWithPluginAndEmptyMatchesAsync(bool shimAllProcesses, bool shouldBeShimmed)
{
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessResult result = await RunWithPluginAsync(
shimAllProcesses : shimAllProcesses,
shouldBeShimmed : shouldBeShimmed,
shouldFindMatch : false,
processMatches : new string[0],
stdOutSb : stdOutSb,
stdErrSb : stdErrSb);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
AssertShimmedIf(stdOut, shimAllProcesses != shouldBeShimmed);
XAssert.Contains(stdOut, GetChildOutputForPluginTest(shouldBeShimmed));
}
public async Task ProbeWithFindFirstFileWhereSearchPathIsFileAndReadNotAllow()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath filePath = WriteEmptyFile(pathTable, @"dir\file");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess);
},
EnumerateWithFindFirstFileEx(@"dir\file\*")); // Note that we are trying to wildcard udner a file.
VerifyReportedAccesses(
pathTable,
result.AllUnexpectedFileAccesses,
allowExtraEnumerations: false,
// Note that there is no directory enumeration reported (this is really a single file probe).
// Note that the probe path is dir\file despite querying dir\file\*
expected: ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.Probe, filePath, status: FileAccessStatus.Denied));
}
private bool TryWriteSandboxedProcessResult(SandboxedProcessResult result)
{
Contract.Requires(result != null);
bool success = false;
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (FileStream stream = File.OpenWrite(Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile)))
{
result.Serialize(stream);
}
success = true;
},
ex =>
{
m_logger.LogError(ex.ToString());
success = false;
});
return(success);
}
public async Task ProbeWithFindFirstFileSingle()
{
var pathTable = new PathTable();
AbsolutePath dirPath = CreateDirectory(pathTable, @"dir");
AbsolutePath fileAPath = WriteEmptyFile(pathTable, @"dir\fileA");
SandboxedProcessResult result = await RunRemoteApiInSandboxAsync(
pathTable,
manifest =>
{
manifest.AddScope(dirPath, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess | FileAccessPolicy.AllowReadAlways);
},
EnumerateWithFindFirstFileEx(@"dir\fileA")); // Note no wildcards; should get a probe but not enumeration.
VerifyReportedAccesses(
pathTable,
result.ExplicitlyReportedFileAccesses,
allowExtraEnumerations: false,
// Note that there is no directory enumeration reported (this is really a single file probe),
// and the report for fileA is correspondingly a Probe rather than EnumerationProbe.
expected: ExpectReport(ReportedFileOperation.FindFirstFileEx, RequestedAccess.Probe, fileAPath));
}
[InlineData(false, "cmd.exe")] // Filter should match child
public async Task CmdWithTestShim(bool useQuotesForChildCmdExe, string processMatch)
{
var context = BuildXLContext.CreateInstanceForTesting();
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = CmdHelper.CmdX64;
string childExecutable = executable;
string quotedExecutable = '"' + executable + '"';
if (useQuotesForChildCmdExe)
{
childExecutable = quotedExecutable;
}
string shimProgram = Path.Combine(currentCodeFolder, "TestSubstituteProcessExecutionShim.exe");
Assert.True(File.Exists(shimProgram), $"Shim test program not found at {shimProgram}");
var shimProgramPath = AbsolutePath.Create(context.PathTable, shimProgram);
var fam =
new FileAccessManifest(context.PathTable)
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false,
SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: processMatch == null, // When we have a process to match, make the shim list opt-in to ensure a match
processMatch == null ? new ShimProcessMatch[0] : new[] { new ShimProcessMatch(PathAtom.Create(context.StringTable, processMatch), PathAtom.Invalid) })
};
Guid sessionId = Guid.NewGuid();
string sessionIdStr = sessionId.ToString("N");
var loggingContext = new LoggingContext(sessionId, "TestSession", new LoggingContext.SessionInfo(sessionIdStr, "env", sessionId));
string childOutput = "Child cmd that should be shimmed";
string childArgs = $"{childExecutable} /D /C @echo {childOutput}";
// Detours logic should wrap the initial cmd in quotes for easier parsing by shim logic.
// However, since we're indirecting through a cmd.exe command line it gets dropped along the way.
string childShimArgs = $"/D /C @echo {childOutput}";
string args = "/D /C echo Top-level cmd. Running child process && " + childArgs;
var stdoutSb = new StringBuilder(128);
var stderrSb = new StringBuilder();
var sandboxedProcessInfo = new SandboxedProcessInfo(
context.PathTable,
new LocalSandboxedFileStorage(),
executable,
disableConHostSharing: true,
loggingContext: loggingContext,
fileAccessManifest: fam)
{
PipDescription = executable,
Arguments = args,
WorkingDirectory = Environment.CurrentDirectory,
StandardOutputEncoding = Encoding.UTF8,
StandardOutputObserver = stdoutStr => stdoutSb.AppendLine(stdoutStr),
StandardErrorEncoding = Encoding.UTF8,
StandardErrorObserver = stderrStr => stderrSb.AppendLine(stderrStr),
EnvironmentVariables = BuildParameters.GetFactory().PopulateFromEnvironment(),
Timeout = TimeSpan.FromMinutes(1),
};
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
string stdout = stdoutSb.ToString();
m_output.WriteLine($"stdout: {stdout}");
string stderr = stderrSb.ToString();
m_output.WriteLine($"stderr: {stderr}");
Assert.Equal(0, result.ExitCode);
Assert.Equal(0, stderr.Length);
// The shim is an exe on netframework, dll in a temp dir on netcore, so don't try to match it.
const string shimOutput = "TestShim: Entered with command line: ";
int indexOfShim = stdout.IndexOf(shimOutput, StringComparison.Ordinal);
Assert.True(indexOfShim > 0, shimOutput);
m_output.WriteLine($"Expecting shim args: {childShimArgs}");
int indexOfShimArgs = stdout.LastIndexOf(childShimArgs);
Assert.True(indexOfShimArgs > indexOfShim);
}
[InlineData(true, "foo.exe")] // Filter should not match
public async Task CmdWithTestShim_ShimNothingRunsChildProcessWithoutShim(bool shimAllProcesses, string processMatch)
{
var context = BuildXLContext.CreateInstanceForTesting();
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = CmdHelper.CmdX64;
string shimProgram = Path.Combine(currentCodeFolder, "TestSubstituteProcessExecutionShim.exe");
Assert.True(File.Exists(shimProgram), $"Shim test program not found at {shimProgram}");
var shimProgramPath = AbsolutePath.Create(context.PathTable, shimProgram);
var fam =
new FileAccessManifest(context.PathTable)
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false,
SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: false,
processMatch == null ? new ShimProcessMatch[0] : new[] { new ShimProcessMatch(PathAtom.Create(context.StringTable, processMatch), PathAtom.Invalid) })
};
Guid sessionId = Guid.NewGuid();
string sessionIdStr = sessionId.ToString("N");
var loggingContext = new LoggingContext(sessionId, "TestSession", new LoggingContext.SessionInfo(sessionIdStr, "env", sessionId));
string childOutput = "Child cmd that should not be shimmed";
string childArgs = $"{executable} /D /C @echo {childOutput}";
string args = "/D /C echo Top-level cmd. Running child process && " + childArgs;
var stdoutSb = new StringBuilder(128);
var stderrSb = new StringBuilder();
var sandboxedProcessInfo = new SandboxedProcessInfo(
context.PathTable,
new LocalSandboxedFileStorage(),
executable,
disableConHostSharing: true,
loggingContext: loggingContext,
fileAccessManifest: fam)
{
PipDescription = executable,
Arguments = args,
WorkingDirectory = Environment.CurrentDirectory,
StandardOutputEncoding = Encoding.UTF8,
StandardOutputObserver = stdoutStr => stdoutSb.AppendLine(stdoutStr),
StandardErrorEncoding = Encoding.UTF8,
StandardErrorObserver = stderrStr => stderrSb.AppendLine(stderrStr),
EnvironmentVariables = BuildParameters.GetFactory().PopulateFromEnvironment(),
Timeout = TimeSpan.FromMinutes(1),
};
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
Assert.Equal(0, result.ExitCode);
string stdout = stdoutSb.ToString();
m_output.WriteLine($"stdout: {stdout}");
string stderr = stderrSb.ToString();
m_output.WriteLine($"stderr: {stderr}");
Assert.Equal(0, stderr.Length);
string shimOutput = "TestShim: Entered with command line";
int indexOfShim = stdout.IndexOf(shimOutput, StringComparison.Ordinal);
Assert.True(indexOfShim == -1);
int indexOfChild = stdout.LastIndexOf(childOutput, StringComparison.Ordinal);
Assert.True(indexOfChild > 0, "Child should have run and written output");
}
