public void BreakawayProcessCanReportAugmentedAccesses()
{
var fam = new FileAccessManifest(
Context.PathTable,
childProcessesToBreakawayFromSandbox: new[] { TestProcessToolName })
{
FailUnexpectedFileAccesses = false,
ReportUnexpectedFileAccesses = true,
ReportFileAccesses = true
};
var srcFile = CreateSourceFile();
var output = CreateOutputFileArtifact();
fam.AddScope(srcFile, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess);
fam.AddScope(output, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess);
var info = ToProcessInfo(
ToProcess(
Operation.AugmentedRead(srcFile),
Operation.AugmentedWrite(output)),
fileAccessManifest: fam);
var result = RunProcess(info).GetAwaiter().GetResult();
XAssert.AreEqual(0, result.ExitCode);
var observedAccesses = result.FileAccesses.Select(fa => fa.ManifestPath);
XAssert.Contains(observedAccesses, srcFile.Path, output.Path);
}
public void ValidateCompleteness()
{
// TODO: Dig into whether ipcSend should be allowed in config
HashSet <string> ignoredMethods = new HashSet <string>()
{
"ScheduleProcessPip",
"WriteDataCore",
"GetBuildXLBinDirectoryToBeDeprecated",
"GetNewIpcMoniker",
"GetBuildEngineDirectoryToBeDeprecated",
};
// Extract string parameter for each InlineDataAttribute for each method.
string[] inlineDataContent = this.GetType().GetMethods()
.SelectMany(mi => mi.GetCustomAttributes <InlineDataAttribute>().Select(attr => (string)attr?.GetData(null)?.FirstOrDefault()[0]))
.Select(s => s.Substring(0, s.IndexOf("(")))
.ToArray();
XAssert.All(GetAmbientMethods(AmbientContext.ContextName, typeof(AmbientContext))
.Concat(GetAmbientMethods(AmbientTransformerOriginal.Name, typeof(AmbientTransformerOriginal)))
.Concat(GetAmbientMethods(AmbientContract.ContractName, typeof(AmbientContract)))
.Concat(GetAmbientMethods(AmbientFile.FileName, typeof(AmbientFile)))
.ToArray(),
funcName => XAssert.Contains(inlineDataContent, funcName));
IEnumerable <string> GetAmbientMethods(string name, System.Type type)
{
return(type.GetMethods(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.DeclaredOnly)
.Where(mi => mi.ReturnType == typeof(EvaluationResult) && !ignoredMethods.Contains(mi.Name))
.Select(mi => $"{name}.{char.ToLowerInvariant(mi.Name[0])}{mi.Name.Substring(1)}"));
}
}
public async Task TestDynamicallyLoadedLibrariesReportedOnLinux()
{
if (!OperatingSystemHelper.IsLinuxOS)
{
return;
}
var proc = ToProcess(Operation.Echo("hi"));
var info = ToProcessInfo(proc, nameof(TestDynamicallyLoadedLibrariesReportedOnLinux));
info.FileAccessManifest.ReportFileAccesses = true;
info.FileAccessManifest.FailUnexpectedFileAccesses = false;
using ISandboxedProcess process = await StartProcessAsync(info);
var result = await process.GetResultAsync();
XAssert.AreEqual(0, result.ExitCode);
var accesses = result.FileAccesses
.Select(fa => fa.GetPath(Context.PathTable))
.Where(p => p.EndsWith(".so"))
.Select(p => Path.GetFileName(p))
.Select(n => n.Contains('-') ? n.Substring(0, n.IndexOf('-')) + ".so" : n)
.Distinct()
.ToHashSet();
XAssert.Contains(accesses, new[]
{
"ld.so", "libc.so", "libdl.so", "libm.so", "libpthread.so", "librt.so",
"libclrjit.so", "libcoreclr.so", "libcoreclrtraceptprovider.so",
"libhostfxr.so", "libhostpolicy.so", "libmscordaccore.so", "libmscordbi.so",
});
XAssert.ContainsNot(accesses, new[] { "libDetours.so" });
}
public async Task TestShimChildProcessWithPluginAndNonEmptyMatchesAsync(bool shimAllProcesses, bool shouldBeShimmed, bool shouldFindMatch)
{
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessResult result = await RunWithPluginAsync(
shimAllProcesses : shimAllProcesses,
shouldBeShimmed : shouldBeShimmed,
shouldFindMatch : shouldFindMatch,
processMatches : shouldFindMatch?new[] { "cmd.exe" } : new[] { "foo.exe" },
stdOutSb : stdOutSb,
stdErrSb : stdErrSb);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
if (shimAllProcesses)
{
AssertShimmedIf(stdOut, !shouldFindMatch || !shouldBeShimmed);
}
else
{
AssertShimmedIf(stdOut, shouldFindMatch && shouldBeShimmed);
}
XAssert.Contains(stdOut, GetChildOutputForPluginTest(shouldBeShimmed));
}
public async Task TestShimChildProcessWithPluginWithModifiedArgumentAsync()
{
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessResult result = await RunWithPluginAsync(
shimAllProcesses : false,
shouldBeShimmed : true,
shouldFindMatch : false,
processMatches : new string[0],
stdOutSb : stdOutSb,
stdErrSb : stdErrSb,
shimmedText : "@responseFile");
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
AssertShimmed(stdOut);
// Since shimmedText has '@', it will be replaced by "Content".
// CODESYNC: Public\Src\Engine\UnitTests\Processes.TestPrograms\SubstituteProcessExecutionPlugin\dllmain.cpp
XAssert.Contains(stdOut, GetChildOutputForPluginTest(true, "Content"));
}
public void RunSingleProcessWithSharedOpaqueOutputLogging()
{
var sharedOpaqueDir = Path.Combine(ObjectRoot, "partialDir");
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var sharedOpaqueDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir);
var source = CreateSourceFile();
var builder = CreatePipBuilder(new[]
{
Operation.WriteFile(outputInSharedOpaque, content: "sod-out", doNotInfer: true)
});
builder.AddOutputDirectory(sharedOpaqueDirectoryArtifact, SealDirectoryKind.SharedOpaque);
builder.Options |= Process.Options.RequiresAdmin;
var pip = SchedulePipBuilder(builder);
// run once and assert success
var result = RunScheduler().AssertSuccess();
// check that shared opaque outputs have been logged in the sideband file
var writesInSidebandFile = GetJournaledWritesForProcess(result, pip.Process);
XAssert.Contains(writesInSidebandFile, outputInSharedOpaque);
XAssert.ContainsNot(writesInSidebandFile, pip.ProcessOutputs.GetOutputFiles().Select(f => f.Path).ToArray());
// run again and assert cache hit
RunScheduler().AssertCacheHit(pip.Process.PipId);
// assert sideband files were used for scrubbing
AssertInformationalEventLogged(EventId.DeletingOutputsFromSharedOpaqueSidebandFilesStarted, count: 1);
AssertInformationalEventLogged(EventId.DeletingSharedOpaqueSidebandFilesStarted, count: 1);
}
public void QualifierBasedTagsAreSet()
{
// Writes a file using a process pip with user defined tags 'test1' and 'test2'
string spec = @"
import {Cmd, Transformer} from 'Sdk.Transformers';
export declare const qualifier: {platform: 'x86' | 'x64'; configuration: 'debug' | 'release'};
const outDir = Context.getNewOutputDirectory('tags');
const result = Transformer.execute({
tool: {" +
$"exe: f`{(OperatingSystemHelper.IsUnixOS ? "/bin/sh" : @"${Environment.getPathValue(""COMSPEC"")}")}`"
+ @"},
workingDirectory: d`.`,
arguments: [ Cmd.rawArgument('" + $"{(OperatingSystemHelper.IsUnixOS ? "-c echo test > obj/a.txt" : @"/d /c echo test > obj\a.txt")}" + @"'),
],
outputs: [
p`${outDir}/obj/a.txt`,
],
tags: ['test1', 'test2']
});
";
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// Configure the initial qualifier to be configuration=debug, platform=x64
var config = Build()
.Configuration(@"
config({
qualifiers: {
defaultQualifier: {
configuration: 'debug',
platform: 'x64'
}
}
});")
.AddSpec(spec)
.PersistSpecsAndGetConfiguration();
// Scheduling is enough, we just want to inspect the tags
((CommandLineConfiguration)config).Engine.Phase = global::BuildXL.Utilities.Configuration.EnginePhases.Schedule;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: false,
engine: out var engine);
XAssert.IsTrue(engineResult.IsSuccess);
// The corresponding process pip should have both the user defined tags and the current qualifier tags
var process = (PipsOperations.Process)engineResult.EngineState.PipGraph.RetrievePipsOfType(PipsOperations.PipType.Process).Single();
XAssert.Contains(process.Tags.Select(tag => tag.ToString(StringTable)), "test1", "test2", "platform=x64", "configuration=debug");
}
}
public void TestHandleExceptions(IpcResultStatus status)
{
var exceptionMessage = "invalid operation";
var ipcResult = Utils.HandleExceptions(status, () => throw new InvalidOperationException(exceptionMessage));
XAssert.AreEqual(status, ipcResult.ExitCode);
XAssert.Contains(ipcResult.Payload, exceptionMessage);
}
public void TestDeserializationOfUnknownCommand()
{
var cmd = new UnknownCommand();
string str = Command.Serialize(cmd);
var exception = Assert.Throws <ArgumentException>(() => Command.Deserialize(str));
XAssert.Contains(exception.Message, cmd.TypeName);
}
public void ChildProcessCanBreakawayWhenConfigured(bool letInfiniteWaiterSurvive)
{
// We use InfiniteWaiter (a process that waits forever) as a long-living process that we can actually check it can
// escape the job object
var fam = new FileAccessManifest(
Context.PathTable,
childProcessesToBreakawayFromSandbox: letInfiniteWaiterSurvive ? new[] { InfiniteWaiterToolName } : null)
{
FailUnexpectedFileAccesses = false
};
// We instruct the regular test process to spawn InfiniteWaiter as a child
var info = ToProcessInfo(
ToProcess(
Operation.SpawnExe(
Context.PathTable,
CreateFileArtifactWithName(InfiniteWaiterToolName, TestDeploymentDir))),
fileAccessManifest: fam);
// Let's shorten the default time to wait for nested processes, since we are spawning
// a process that never ends and we don't want this test to wait for that long
info.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(10);
var result = RunProcess(info).GetAwaiter().GetResult();
XAssert.AreEqual(0, result.ExitCode);
if (!letInfiniteWaiterSurvive)
{
// If we didn't let infinite waiter escape, we should have killed it when the job object was finalized
XAssert.IsTrue(result.Killed);
XAssert.Contains(
result.SurvivingChildProcesses.Select(p => p?.Path).Where(p => p != null).Select(p => System.IO.Path.GetFileName(p).ToUpperInvariant()),
InfiniteWaiterToolName.ToUpperInvariant());
}
else
{
// If we did let it escape, then nothing should have been killed (nor tried to survive and later killed, from the job object point of view)
XAssert.IsFalse(result.Killed);
if (result.SurvivingChildProcesses != null)
{
var survivors = string.Join(
", ",
result.SurvivingChildProcesses.Select(p => p?.Path != null ? System.IO.Path.GetFileName(p.Path) : "<unknown>"));
XAssert.Fail($"Unexpected {result.SurvivingChildProcesses.Count()} surviving child processes: {survivors}");
}
// Let's retrieve the child process and confirm it survived
var infiniteWaiterInfo = RetrieveChildProcessesCreatedBySpawnExe(result).Single();
// The fact that this does not throw confirms survival
var dummyWaiter = Process.GetProcessById(infiniteWaiterInfo.pid);
// Just being protective, let's make sure we are talking about the same process
XAssert.AreEqual(infiniteWaiterInfo.processName, dummyWaiter.ProcessName);
// Now let's kill the surviving process, since we don't want it to linger around unnecessarily
dummyWaiter.Kill();
}
}
public void TestCatastrophicFailureRecovery()
{
// Run some valid module
SetupTestData();
RunEngine();
// List the files in the engine cache after a valid run
var engineCacheDirectory = Configuration.Layout.EngineCacheDirectory.ToString(Context.PathTable);
var engineCacheFilesList = new List <string>();
FileUtilities.EnumerateDirectoryEntries(engineCacheDirectory, (file, attributes) =>
{
if (!attributes.HasFlag(FileAttributes.Directory))
{
engineCacheFilesList.Add(file);
}
});
var recovery = FailureRecoveryFactory.Create(LoggingContext, Context.PathTable, Configuration);
// This will trigger the recovery mechanism for unknown catastrophic errors, which is to log and remove the engine state (EngineCache folder)
XAssert.IsTrue(recovery.TryMarkFailure(new BuildXLException("fake failure"), ExceptionRootCause.Unknown));
// List the files in the logs directory for corrupt engine cache files
var logsDirectory = Configuration.Logging.EngineCacheCorruptFilesLogDirectory.ToString(Context.PathTable);
var logsFilesList = new HashSet <string>();
FileUtilities.EnumerateDirectoryEntries(logsDirectory, (file, attributes) =>
{
logsFilesList.Add(file);
});
var childrenCount = Directory.GetFiles(engineCacheDirectory, "*", SearchOption.TopDirectoryOnly).Length;
var expectedCount = -1;
// File content table has a special exclusion from the removal policy for performance reasons, but it should still be copied to logs
// (Unless the file content table doesn't exist in the engine cache, then it doesn't need to exist in the logs)
var engineCacheFileContentTablePath = Configuration.Layout.FileContentTableFile.ToString(Context.PathTable);
var fileContentTableFile = Path.GetFileName(engineCacheFileContentTablePath);
// Make sure file content table was copied to logs
XAssert.Contains(logsFilesList, fileContentTableFile);
expectedCount = 1;
// Make sure file content table file exists in the engine cache directory after recovery
XAssert.IsTrue(File.Exists(engineCacheFileContentTablePath));
// Check to make sure the engine cache directory is empty except for maybe the file content table
XAssert.AreEqual(expectedCount, childrenCount);
// Check to make sure all the file from the engine cache directory ended up in the logs directory
foreach (var file in engineCacheFilesList)
{
XAssert.Contains(logsFilesList, file);
}
}
public async Task TestConnectWithRetriesFailsAsync()
{
var errMessage = "can't connect";
var maybeConnection = await Utils.ConnectAsync <int>(
maxRetry : 2,
waitTimeBetweenAttempts : TimeSpan.FromMilliseconds(1),
connectionFactory : () => throw new InvalidOperationException(errMessage));
XAssert.IsFalse(maybeConnection.Succeeded);
XAssert.Contains(maybeConnection.Failure.DescribeIncludingInnerFailures(), errMessage);
}
public void IncorrectSwitchDoesNotFailLogger()
{
var result = RunMSBuild($"Win32ManifestFile='does/not/exist'", out string standardOutput);
// The run should fail
XAssert.AreNotEqual(0, result);
// The reason should be because an unexpected task attribute (MSB4064), but not because of a logger failure
XAssert.ContainsNot(standardOutput, "InvalidOperationException");
XAssert.Contains(standardOutput, "MSB4064");
}
public void TestIpcResultMerge(IpcResultStatus lhsStatus, IpcResultStatus rhsStatus, IpcResultStatus mergeStatus)
{
var lhs = new IpcResult(lhsStatus, "lhs");
var rhs = new IpcResult(rhsStatus, "rhs");
var merged = IpcResult.Merge(lhs, rhs);
// contains both payloads
XAssert.Contains(merged.Payload, lhs.Payload, rhs.Payload);
// has correct status
XAssert.AreEqual(merged.ExitCode, mergeStatus);
}
private static void AssertShimmedIf(string output, bool shimmedCondition)
{
if (shimmedCondition)
{
XAssert.Contains(output, ShimOutput);
}
else
{
XAssert.ContainsNot(output, ShimOutput);
}
}
public void ValidateStringMatch()
{
var aBuild = CreateJavaScriptProject("project-A", "build");
var aTest = CreateJavaScriptProject("project-A", "test");
var bBuild = CreateJavaScriptProject("project-B", "build");
var selector = new JavaScriptProjectSelector(new[] { aBuild, aTest, bBuild });
var result = selector.GetMatches("project-A");
XAssert.Contains(result, aBuild, aTest);
XAssert.ContainsNot(result, bBuild);
}
public async Task TestClientWhenCommandThrowsAsync()
{
var exceptionMessage = "invalid operation";
using var apiClient = CreateApiClient(ipcOperation =>
{
throw new InvalidOperationException(exceptionMessage);
});
var maybeResult = await apiClient.LogMessage("hi");
XAssert.IsFalse(maybeResult.Succeeded);
XAssert.Contains(maybeResult.Failure.DescribeIncludingInnerFailures(), exceptionMessage);
}
public async Task TestClientWhenCommandReturnsBogusValueAsync()
{
var bogusPayload = "bogus payload";
using var apiClient = CreateApiClient(ipcOperation =>
{
return(new IpcResult(IpcResultStatus.Success, bogusPayload));
});
var maybeResult = await apiClient.LogMessage("hi");
XAssert.IsFalse(maybeResult.Succeeded);
XAssert.Contains(maybeResult.Failure.DescribeIncludingInnerFailures(), bogusPayload, Client.ErrorCannotParseIpcResultMessage);
}
public async Task TestClientWhenCommandFailsAsync()
{
var errorPayload = "operation failed";
using var apiClient = CreateApiClient(ipcOperation =>
{
return(new IpcResult(IpcResultStatus.GenericError, errorPayload));
});
var maybeResult = await apiClient.LogMessage("hi");
XAssert.IsFalse(maybeResult.Succeeded);
XAssert.Contains(maybeResult.Failure.DescribeIncludingInnerFailures(), errorPayload);
}
public void ScriptNameIsSetAsTag()
{
var project = CreateRushProject(scriptCommandName: "some-script");
var processTags = Start()
.Add(project)
.ScheduleAll()
.RetrieveSuccessfulProcess(project)
.Tags;
// The script name should be part of the process tags
XAssert.Contains(processTags, StringId.Create(StringTable, "some-script"));
}
public void MaskedReportAugmentedAccessIsNotReported(bool reportFileAccesses)
{
var fam = new FileAccessManifest(
Context.PathTable,
childProcessesToBreakawayFromSandbox: new[] { TestProcessToolName })
{
FailUnexpectedFileAccesses = false,
ReportUnexpectedFileAccesses = true,
ReportFileAccesses = reportFileAccesses
};
var basePath = TestBinRootPath.Combine(Context.PathTable, "foo");
var output1 = CreateOutputFileArtifact(basePath);
var output2 = CreateOutputFileArtifact(basePath);
// We mask reporting accesses for output1 and enable it for output2
fam.AddScope(output1.Path, ~FileAccessPolicy.ReportAccess, FileAccessPolicy.AllowAll);
fam.AddScope(output2.Path, FileAccessPolicy.MaskNothing, FileAccessPolicy.AllowAll | FileAccessPolicy.ReportAccess);
var collector = new FileAccessCollector(Context.PathTable);
var info = ToProcessInfo(
ToProcess(
Operation.AugmentedWrite(output1),
Operation.AugmentedWrite(output2)),
fileAccessManifest: fam,
detoursListener: collector);
var result = RunProcess(info).GetAwaiter().GetResult();
XAssert.AreEqual(0, result.ExitCode);
// We should get a single explicit access with output2, since output1 shouldn't be reported
var accessPath = result.ExplicitlyReportedFileAccesses.Single(rfa => rfa.Method == FileAccessStatusMethod.TrustedTool).ManifestPath;
XAssert.AreEqual(output2.Path, accessPath);
// We should get both accesses as part of the (optional) FileAccess on request
if (reportFileAccesses)
{
var allTrustedAcceses = result.FileAccesses.Where(rfa => rfa.Method == FileAccessStatusMethod.TrustedTool).Select(rfa => rfa.ManifestPath);
XAssert.Contains(allTrustedAcceses, output1.Path, output2.Path);
}
else
{
// Make sure the access related to output1 is not actually reported, and the only one the listener got is output2
XAssert.Contains(collector.FileAccessPaths, output2.Path);
XAssert.ContainsNot(collector.FileAccessPaths, output1.Path);
}
}
public void ExtractFilesDiff()
{
var oldData = new[] { A("X", "Y", "a"), A("X", "Y", "b"), A("X", "Y", "c"), A("X", "Y", "d") };
var newData = new[] { A("X", "Y", "a"), A("X", "Y", "b"), A("X", "Y", "f"), A("X", "Y", "g") };
bool hasDiff = ExtractUnorderedListDiff(oldData, newData, out var added, out var removed);
XAssert.IsTrue(hasDiff);
XAssert.AreEqual(2, added.Count);
XAssert.AreEqual(2, removed.Count);
XAssert.Contains(added, A("X", "Y", "f"), A("X", "Y", "g"));
XAssert.Contains(removed, A("X", "Y", "c"), A("X", "Y", "d"));
}
public void BreakawayProcessIsHonored()
{
var project = CreateRushProject();
var breakawayTest = PathAtom.Create(StringTable, "test.exe");
var breakawayProcesses = Start(new RushResolverSettings {
ChildProcessesToBreakawayFromSandbox = new[] { breakawayTest }
})
.Add(project)
.ScheduleAll()
.RetrieveSuccessfulProcess(project)
.ChildProcessesToBreakawayFromSandbox;
XAssert.Contains(breakawayProcesses, breakawayTest);
}
public void SuccessfulCodesAreHonored()
{
var project = CreateRushProject();
var successfullCodes = Start(new RushResolverSettings {
SuccessExitCodes = new[] { 42 }
})
.Add(project)
.ScheduleAll()
.RetrieveSuccessfulProcess(project)
.SuccessExitCodes;
XAssert.Contains(successfullCodes, 42);
}
public async Task CmdWithTestShim_ShimNothingRunsChildProcessWithoutShimAsync(bool shimAllProcess, bool filterMatch)
{
var context = BuildXLContext.CreateInstanceForTesting();
var shimProgramPath = GetShimProgramPath(context);
string executable = CmdHelper.CmdX64;
string processMatch = filterMatch ? null : "foo.exe"; // Filter should never match foo.exe.
var fam = CreateCommonFileAccessManifest(context.PathTable);
fam.SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: shimAllProcess,
processMatches: processMatch == null
? new ShimProcessMatch[0]
: new[] { new ShimProcessMatch(PathAtom.Create(context.StringTable, processMatch), PathAtom.Invalid) });
string predicate = shimAllProcess ? string.Empty : "not ";
string childOutput = $"Child cmd that should {predicate}be shimmed";
string childArgs = $"{executable} /D /C @echo {childOutput}";
string args = "/D /C echo Top-level cmd. Running child process && " + childArgs;
var stdOutSb = new StringBuilder(128);
var stdErrSb = new StringBuilder();
SandboxedProcessInfo sandboxedProcessInfo = CreateCommonSandboxedProcessInfo(
context,
executable,
args,
fam,
stdOutSb,
stdErrSb);
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
string stdOut = stdOutSb.ToString();
string stdErr = stdErrSb.ToString();
m_output.WriteLine($"stdout: {stdOut}");
m_output.WriteLine($"stderr: {stdErr}");
AssertSuccess(result, stdErr);
AssertShimmedIf(stdOut, shimAllProcess);
XAssert.Contains(stdOut, childOutput);
}
public void BreakawayProcessIsNotDetoured()
{
// TODO: doesn't currently work on Linux
if (OperatingSystemHelper.IsLinuxOS)
{
return;
}
var fam = new FileAccessManifest(
Context.PathTable,
childProcessesToBreakawayFromSandbox: new[] { TestProcessToolName })
{
FailUnexpectedFileAccesses = false,
ReportUnexpectedFileAccesses = true,
ReportFileAccesses = true
};
var srcFile1 = CreateSourceFile();
var srcFile2 = CreateSourceFile();
var info = ToProcessInfo(
ToProcess(
Operation.ReadFile(srcFile1),
Operation.Spawn(
Context.PathTable,
true,
Operation.ReadFile(srcFile2))),
fileAccessManifest: fam);
var result = RunProcess(info).GetAwaiter().GetResult();
XAssert.AreEqual(0, result.ExitCode);
var observedAccesses = result.FileAccesses
.Select(reportedAccess => AbsolutePath.TryCreate(Context.PathTable, reportedAccess.GetPath(Context.PathTable), out AbsolutePath result) ? result : AbsolutePath.Invalid)
.ToArray();
// We should see the access that happens on the main test process
XAssert.Contains(observedAccesses, srcFile1.Path);
// We shouldn't see the access that happens on the spawned process
XAssert.ContainsNot(observedAccesses, srcFile2.Path);
// Only a single process should be reported: the parent one
var testProcess = ExcludeInjectedOnes(result.Processes).Single();
XAssert.AreEqual(TestProcessToolName.ToLowerInvariant(), Path.GetFileName(testProcess.Path).ToLowerInvariant());
}
public void ExistingErrorFileGetsProperlyReported()
{
Configuration.Sandbox.OutputReportingMode = global::BuildXL.Utilities.Configuration.OutputReportingMode.TruncatedOutputOnError;
var text = @"
* BEFORE *
* <error> *
* err1 *
* </error> *
* AFTER *
* <error>err2</error> * <error>err3</error> *
";
var errRegex = "error";
var ops = SplitLines(text)
.Select(l => Operation.Echo(l, true))
.Concat(new[]
{
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.Fail()
});
var pipBuilder = CreatePipBuilder(ops);
pipBuilder.ErrorRegex = new RegexDescriptor(StringId.Create(Context.StringTable, errRegex), RegexOptions.None);
pipBuilder.EnableMultiLineErrorScanning = false;
Process pip = SchedulePipBuilder(pipBuilder).Process;
// Let's make sure a pip process error gets reported
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.PipProcessError);
// Rerun the pip. Same type of error should occur
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.PipProcessError);
// Retrieve the error messages for the pip process error for both runs
var pipProcessErrorMessages = EventListener.GetLogMessagesForEventId((int)LogEventId.PipProcessError);
XAssert.AreEqual(2, pipProcessErrorMessages.Length);
// On the first run, standard error should have been copied to a file named err.txt
XAssert.Contains(pipProcessErrorMessages[0], "err.txt");
// On the second run, the file existed already, so it should have been renamed
XAssert.Contains(pipProcessErrorMessages[1], "err.1.txt");
}
private static bool TryVerifySingleReport(PathTable pathTable, ReportedFileAccess actualReport, AccessType access, Dictionary <AbsolutePath, ExpectedReportEntry> pathsToExpectations, out AbsolutePath actualReportedPath)
{
string actualReportedPathString = actualReport.GetPath(pathTable);
if (!AbsolutePath.TryCreate(pathTable, actualReportedPathString, out actualReportedPath))
{
return(false);
}
ExpectedReportEntry expected;
if (!pathsToExpectations.TryGetValue(actualReportedPath, out expected))
{
return(false);
}
XAssert.AreEqual(expected.Exists, !actualReport.IsNonexistent, "Bad assumption on file existence of {0}; this can break ACL decisions", actualReportedPathString);
XAssert.AreEqual(
expected.Allowed ? FileAccessStatus.Allowed : FileAccessStatus.Denied,
actualReport.Status,
"Incorrect ACL decision for " + actualReportedPathString);
// on MacOS we cannot distinguish Read vs. Probe for absent files so we always report Probe in those cases
var expectedAccess = OperatingSystemHelper.IsUnixOS && actualReport.IsNonexistent
? RequestedAccess.Probe
: access == AccessType.Read ? RequestedAccess.Read : RequestedAccess.Probe;
XAssert.AreEqual(
expectedAccess,
actualReport.RequestedAccess,
"Incorrect access type for path " + actualReportedPathString);
if (expectedAccess == RequestedAccess.Read)
{
var allowedOperations = !OperatingSystemHelper.IsUnixOS
? new[] { ReportedFileOperation.CreateFile }
: expected.Exists
? new[] { ReportedFileOperation.KAuthVNodeRead, ReportedFileOperation.KAuthReadFile }
: new[] { ReportedFileOperation.MacLookup };
XAssert.Contains(allowedOperations, actualReport.Operation);
}
return(true);
}
public void DirectoryMembershipExistenceTest()
{
CacheMissData cacheMiss;
DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(ReadonlyRoot));
Directory.CreateDirectory(ArtifactToString(dir));
Process pip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.EnumerateDir(dir),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnorePreloadedDlls = true;
FileArtifact srcFile1 = CreateSourceFile(dir);
File.WriteAllText(ArtifactToString(srcFile1), "member1");
RunScheduler().AssertCacheMiss(pip.PipId);
ScheduleRunResult buildA = RunScheduler().AssertCacheHit(pip.PipId);
FileArtifact srcFile2 = CreateSourceFile(dir);
File.WriteAllText(ArtifactToString(srcFile2), "member2");
ScheduleRunResult buildB = RunScheduler(m_testHooks).AssertCacheMiss(pip.PipId);
XAssert.IsTrue(m_testHooks.FingerprintStoreTestHooks.TryGetCacheMiss(pip.PipId, out cacheMiss));
XAssert.Contains(cacheMiss.DetailAndResult.Detail.Info.ToString(), srcFile2.Path.GetName(Context.PathTable).ToString(Context.PathTable.StringTable));
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnorePreloadedDlls = false;
ScheduleRunResult buildC = RunScheduler(m_testHooks).AssertCacheMiss(pip.PipId);
XAssert.IsTrue(m_testHooks.FingerprintStoreTestHooks.TryGetCacheMiss(pip.PipId, out cacheMiss));
XAssert.Contains(cacheMiss.DetailAndResult.Detail.Info.ToString(), ObservedPathSet.Labels.UnsafeOptions);
FileArtifact srcFile3 = CreateSourceFile(dir);
File.WriteAllText(ArtifactToString(srcFile3), "member3");
ScheduleRunResult buildD = RunScheduler(m_testHooks).AssertCacheMiss(pip.PipId);
XAssert.IsTrue(m_testHooks.FingerprintStoreTestHooks.TryGetCacheMiss(pip.PipId, out cacheMiss));
XAssert.Contains(cacheMiss.DetailAndResult.Detail.Info.ToString(), srcFile3.Path.GetName(Context.PathTable).ToString(Context.PathTable.StringTable));
XAssert.ContainsNot(cacheMiss.DetailAndResult.Detail.Info.ToString(), RepeatedStrings.MissingDirectoryMembershipFingerprint);
}
public void BreakawayProcessesAreUntracked()
{
var outerSourceFile = CreateSourceFileWithPrefix(SourceRoot, prefix: $"{nameof(BreakawayProcessesAreUntracked)}-outer-src");
var innerSourceFile = CreateSourceFileWithPrefix(SourceRoot, prefix: $"{nameof(BreakawayProcessesAreUntracked)}-inner-src");
var outerOutputFile = CreateOutputFileArtifact(ObjectRoot, prefix: $"{nameof(BreakawayProcessesAreUntracked)}-outer-out");
var innerOutputFile = CreateOutputFileArtifact(ObjectRoot, prefix: $"{nameof(BreakawayProcessesAreUntracked)}-inner-out");
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(outerSourceFile),
Operation.WriteFile(outerOutputFile),
Operation.Spawn(Context.PathTable, waitToFinish: true,
Operation.WriteFile(innerOutputFile, doNotInfer: true),
Operation.ReadFile(innerSourceFile, doNotInfer: true))
});
// Configure the test process itself to escape the sandbox
builder.ChildProcessesToBreakawayFromSandbox = ReadOnlyArray <PathAtom> .FromWithoutCopy(new[] { PathAtom.Create(Context.StringTable, TestProcessToolName) });
var pip = SchedulePipBuilder(builder);
// 1st run: success + cache miss
var result = RunScheduler().AssertSuccess().AssertCacheMiss(pip.Process.PipId);
// there should be a single "produced output" message and it should be for the outer output file
var producedOutputLogMessage = EventListener.GetLogMessagesForEventId((int)LogEventId.PipOutputProduced).Single().ToCanonicalizedPath();
XAssert.Contains(producedOutputLogMessage, ToString(outerOutputFile).ToCanonicalizedPath());
XAssert.ContainsNot(producedOutputLogMessage, ToString(innerOutputFile).ToCanonicalizedPath());
// 2nd run (no changes): success + up to date
RunScheduler().AssertSuccess().AssertCacheHit(pip.Process.PipId);
// 3nd run (change inner source file): success + cache hit (because inner source is untracked)
File.WriteAllText(ToString(innerSourceFile), contents: Guid.NewGuid().ToString());
RunScheduler().AssertSuccess().AssertCacheHit(pip.Process.PipId);
// 4th run (change outer source file): success + cache miss
File.WriteAllText(ToString(outerSourceFile), contents: Guid.NewGuid().ToString());
RunScheduler().AssertSuccess().AssertCacheMiss(pip.Process.PipId);
}
