public async Task TombstoneFileDoesNotRepresentARequiredOutput()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var renamedFile = $"{outputFile}.renamed";
var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);
// Arguments to create an output file that gets immediately renamed.
// However, both files are marked as required, even though the original one
// is not there after the rename happens
var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
// the move command under the console seems to have some issue with the detours policy
// This is orthogonal to the test, and we don't care about detours at this point
var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);
// The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
}
}
/// <summary>
/// Verifies the resulting arguments after full graph construction.
/// </summary>
/// <param name="graph">Resulting graph.</param>
/// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param>
/// <param name="outputInFragmentOrigin">Output file to identify pip.</param>
/// <param name="expectedArguments">Expected arguments.</param>
private void VerifyResultingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, FileArtifact outputInFragmentOrigin, PipData expectedArguments)
{
var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputInFragmentOrigin)));
XAssert.IsTrue(pipId.IsValid);
Pip pip = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip);
XAssert.IsNotNull(pip);
PipData actualArguments = PipData.Invalid;
if (pip is Process process)
{
actualArguments = process.Arguments;
}
else if (pip is IpcPip ipcPip)
{
actualArguments = ipcPip.MessageBody;
}
else
{
XAssert.Fail("No arguments associated with pip");
}
string expected = expectedArguments.ToString(Context.PathTable).ToUpperInvariant();
string actual = actualArguments.ToString(Context.PathTable).ToUpperInvariant();
XAssert.AreEqual(expected, actual);
}
private ContentFingerprint CreateFingerprintForPartialSealWithMember(string fileName)
{
using (TestEnv env = TestEnv.CreateTestEnvWithPausedScheduler())
{
AbsolutePath root = env.Paths.CreateAbsolutePath(@"\\dummyPath\Root");
FileArtifact member = FileArtifact.CreateSourceFile(root.Combine(env.PathTable, fileName));
var staticDirectory = env.PipConstructionHelper.SealDirectoryPartial(
root,
new[] { member });
var pipBuilder = CreatePipBuilderWithTag(env, nameof(TestPartialSealDirectoryMembersAffectFingerprints));
var outputPath = env.Paths.CreateAbsolutePath(@"\\dummyPath\out");
pipBuilder.AddOutputFile(outputPath);
pipBuilder.AddInputDirectory(staticDirectory);
env.PipConstructionHelper.AddProcess(pipBuilder);
var graph = AssertSuccessGraphBuilding(env);
var producerId = graph.TryGetProducer(FileArtifact.CreateOutputFile(outputPath));
XAssert.IsTrue(producerId.IsValid);
XAssert.IsTrue(graph.TryGetPipFingerprint(producerId, out ContentFingerprint fingerprint));
return(fingerprint);
}
}
public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"outputs\");
var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);
FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));
// Arguments to create two output files, one nested under the other
var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
private void DoTestRenderer(PathTable pathTable, PipFragmentRenderer renderer, string expectedHash)
{
// StringId
var strValue = "my string";
XAssert.AreEqual(strValue, renderer.Render(PipFragment.FromString(strValue, pathTable.StringTable)));
var pathStr = A("t", "file1.txt");
var path = AbsolutePath.Create(pathTable, pathStr);
var srcFile = FileArtifact.CreateSourceFile(path);
var outFile = FileArtifact.CreateOutputFile(srcFile);
var rwFile = outFile.CreateNextWrittenVersion();
var rw2File = rwFile.CreateNextWrittenVersion();
var opaqueDir = DirectoryArtifact.CreateDirectoryArtifactForTesting(path, 0);
var sharedDir = new DirectoryArtifact(path, 1, isSharedOpaque: true);
// AbsolutePath
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(path)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(srcFile)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(outFile)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rwFile)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rw2File)));
// VsoHash
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(srcFile)));
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(outFile)));
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rwFile)));
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rw2File)));
XAssert.AreEqual(DirectoryId.ToString(opaqueDir), renderer.Render(PipFragment.DirectoryIdForTesting(opaqueDir)));
XAssert.AreEqual(DirectoryId.ToString(sharedDir), renderer.Render(PipFragment.DirectoryIdForTesting(sharedDir)));
}
/// <summary>
/// Creates an output directory.
/// </summary>
protected ValueTuple <DirectoryArtifact, ReadOnlyArray <FileArtifactWithAttributes> > CreateOutputDirectory(
AbsolutePath rootPath = default(AbsolutePath),
RelativePath relativePathToDirectory = default(RelativePath),
RelativePath[] relativePathToMembers = null)
{
var directory = CreateDirectory(rootPath, relativePathToDirectory);
ReadOnlyArray <FileArtifactWithAttributes> members;
if (relativePathToMembers == null || relativePathToMembers.Length == 0)
{
members = ReadOnlyArray <FileArtifactWithAttributes> .Empty;
}
else
{
var fullMemberNames = new FileArtifactWithAttributes[relativePathToMembers.Length];
for (int i = 0; i < fullMemberNames.Length; ++i)
{
fullMemberNames[i] = FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(directory.Path.Combine(Context.PathTable, relativePathToMembers[i])), FileExistence.Required);
}
members = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(fullMemberNames);
}
return(directory, members);
}
/// <summary>
/// Creates an IpcPip from a single string representing IPC operation payload.
///
/// The standard output property of the created pip is set to <paramref name="workingDir"/>\stdout.txt
/// </summary>
internal static IpcPip CreateFromStringPayload(
PipExecutionContext context,
AbsolutePath workingDir,
IpcClientInfo ipcInfo,
string operationPayload,
PipProvenance provenance,
FileArtifact outputFile = default(FileArtifact),
IEnumerable <PipId> servicePipDependencies = null,
IEnumerable <FileArtifact> fileDependencies = null,
IEnumerable <DirectoryArtifact> directoryDependencies = null,
IEnumerable <StringId> tags = null,
bool isServiceFinalization = false,
bool mustRunOnMaster = false)
{
var stdoutPath = workingDir.Combine(context.PathTable, PathAtom.Create(context.StringTable, "stdout.txt"));
var stdoutFile = outputFile.IsValid ? outputFile : FileArtifact.CreateOutputFile(stdoutPath);
var pipDataBuilder = new PipDataBuilder(context.StringTable);
pipDataBuilder.Add(operationPayload);
return(new IpcPip(
ipcInfo,
arguments: pipDataBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
outputFile: stdoutFile,
servicePipDependencies: ToReadOnlyArray(servicePipDependencies),
fileDependencies: ToReadOnlyArray(fileDependencies),
directoryDependencies: ToReadOnlyArray(directoryDependencies),
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty,
tags: ToReadOnlyArray(tags),
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: mustRunOnMaster,
provenance: provenance));
}
/// <summary>
/// Creates the following process
///
/// CreateDir (sodPath)/(NestedDirSrc)
/// WriteFile (sodPath)/(NestedDirSrc)/(FileNameInSrc)
/// MoveDir (sodPath)/(NestedDirSrc) (sodPath)/(NestedDirDest)
/// WriteFile (sodPath)/(NestedDirDest)/(FileNameInDest)
/// </summary>
private ProcessBuilder CreateMoveDirectoryProcessBuilder(AbsolutePath rootDir)
{
return(CreatePipBuilder(new[]
{
OpCreateDir(NestedDirSrc),
OpWriteFile(NestedDirSrc, FileNameInSrc),
OpMoveDir(NestedDirSrc, NestedDirDest),
OpWriteFile(NestedDirDest, FileNameInDest)
}));
Operation OpCreateDir(string nestedDirName)
{
return(Operation.CreateDir(OutDir(nestedDirName), doNotInfer: true));
}
Operation OpWriteFile(string nestedDirName, string fileName)
{
var file = FileArtifact.CreateOutputFile(OutDir(nestedDirName).Path.Combine(Context.PathTable, fileName));
return(Operation.WriteFile(file, doNotInfer: true));
}
Operation OpMoveDir(string srcDir, string destDir)
{
return(Operation.MoveDir(srcPath: OutDir(srcDir), destPath: OutDir(destDir)));
}
DirectoryArtifact OutDir(string nestedDirName)
{
return(OutputDirectory.Create(rootDir.Combine(Context.PathTable, nestedDirName)));
}
}
public void ToolEnumeratingFile()
{
var sealDirectoryPath = CreateUniqueDirectory(ObjectRoot);
var path = sealDirectoryPath.ToString(Context.PathTable);
var nestedFile1 = CreateSourceFile(path);
var nestedFile2 = CreateUniquePath("nonExistentFile", path);
DirectoryArtifact dir = SealDirectory(sealDirectoryPath, SealDirectoryKind.SourceAllDirectories);
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.UnexpectedFileAccessesAreErrors = false;
var ops = new Operation[]
{
Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(nestedFile2), doNotInfer: true, enumeratePattern: "*"),
Operation.ReadFile(nestedFile1, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()),
// This write will be allowed because UnexpectedFileAccessesAreErrors = false.
// But this will make the above reporting of enumerating non-existing file as enumerating existing file.
Operation.WriteFile(FileArtifact.CreateOutputFile(nestedFile2), doNotInfer: true)
};
var builder = CreatePipBuilder(ops);
builder.AddInputDirectory(dir);
Process pip = SchedulePipBuilder(builder).Process;
var result = RunScheduler().AssertSuccess();
AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 2);
AssertWarningEventLogged(EventId.FileMonitoringWarning, count: 1);
}
public void DBEventCountVerification()
{
Configuration.Logging.LogExecution = true;
var fileOne = FileArtifact.CreateOutputFile(Combine(TestDirPath, "foo.txt"));
var fileTwo = FileArtifact.CreateOutputFile(Combine(TestDirPath, "bar.txt"));
var dirOne = Path.Combine(ReadonlyRoot, "baz");
Directory.CreateDirectory(dirOne);
File.WriteAllText(Path.Combine(dirOne, "abc.txt"), "text");
File.WriteAllText(Path.Combine(dirOne, "xyz.txt"), "text12");
var pipA = CreateAndSchedulePipBuilder(new[]
{
Operation.WriteFile(fileOne),
}).Process;
var pipB = CreateAndSchedulePipBuilder(new[]
{
Operation.WriteFile(fileTwo),
}).Process;
var pipC = CreateAndSchedulePipBuilder(new[]
{
Operation.ReadFile(fileOne),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
var pipD = CreateAndSchedulePipBuilder(new[]
{
Operation.EnumerateDir(new DirectoryArtifact(AbsolutePath.Create(Context.PathTable, dirOne), 0, false)),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
var buildA = RunScheduler().AssertSuccess();
var analyzerRes = RunAnalyzer(buildA).AssertSuccess();
var dataStore = new XldbDataStore(storeDirectory: OutputDirPath.ToString(Context.PathTable));
// As per Mike's offline comment, non-zero vs zero event count tests for now until we can rent out some mac machines
// and figure out the true reason why the windows and mac event log counts differ by so much
// For these tests, there should be a non-zero number of events logged
XAssert.AreNotEqual(0, dataStore.GetFileArtifactContentDecidedEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetPipExecutionPerformanceEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetDirectoryMembershipHashedEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetProcessExecutionMonitoringReportedEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetProcessFingerprintComputationEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetBuildSessionConfigurationEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetPipExecutionStepPerformanceReportedEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetPipCacheMissEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetStatusReportedEvents().Count());
XAssert.AreNotEqual(0, dataStore.GetBxlInvocationEvents().Count());
// For these tests, there should be no events logged
XAssert.AreEqual(0, dataStore.GetPipExecutionDirectoryOutputsEvents().Count());
XAssert.AreEqual(0, dataStore.GetWorkerListEvents().Count());
XAssert.AreEqual(0, dataStore.GetDependencyViolationReportedEvents().Count());
}
/// <nodoc />
public void SetStandardErrorFile(AbsolutePath path)
{
Contract.Requires(path.IsValid);
Contract.Assert(!m_standardErrorFile.IsValid, "Value already set");
AddOutputFile(path, FileExistence.Required);
m_standardErrorFile = FileArtifact.CreateOutputFile(path);
}
public void WritesAreRedirected()
{
CreateSourceAndDestinationDirectories(out string sourceDirectory, out string destinationDirectory);
var destinationFile = Path.Combine(sourceDirectory, "output.txt");
var operations = new Operation[] { Operation.WriteFile(FileArtifact.CreateOutputFile(AbsolutePath.Create(PathTable, destinationFile))) };
var containerConfiguration = ContainerConfiguration.CreateConfigurationForTesting(PathTable, new[] { (sourceDirectory, destinationDirectory) });
public async Task CorrelateMoveFileAsync()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source");
AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt");
var destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt");
WriteFile(pathTable, sourceFile, "content");
var process = CreateDetourProcess(
context,
pathTable,
tempFiles,
argumentStr: "CorrelateMoveFile",
inputFiles: ReadOnlyArray <FileArtifact> .Empty,
inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
FileArtifactWithAttributes.Create(
FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)),
outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory));
var correlator = new Correlator(pathTable);
SandboxedProcessPipExecutionResult result = await RunProcessAsync(
pathTable : pathTable,
ignoreSetFileInformationByHandle : false,
ignoreZwRenameFileInformation : false,
monitorNtCreate : true,
ignoreReparsePoints : false,
ignoreNonCreateFileReparsePoints : false,
monitorZwCreateOpenQueryFile : false,
context : context,
pip : process,
detoursListener : correlator,
errorString : out _);
VerifyNormalSuccess(context, result);
XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable)));
var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
{
(sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed),
(destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed)
};
VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation(
destinationFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressDest,
sourceFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressSource));
}
}
private static Process CreateOutputFileProcess(BuildXLContext context, TempFileStorage tempFiles, AbsolutePath outputFilePath)
{
// Arguments to create an output file
var arguments = $"echo hi > {outputFilePath.ToString(context.PathTable)}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required) });
var pip = CreateConsoleProcessInContainer(context, tempFiles, context.PathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
return(pip);
}
/// <summary>
/// Verifies that the file/directory output by a fragment exists in the resulting graph.
/// </summary>
/// <param name="graph">Resulting graph.</param>
/// <param name="fragmentOrigin">Graph fragment where the output originates.</param>
/// <param name="outputPath">Path to output file/directory.</param>
private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath)
{
AbsolutePath remappedOutputPath = RemapFragmentPath(fragmentOrigin, outputPath);
var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(remappedOutputPath));
if (!pipId.IsValid)
{
pipId = graph.TryGetProducer(DirectoryArtifact.CreateWithZeroPartialSealId(remappedOutputPath));
}
XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph");
}
public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel)
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"fileOutputs\");
var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile);
var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
});
var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy(
new[]
{
new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories),
}
);
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel);
var pipExecutionResult = await RunProcess(context, pip);
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs");
var redirectedFile = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable);
var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs");
var redirectedOpaqueFile = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable);
// Make sure outputs got redirected based on the configured isolation level
switch (containerIsolationLevel)
{
case ContainerIsolationLevel.IsolateOutputFiles:
XAssert.IsTrue(File.Exists(redirectedFile));
break;
case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories:
case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories:
XAssert.IsTrue(File.Exists(redirectedOpaqueFile));
break;
}
}
}
/// <nodoc/>
public bool TryAssertOutputExistenceInOpaqueDirectory(DirectoryArtifact outputDirectoryArtifact, AbsolutePath outputInOpaque, out FileArtifact fileArtifact)
{
Contract.Requires(outputDirectoryArtifact.IsValid);
Contract.Requires(outputInOpaque.IsValid);
// A null pip graph is the case of, for example, /phase:evaluate. Since there is no real execution going on, let's pretend the asserted file exists
if (PipGraph is null)
{
// Outputs in opaques always have rewrite count 1
fileArtifact = FileArtifact.CreateOutputFile(outputInOpaque);
return(true);
}
return(PipGraph.TryAssertOutputExistenceInOpaqueDirectory(outputDirectoryArtifact, outputInOpaque, out fileArtifact));
}
/// <summary>
/// Verifies that the arguments constructed in the fragment (string) matches with the one in the resulting graph.
/// </summary>
/// <param name="graph">Resulting graph.</param>
/// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param>
/// <param name="processInFragment">Process in fragments whose arguments are to be verified.</param>
private void VerifyMatchingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, Process processInFragment)
{
var outputPath = processInFragment.FileOutputs.First().ToFileArtifact().Path;
var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath)));
XAssert.IsTrue(pipId.IsValid);
Process processInGraph = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip) as Process;
XAssert.IsNotNull(processInGraph);
string argumentsInFragment = processInFragment.Arguments.ToString(fragmentOrigin.Context.PathTable).ToUpperInvariant();
string argumentsInGraph = processInGraph.Arguments.ToString(Context.PathTable).ToUpperInvariant();
XAssert.AreEqual(argumentsInFragment, argumentsInGraph);
}
[Trait("Category", "WindowsOSOnly")] // WriteFile operation failed on MacOS; need further investigation.
public void MoveDirectoryFailed()
{
// Create \temp.
var tempDirectoryPath = CreateUniqueDirectory(root: ObjectRoot, prefix: "temp");
// Create \temp\out.
var tempOutputDirectoryPath = CreateUniqueDirectory(root: tempDirectoryPath, prefix: "out");
// Specify \temp\out\f.txt.
FileArtifact tempOutput = FileArtifact.CreateOutputFile(tempOutputDirectoryPath.Combine(Context.PathTable, "f.txt"));
// Specify \finalOut.
DirectoryArtifact finalDirectory = CreateOutputDirectoryArtifact(ObjectRoot);
// Specify \finalOut\f.txt.
AbsolutePath finalOutput = finalDirectory.Path.Combine(Context.PathTable, "f.txt");
var ops = new Operation[]
{
// Write to \temp\out\f.txt.
Operation.WriteFile(tempOutput, content: "Hello", doNotInfer: true),
// Move \temp\out to \finalOut
Operation.MoveDir(DirectoryArtifact.CreateWithZeroPartialSealId(tempOutputDirectoryPath), finalDirectory),
};
var builder = CreatePipBuilder(ops);
// Untrack \temp.
builder.AddUntrackedDirectoryScope(tempDirectoryPath);
// Specify \finalOut\f.txt as output.
builder.AddOutputFile(finalOutput);
SchedulePipBuilder(builder);
// This test failed because in order to move \temp\out to \finalOut, \finalOut needs to be wiped out first.
// However, to wipe out \finalOut, one needs to have a write access to \finalOut. Currently, we only have create-directory access
// to \finalOut.
RunScheduler().AssertFailure();
// Error DX0064: The test process failed to execute an operation: MoveDir (access denied)
// Error DX0500: Disallowed write access.
SetExpectedFailures(2, 0);
}
[Trait("Category", "WindowsOSOnly")] // WriteFile operation failed on MacOS; need further investigation.
public void MoveDirectory()
{
// Create \temp.
var tempDirectoryPath = CreateUniqueDirectory(root: ObjectRoot, prefix: "temp");
// Create \temp\out.
var tempOutputDirectoryPath = CreateUniqueDirectory(root: tempDirectoryPath, prefix: "out");
// Specify \temp\out\f.txt.
FileArtifact tempOutput = FileArtifact.CreateOutputFile(tempOutputDirectoryPath.Combine(Context.PathTable, "f.txt"));
// Specify \final.
DirectoryArtifact finalDirectory = CreateOutputDirectoryArtifact(ObjectRoot);
// Specify \final\out.
DirectoryArtifact finalOutputDirectory = CreateOutputDirectoryArtifact(finalDirectory.Path.ToString(Context.PathTable));
// Specify \final\out\f.txt.
AbsolutePath finalOutput = finalOutputDirectory.Path.Combine(Context.PathTable, "f.txt");
var ops = new Operation[]
{
// Write to \temp\out\f.txt.
Operation.WriteFile(tempOutput, content: "Hello", doNotInfer: true),
// Move \temp\out to \final\out.
Operation.MoveDir(DirectoryArtifact.CreateWithZeroPartialSealId(tempOutputDirectoryPath), finalOutputDirectory),
};
var builder = CreatePipBuilder(ops);
// Untrack \temp.
builder.AddUntrackedDirectoryScope(tempDirectoryPath);
// Specify \final as opaque directory.
builder.AddOutputDirectory(finalDirectory.Path, SealDirectoryKind.Opaque);
// Specify explicitly \final\out\f.txt as output.
builder.AddOutputFile(finalOutput);
SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
}
public void ToStringTest()
{
var pt = new PathTable();
string f = FileArtifact.CreateSourceFile(AbsolutePath.Create(pt, A("F"))).ToString();
XAssert.IsTrue(f.Contains("File"), f);
XAssert.IsTrue(f.Contains("(source)"), f);
string g = FileArtifact.CreateOutputFile(AbsolutePath.Create(pt, A("G"))).ToString();
XAssert.IsTrue(g.Contains("File"), g);
XAssert.IsTrue(g.Contains("(output)"), g);
string h = FileArtifact.CreateOutputFile(AbsolutePath.Create(pt, A("H"))).CreateNextWrittenVersion().ToString();
XAssert.IsTrue(h.Contains("File"), h);
XAssert.IsTrue(h.Contains("(rewrite:2)"), h);
string i = FileArtifact.Invalid.ToString();
XAssert.IsTrue(i.Contains("Invalid"), i);
}
public void DBCreatedAndPopulated()
{
Configuration.Logging.LogExecution = true;
var file = FileArtifact.CreateOutputFile(Combine(TestDirPath, "blah.txt"));
var pipA = CreateAndSchedulePipBuilder(new[]
{
Operation.WriteFile(file),
}).Process;
var pipB = CreateAndSchedulePipBuilder(new[]
{
Operation.ReadFile(file),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
var buildA = RunScheduler().AssertSuccess();
var analyzerRes = RunAnalyzer(buildA).AssertSuccess();
XAssert.AreNotEqual(Directory.GetFiles(OutputDirPath.ToString(Context.PathTable), "*.sst").Length, 0);
}
private FileArtifact CreateOutputFile(string path)
{
return(FileArtifact.CreateOutputFile(CreateAbsolutePath(path)));
}
/// <summary> /// Creates an output file artifact. /// </summary> public FileArtifact CreateOutputFile(string relative) => FileArtifact.CreateOutputFile(CreateAbsolutePath(m_objectRoot, relative));
/// <summary>
/// Compute the SHA-256 hash for file stored in Cache. Required for Build Manifets generation.
/// </summary>
private async Task <Possible <ContentHash> > ComputeBuildManifestHashFromCacheAsync(BuildManifestEntry buildManifestEntry)
{
if (!File.Exists(buildManifestEntry.FullFilePath))
{
// Ensure file is materialized locally
if (!AbsolutePath.TryCreate(m_context.PathTable, buildManifestEntry.FullFilePath, out AbsolutePath path))
{
return(new Failure <string>($"Invalid absolute path: '{buildManifestEntry.FullFilePath}'"));
}
MaterializeFileCommand materializeCommand = new MaterializeFileCommand(FileArtifact.CreateOutputFile(path), buildManifestEntry.FullFilePath);
IIpcResult materializeResult = await ExecuteMaterializeFileAsync(materializeCommand);
if (!materializeResult.Succeeded)
{
return(new Failure <string>($"Unable to materialize file: '{buildManifestEntry.FullFilePath}' with hash: '{buildManifestEntry.Hash.Serialize()}'. Failure: {materializeResult.Payload}"));
}
}
return(await TryGetBuildManifestHashFromLocalFileAsync(buildManifestEntry.FullFilePath));
}
/// <summary>
/// Verifies that the file output by a fragment exists in the resulting graph.
/// </summary>
/// <param name="graph">Resulting graph.</param>
/// <param name="fragmentOrigin">Graph fragment where the output originates.</param>
/// <param name="outputPath">Path to output file.</param>
private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath)
{
var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath)));
XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph");
}
private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId)
{
// IpcClientInfo
IIpcMoniker moniker = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false);
int? numRetries = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true);
int? retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true);
var clientConfig = new ClientConfig(numRetries, retryDelayMillis);
var ipcClientInfo = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig);
// target service pip
PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService);
// arguments
PipData arguments;
ReadOnlyArray <FileArtifact> fileDependencies;
ReadOnlyArray <DirectoryArtifact> directoryDependencies;
using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder()))
{
// process arguments
ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody);
TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral);
// input file dependencies
var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true);
if (dependenciesArray != null)
{
for (int i = 0; i < dependenciesArray.Length; i++)
{
ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray));
}
}
arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
fileDependencies = ipcProcessBuilder.GetInputFilesSoFar();
directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar();
}
// output
AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true);
if (!output.IsValid)
{
output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName);
}
// tags
string[] tags = null;
var tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true);
if (tagsArray != null && tagsArray.Count > 0)
{
tags = new string[tagsArray.Count];
for (int i = 0; i < tagsArray.Count; i++)
{
tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray));
}
}
// skip materialization for files
FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>();
ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true);
if (skipMaterializationLiteral != null)
{
skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length];
for (int i = 0; i < skipMaterializationLiteral.Length; i++)
{
Converter.ExpectFileOrStaticDirectory(
skipMaterializationLiteral[i],
out var fileArtifact,
out var staticDirectory,
context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral));
Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null);
skipMaterializationArtifacts[i] = fileArtifact.IsValid
? FileOrDirectoryArtifact.Create(fileArtifact)
: FileOrDirectoryArtifact.Create(staticDirectory.Root);
}
}
// must run on master
var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true;
outputFile = FileArtifact.CreateOutputFile(output);
// create IPC pip and add it to the graph
bool result = context.GetPipConstructionHelper().TryAddIpc(
ipcClientInfo,
arguments,
outputFile,
servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
fileDependencies: fileDependencies,
directoryDependencies: directoryDependencies,
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts),
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: mustRunOnMaster,
tags: tags,
out var ipcPip);
pipId = ipcPip.PipId;
return(result);
}
public void TestAddingAndUnifyingIpcPip()
{
var fragment = CreatePipGraphFragmentTest(nameof(TestAddingAndUnifyingIpcPip));
(IIpcMoniker moniker, PipId servicePipId) = TestPipGraphFragmentUtils.CreateService(fragment);
var processBuilder = fragment.GetProcessBuilder();
var argumentsBuilder = new ArgumentsBuilder(processBuilder);
FileArtifact outputFileToVerify = fragment.CreateOutputFile("g");
AbsolutePath outputDirectoryToVerify = fragment.CreateOutputDirectory("d").Path;
argumentsBuilder
.AddInputFileOption("/input:", fragment.CreateSourceFile("f"))
.AddOutputFileOption("/output:", outputFileToVerify.Path)
.AddOutputDirectoryOption("/outputDir:", outputDirectoryToVerify)
.Finish();
(Process process, ProcessOutputs processOutputs) = fragment.ScheduleProcessBuilder(processBuilder);
XAssert.IsTrue(processOutputs.TryGetOutputDirectory(outputDirectoryToVerify, out var outputDirectory));
var addFileProcessBuilder = fragment.GetIpcProcessBuilder();
new ArgumentsBuilder(addFileProcessBuilder)
.AddStringOption("--command ", "addFile")
.AddIpcMonikerOption("--ipcMoniker ", moniker)
.AddInputFileOption("--file ", outputFileToVerify)
.AddInputDirectoryOption("--directory ", outputDirectory.Root)
.AddFileIdOption("--fileId ", outputFileToVerify)
.AddDirectoryIdOption("--directoryId ", outputDirectory.Root)
.AddVsoHashOption("--vsoHash ", outputFileToVerify)
.Finish();
FileArtifact ipcOutputFileToVerify;
IpcPip ipcPip = fragment.ScheduleIpcPip(
moniker,
servicePipId,
addFileProcessBuilder,
ipcOutputFileToVerify = fragment.CreateOutputFile("add"),
false);
var graph = SerializeAndDeserializeIndependentFragments(fragment, fragment);
VerifyGraphSuccessfullyConstructed(graph);
VerifyProducerExists(graph, fragment, outputFileToVerify.Path);
VerifyProducerExists(graph, fragment, outputDirectoryToVerify);
VerifyProducerExists(graph, fragment, ipcOutputFileToVerify);
var remappedOutputFile = FileArtifact.CreateOutputFile(RemapFragmentPath(fragment, outputFileToVerify.Path));
var remappedOutputDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(RemapFragmentPath(fragment, outputDirectory.Root));
PipData expectedArguments = new ArgumentsDataBuilder(Context.StringTable)
.AddStringOption("--command ", "addFile")
.AddIpcMonikerOption("--ipcMoniker ", moniker)
.AddPathOption("--file ", remappedOutputFile.Path)
.AddPathOption("--directory ", remappedOutputDirectory.Path)
.AddFileIdOption("--fileId ", remappedOutputFile)
.AddDirectoryIdOption("--directoryId ", remappedOutputDirectory)
.AddVsoHashOption("--vsoHash ", remappedOutputFile)
.Finish();
VerifyResultingArguments(graph, fragment, ipcOutputFileToVerify, expectedArguments);
}
/// <summary>
/// Creates an output file artifact.
/// </summary>
public FileArtifact CreateOutputFile(string relative) =>
FileArtifact.CreateOutputFile(m_objectRoot.Combine(
Context.PathTable, RelativePath.Create(Context.StringTable, relative)));
private FileArtifact OutputFile(string path) => FileArtifact.CreateOutputFile(AbsolutePath.Create(PathTable, path));
