public async Task TombstoneFileDoesNotRepresentARequiredOutput() { var context = BuildXLContext.CreateInstanceForTesting(); using (var tempFiles = new TempFileStorage(canGetFileNames: true)) { var pathTable = context.PathTable; var outputFile = tempFiles.GetUniqueFileName(); var outputFilePath = AbsolutePath.Create(pathTable, outputFile); var renamedFile = $"{outputFile}.renamed"; var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile); // Arguments to create an output file that gets immediately renamed. // However, both files are marked as required, even though the original one // is not there after the rename happens var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}"; var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy( new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required), FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required) }); var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray()); // the move command under the console seems to have some issue with the detours policy // This is orthogonal to the test, and we don't care about detours at this point var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false); // The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit); AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs); } }
/// <summary> /// Verifies the resulting arguments after full graph construction. /// </summary> /// <param name="graph">Resulting graph.</param> /// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param> /// <param name="outputInFragmentOrigin">Output file to identify pip.</param> /// <param name="expectedArguments">Expected arguments.</param> private void VerifyResultingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, FileArtifact outputInFragmentOrigin, PipData expectedArguments) { var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputInFragmentOrigin))); XAssert.IsTrue(pipId.IsValid); Pip pip = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip); XAssert.IsNotNull(pip); PipData actualArguments = PipData.Invalid; if (pip is Process process) { actualArguments = process.Arguments; } else if (pip is IpcPip ipcPip) { actualArguments = ipcPip.MessageBody; } else { XAssert.Fail("No arguments associated with pip"); } string expected = expectedArguments.ToString(Context.PathTable).ToUpperInvariant(); string actual = actualArguments.ToString(Context.PathTable).ToUpperInvariant(); XAssert.AreEqual(expected, actual); }
private ContentFingerprint CreateFingerprintForPartialSealWithMember(string fileName) { using (TestEnv env = TestEnv.CreateTestEnvWithPausedScheduler()) { AbsolutePath root = env.Paths.CreateAbsolutePath(@"\\dummyPath\Root"); FileArtifact member = FileArtifact.CreateSourceFile(root.Combine(env.PathTable, fileName)); var staticDirectory = env.PipConstructionHelper.SealDirectoryPartial( root, new[] { member }); var pipBuilder = CreatePipBuilderWithTag(env, nameof(TestPartialSealDirectoryMembersAffectFingerprints)); var outputPath = env.Paths.CreateAbsolutePath(@"\\dummyPath\out"); pipBuilder.AddOutputFile(outputPath); pipBuilder.AddInputDirectory(staticDirectory); env.PipConstructionHelper.AddProcess(pipBuilder); var graph = AssertSuccessGraphBuilding(env); var producerId = graph.TryGetProducer(FileArtifact.CreateOutputFile(outputPath)); XAssert.IsTrue(producerId.IsValid); XAssert.IsTrue(graph.TryGetPipFingerprint(producerId, out ContentFingerprint fingerprint)); return(fingerprint); } }
public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations() { var context = BuildXLContext.CreateInstanceForTesting(); using (var tempFiles = new TempFileStorage(canGetFileNames: true)) { var pathTable = context.PathTable; var outputFile = tempFiles.GetUniqueFileName(@"outputs\"); var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\"); var outputFilePath = AbsolutePath.Create(pathTable, outputFile); var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested); FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable)); // Arguments to create two output files, one nested under the other var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}"; var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy( new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required), FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required) }); var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray()); var pipExecutionResult = await RunProcess(context, pip); // Observe that the fact the execution result succeeds means that the expected outputs are in their expected place XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status); } }
private void DoTestRenderer(PathTable pathTable, PipFragmentRenderer renderer, string expectedHash) { // StringId var strValue = "my string"; XAssert.AreEqual(strValue, renderer.Render(PipFragment.FromString(strValue, pathTable.StringTable))); var pathStr = A("t", "file1.txt"); var path = AbsolutePath.Create(pathTable, pathStr); var srcFile = FileArtifact.CreateSourceFile(path); var outFile = FileArtifact.CreateOutputFile(srcFile); var rwFile = outFile.CreateNextWrittenVersion(); var rw2File = rwFile.CreateNextWrittenVersion(); var opaqueDir = DirectoryArtifact.CreateDirectoryArtifactForTesting(path, 0); var sharedDir = new DirectoryArtifact(path, 1, isSharedOpaque: true); // AbsolutePath XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(path))); XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(srcFile))); XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(outFile))); XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rwFile))); XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rw2File))); // VsoHash XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(srcFile))); XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(outFile))); XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rwFile))); XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rw2File))); XAssert.AreEqual(DirectoryId.ToString(opaqueDir), renderer.Render(PipFragment.DirectoryIdForTesting(opaqueDir))); XAssert.AreEqual(DirectoryId.ToString(sharedDir), renderer.Render(PipFragment.DirectoryIdForTesting(sharedDir))); }
/// <summary> /// Creates an output directory. /// </summary> protected ValueTuple <DirectoryArtifact, ReadOnlyArray <FileArtifactWithAttributes> > CreateOutputDirectory( AbsolutePath rootPath = default(AbsolutePath), RelativePath relativePathToDirectory = default(RelativePath), RelativePath[] relativePathToMembers = null) { var directory = CreateDirectory(rootPath, relativePathToDirectory); ReadOnlyArray <FileArtifactWithAttributes> members; if (relativePathToMembers == null || relativePathToMembers.Length == 0) { members = ReadOnlyArray <FileArtifactWithAttributes> .Empty; } else { var fullMemberNames = new FileArtifactWithAttributes[relativePathToMembers.Length]; for (int i = 0; i < fullMemberNames.Length; ++i) { fullMemberNames[i] = FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(directory.Path.Combine(Context.PathTable, relativePathToMembers[i])), FileExistence.Required); } members = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(fullMemberNames); } return(directory, members); }
/// <summary> /// Creates an IpcPip from a single string representing IPC operation payload. /// /// The standard output property of the created pip is set to <paramref name="workingDir"/>\stdout.txt /// </summary> internal static IpcPip CreateFromStringPayload( PipExecutionContext context, AbsolutePath workingDir, IpcClientInfo ipcInfo, string operationPayload, PipProvenance provenance, FileArtifact outputFile = default(FileArtifact), IEnumerable <PipId> servicePipDependencies = null, IEnumerable <FileArtifact> fileDependencies = null, IEnumerable <DirectoryArtifact> directoryDependencies = null, IEnumerable <StringId> tags = null, bool isServiceFinalization = false, bool mustRunOnMaster = false) { var stdoutPath = workingDir.Combine(context.PathTable, PathAtom.Create(context.StringTable, "stdout.txt")); var stdoutFile = outputFile.IsValid ? outputFile : FileArtifact.CreateOutputFile(stdoutPath); var pipDataBuilder = new PipDataBuilder(context.StringTable); pipDataBuilder.Add(operationPayload); return(new IpcPip( ipcInfo, arguments: pipDataBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping), outputFile: stdoutFile, servicePipDependencies: ToReadOnlyArray(servicePipDependencies), fileDependencies: ToReadOnlyArray(fileDependencies), directoryDependencies: ToReadOnlyArray(directoryDependencies), skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty, tags: ToReadOnlyArray(tags), isServiceFinalization: isServiceFinalization, mustRunOnMaster: mustRunOnMaster, provenance: provenance)); }
/// <summary> /// Creates the following process /// /// CreateDir (sodPath)/(NestedDirSrc) /// WriteFile (sodPath)/(NestedDirSrc)/(FileNameInSrc) /// MoveDir (sodPath)/(NestedDirSrc) (sodPath)/(NestedDirDest) /// WriteFile (sodPath)/(NestedDirDest)/(FileNameInDest) /// </summary> private ProcessBuilder CreateMoveDirectoryProcessBuilder(AbsolutePath rootDir) { return(CreatePipBuilder(new[] { OpCreateDir(NestedDirSrc), OpWriteFile(NestedDirSrc, FileNameInSrc), OpMoveDir(NestedDirSrc, NestedDirDest), OpWriteFile(NestedDirDest, FileNameInDest) })); Operation OpCreateDir(string nestedDirName) { return(Operation.CreateDir(OutDir(nestedDirName), doNotInfer: true)); } Operation OpWriteFile(string nestedDirName, string fileName) { var file = FileArtifact.CreateOutputFile(OutDir(nestedDirName).Path.Combine(Context.PathTable, fileName)); return(Operation.WriteFile(file, doNotInfer: true)); } Operation OpMoveDir(string srcDir, string destDir) { return(Operation.MoveDir(srcPath: OutDir(srcDir), destPath: OutDir(destDir))); } DirectoryArtifact OutDir(string nestedDirName) { return(OutputDirectory.Create(rootDir.Combine(Context.PathTable, nestedDirName))); } }
public void ToolEnumeratingFile() { var sealDirectoryPath = CreateUniqueDirectory(ObjectRoot); var path = sealDirectoryPath.ToString(Context.PathTable); var nestedFile1 = CreateSourceFile(path); var nestedFile2 = CreateUniquePath("nonExistentFile", path); DirectoryArtifact dir = SealDirectory(sealDirectoryPath, SealDirectoryKind.SourceAllDirectories); Configuration.Sandbox.UnsafeSandboxConfigurationMutable.UnexpectedFileAccessesAreErrors = false; var ops = new Operation[] { Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(nestedFile2), doNotInfer: true, enumeratePattern: "*"), Operation.ReadFile(nestedFile1, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()), // This write will be allowed because UnexpectedFileAccessesAreErrors = false. // But this will make the above reporting of enumerating non-existing file as enumerating existing file. Operation.WriteFile(FileArtifact.CreateOutputFile(nestedFile2), doNotInfer: true) }; var builder = CreatePipBuilder(ops); builder.AddInputDirectory(dir); Process pip = SchedulePipBuilder(builder).Process; var result = RunScheduler().AssertSuccess(); AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 2); AssertWarningEventLogged(EventId.FileMonitoringWarning, count: 1); }
public void DBEventCountVerification() { Configuration.Logging.LogExecution = true; var fileOne = FileArtifact.CreateOutputFile(Combine(TestDirPath, "foo.txt")); var fileTwo = FileArtifact.CreateOutputFile(Combine(TestDirPath, "bar.txt")); var dirOne = Path.Combine(ReadonlyRoot, "baz"); Directory.CreateDirectory(dirOne); File.WriteAllText(Path.Combine(dirOne, "abc.txt"), "text"); File.WriteAllText(Path.Combine(dirOne, "xyz.txt"), "text12"); var pipA = CreateAndSchedulePipBuilder(new[] { Operation.WriteFile(fileOne), }).Process; var pipB = CreateAndSchedulePipBuilder(new[] { Operation.WriteFile(fileTwo), }).Process; var pipC = CreateAndSchedulePipBuilder(new[] { Operation.ReadFile(fileOne), Operation.WriteFile(CreateOutputFileArtifact()) }).Process; var pipD = CreateAndSchedulePipBuilder(new[] { Operation.EnumerateDir(new DirectoryArtifact(AbsolutePath.Create(Context.PathTable, dirOne), 0, false)), Operation.WriteFile(CreateOutputFileArtifact()) }).Process; var buildA = RunScheduler().AssertSuccess(); var analyzerRes = RunAnalyzer(buildA).AssertSuccess(); var dataStore = new XldbDataStore(storeDirectory: OutputDirPath.ToString(Context.PathTable)); // As per Mike's offline comment, non-zero vs zero event count tests for now until we can rent out some mac machines // and figure out the true reason why the windows and mac event log counts differ by so much // For these tests, there should be a non-zero number of events logged XAssert.AreNotEqual(0, dataStore.GetFileArtifactContentDecidedEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetPipExecutionPerformanceEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetDirectoryMembershipHashedEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetProcessExecutionMonitoringReportedEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetProcessFingerprintComputationEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetBuildSessionConfigurationEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetPipExecutionStepPerformanceReportedEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetPipCacheMissEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetStatusReportedEvents().Count()); XAssert.AreNotEqual(0, dataStore.GetBxlInvocationEvents().Count()); // For these tests, there should be no events logged XAssert.AreEqual(0, dataStore.GetPipExecutionDirectoryOutputsEvents().Count()); XAssert.AreEqual(0, dataStore.GetWorkerListEvents().Count()); XAssert.AreEqual(0, dataStore.GetDependencyViolationReportedEvents().Count()); }
/// <nodoc /> public void SetStandardErrorFile(AbsolutePath path) { Contract.Requires(path.IsValid); Contract.Assert(!m_standardErrorFile.IsValid, "Value already set"); AddOutputFile(path, FileExistence.Required); m_standardErrorFile = FileArtifact.CreateOutputFile(path); }
public void WritesAreRedirected() { CreateSourceAndDestinationDirectories(out string sourceDirectory, out string destinationDirectory); var destinationFile = Path.Combine(sourceDirectory, "output.txt"); var operations = new Operation[] { Operation.WriteFile(FileArtifact.CreateOutputFile(AbsolutePath.Create(PathTable, destinationFile))) }; var containerConfiguration = ContainerConfiguration.CreateConfigurationForTesting(PathTable, new[] { (sourceDirectory, destinationDirectory) });
public async Task CorrelateMoveFileAsync() { var context = BuildXLContext.CreateInstanceForTesting(); var pathTable = context.PathTable; using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory)) { AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source"); AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt"); var destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt"); WriteFile(pathTable, sourceFile, "content"); var process = CreateDetourProcess( context, pathTable, tempFiles, argumentStr: "CorrelateMoveFile", inputFiles: ReadOnlyArray <FileArtifact> .Empty, inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty, outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy( FileArtifactWithAttributes.Create( FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)), outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty, untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory)); var correlator = new Correlator(pathTable); SandboxedProcessPipExecutionResult result = await RunProcessAsync( pathTable : pathTable, ignoreSetFileInformationByHandle : false, ignoreZwRenameFileInformation : false, monitorNtCreate : true, ignoreReparsePoints : false, ignoreNonCreateFileReparsePoints : false, monitorZwCreateOpenQueryFile : false, context : context, pip : process, detoursListener : correlator, errorString : out _); VerifyNormalSuccess(context, result); XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable))); var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)> { (sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed), (destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed) }; VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray()); correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation( destinationFile.ToString(pathTable), ReportedFileOperation.MoveFileWithProgressDest, sourceFile.ToString(pathTable), ReportedFileOperation.MoveFileWithProgressSource)); } }
private static Process CreateOutputFileProcess(BuildXLContext context, TempFileStorage tempFiles, AbsolutePath outputFilePath) { // Arguments to create an output file var arguments = $"echo hi > {outputFilePath.ToString(context.PathTable)}"; var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy( new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required) }); var pip = CreateConsoleProcessInContainer(context, tempFiles, context.PathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray()); return(pip); }
/// <summary> /// Verifies that the file/directory output by a fragment exists in the resulting graph. /// </summary> /// <param name="graph">Resulting graph.</param> /// <param name="fragmentOrigin">Graph fragment where the output originates.</param> /// <param name="outputPath">Path to output file/directory.</param> private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath) { AbsolutePath remappedOutputPath = RemapFragmentPath(fragmentOrigin, outputPath); var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(remappedOutputPath)); if (!pipId.IsValid) { pipId = graph.TryGetProducer(DirectoryArtifact.CreateWithZeroPartialSealId(remappedOutputPath)); } XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph"); }
public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel) { var context = BuildXLContext.CreateInstanceForTesting(); using (var tempFiles = new TempFileStorage(canGetFileNames: true)) { var pathTable = context.PathTable; var outputFile = tempFiles.GetUniqueFileName(@"fileOutputs\"); var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\"); var outputFilePath = AbsolutePath.Create(pathTable, outputFile); var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile); var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}"; var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy( new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required), }); var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy( new[] { new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories), } ); var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel); var pipExecutionResult = await RunProcess(context, pip); XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status); var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs"); var redirectedFile = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable); var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs"); var redirectedOpaqueFile = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable); // Make sure outputs got redirected based on the configured isolation level switch (containerIsolationLevel) { case ContainerIsolationLevel.IsolateOutputFiles: XAssert.IsTrue(File.Exists(redirectedFile)); break; case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories: case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories: XAssert.IsTrue(File.Exists(redirectedOpaqueFile)); break; } } }
/// <nodoc/> public bool TryAssertOutputExistenceInOpaqueDirectory(DirectoryArtifact outputDirectoryArtifact, AbsolutePath outputInOpaque, out FileArtifact fileArtifact) { Contract.Requires(outputDirectoryArtifact.IsValid); Contract.Requires(outputInOpaque.IsValid); // A null pip graph is the case of, for example, /phase:evaluate. Since there is no real execution going on, let's pretend the asserted file exists if (PipGraph is null) { // Outputs in opaques always have rewrite count 1 fileArtifact = FileArtifact.CreateOutputFile(outputInOpaque); return(true); } return(PipGraph.TryAssertOutputExistenceInOpaqueDirectory(outputDirectoryArtifact, outputInOpaque, out fileArtifact)); }
/// <summary> /// Verifies that the arguments constructed in the fragment (string) matches with the one in the resulting graph. /// </summary> /// <param name="graph">Resulting graph.</param> /// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param> /// <param name="processInFragment">Process in fragments whose arguments are to be verified.</param> private void VerifyMatchingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, Process processInFragment) { var outputPath = processInFragment.FileOutputs.First().ToFileArtifact().Path; var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath))); XAssert.IsTrue(pipId.IsValid); Process processInGraph = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip) as Process; XAssert.IsNotNull(processInGraph); string argumentsInFragment = processInFragment.Arguments.ToString(fragmentOrigin.Context.PathTable).ToUpperInvariant(); string argumentsInGraph = processInGraph.Arguments.ToString(Context.PathTable).ToUpperInvariant(); XAssert.AreEqual(argumentsInFragment, argumentsInGraph); }
[Trait("Category", "WindowsOSOnly")] // WriteFile operation failed on MacOS; need further investigation. public void MoveDirectoryFailed() { // Create \temp. var tempDirectoryPath = CreateUniqueDirectory(root: ObjectRoot, prefix: "temp"); // Create \temp\out. var tempOutputDirectoryPath = CreateUniqueDirectory(root: tempDirectoryPath, prefix: "out"); // Specify \temp\out\f.txt. FileArtifact tempOutput = FileArtifact.CreateOutputFile(tempOutputDirectoryPath.Combine(Context.PathTable, "f.txt")); // Specify \finalOut. DirectoryArtifact finalDirectory = CreateOutputDirectoryArtifact(ObjectRoot); // Specify \finalOut\f.txt. AbsolutePath finalOutput = finalDirectory.Path.Combine(Context.PathTable, "f.txt"); var ops = new Operation[] { // Write to \temp\out\f.txt. Operation.WriteFile(tempOutput, content: "Hello", doNotInfer: true), // Move \temp\out to \finalOut Operation.MoveDir(DirectoryArtifact.CreateWithZeroPartialSealId(tempOutputDirectoryPath), finalDirectory), }; var builder = CreatePipBuilder(ops); // Untrack \temp. builder.AddUntrackedDirectoryScope(tempDirectoryPath); // Specify \finalOut\f.txt as output. builder.AddOutputFile(finalOutput); SchedulePipBuilder(builder); // This test failed because in order to move \temp\out to \finalOut, \finalOut needs to be wiped out first. // However, to wipe out \finalOut, one needs to have a write access to \finalOut. Currently, we only have create-directory access // to \finalOut. RunScheduler().AssertFailure(); // Error DX0064: The test process failed to execute an operation: MoveDir (access denied) // Error DX0500: Disallowed write access. SetExpectedFailures(2, 0); }
[Trait("Category", "WindowsOSOnly")] // WriteFile operation failed on MacOS; need further investigation. public void MoveDirectory() { // Create \temp. var tempDirectoryPath = CreateUniqueDirectory(root: ObjectRoot, prefix: "temp"); // Create \temp\out. var tempOutputDirectoryPath = CreateUniqueDirectory(root: tempDirectoryPath, prefix: "out"); // Specify \temp\out\f.txt. FileArtifact tempOutput = FileArtifact.CreateOutputFile(tempOutputDirectoryPath.Combine(Context.PathTable, "f.txt")); // Specify \final. DirectoryArtifact finalDirectory = CreateOutputDirectoryArtifact(ObjectRoot); // Specify \final\out. DirectoryArtifact finalOutputDirectory = CreateOutputDirectoryArtifact(finalDirectory.Path.ToString(Context.PathTable)); // Specify \final\out\f.txt. AbsolutePath finalOutput = finalOutputDirectory.Path.Combine(Context.PathTable, "f.txt"); var ops = new Operation[] { // Write to \temp\out\f.txt. Operation.WriteFile(tempOutput, content: "Hello", doNotInfer: true), // Move \temp\out to \final\out. Operation.MoveDir(DirectoryArtifact.CreateWithZeroPartialSealId(tempOutputDirectoryPath), finalOutputDirectory), }; var builder = CreatePipBuilder(ops); // Untrack \temp. builder.AddUntrackedDirectoryScope(tempDirectoryPath); // Specify \final as opaque directory. builder.AddOutputDirectory(finalDirectory.Path, SealDirectoryKind.Opaque); // Specify explicitly \final\out\f.txt as output. builder.AddOutputFile(finalOutput); SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); }
public void ToStringTest() { var pt = new PathTable(); string f = FileArtifact.CreateSourceFile(AbsolutePath.Create(pt, A("F"))).ToString(); XAssert.IsTrue(f.Contains("File"), f); XAssert.IsTrue(f.Contains("(source)"), f); string g = FileArtifact.CreateOutputFile(AbsolutePath.Create(pt, A("G"))).ToString(); XAssert.IsTrue(g.Contains("File"), g); XAssert.IsTrue(g.Contains("(output)"), g); string h = FileArtifact.CreateOutputFile(AbsolutePath.Create(pt, A("H"))).CreateNextWrittenVersion().ToString(); XAssert.IsTrue(h.Contains("File"), h); XAssert.IsTrue(h.Contains("(rewrite:2)"), h); string i = FileArtifact.Invalid.ToString(); XAssert.IsTrue(i.Contains("Invalid"), i); }
public void DBCreatedAndPopulated() { Configuration.Logging.LogExecution = true; var file = FileArtifact.CreateOutputFile(Combine(TestDirPath, "blah.txt")); var pipA = CreateAndSchedulePipBuilder(new[] { Operation.WriteFile(file), }).Process; var pipB = CreateAndSchedulePipBuilder(new[] { Operation.ReadFile(file), Operation.WriteFile(CreateOutputFileArtifact()) }).Process; var buildA = RunScheduler().AssertSuccess(); var analyzerRes = RunAnalyzer(buildA).AssertSuccess(); XAssert.AreNotEqual(Directory.GetFiles(OutputDirPath.ToString(Context.PathTable), "*.sst").Length, 0); }
private FileArtifact CreateOutputFile(string path) { return(FileArtifact.CreateOutputFile(CreateAbsolutePath(path))); }
/// <summary> /// Creates an output file artifact. /// </summary> public FileArtifact CreateOutputFile(string relative) => FileArtifact.CreateOutputFile(CreateAbsolutePath(m_objectRoot, relative));
/// <summary> /// Compute the SHA-256 hash for file stored in Cache. Required for Build Manifets generation. /// </summary> private async Task <Possible <ContentHash> > ComputeBuildManifestHashFromCacheAsync(BuildManifestEntry buildManifestEntry) { if (!File.Exists(buildManifestEntry.FullFilePath)) { // Ensure file is materialized locally if (!AbsolutePath.TryCreate(m_context.PathTable, buildManifestEntry.FullFilePath, out AbsolutePath path)) { return(new Failure <string>($"Invalid absolute path: '{buildManifestEntry.FullFilePath}'")); } MaterializeFileCommand materializeCommand = new MaterializeFileCommand(FileArtifact.CreateOutputFile(path), buildManifestEntry.FullFilePath); IIpcResult materializeResult = await ExecuteMaterializeFileAsync(materializeCommand); if (!materializeResult.Succeeded) { return(new Failure <string>($"Unable to materialize file: '{buildManifestEntry.FullFilePath}' with hash: '{buildManifestEntry.Hash.Serialize()}'. Failure: {materializeResult.Payload}")); } } return(await TryGetBuildManifestHashFromLocalFileAsync(buildManifestEntry.FullFilePath)); }
/// <summary> /// Verifies that the file output by a fragment exists in the resulting graph. /// </summary> /// <param name="graph">Resulting graph.</param> /// <param name="fragmentOrigin">Graph fragment where the output originates.</param> /// <param name="outputPath">Path to output file.</param> private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath) { var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath))); XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph"); }
private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId) { // IpcClientInfo IIpcMoniker moniker = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false); int? numRetries = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true); int? retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true); var clientConfig = new ClientConfig(numRetries, retryDelayMillis); var ipcClientInfo = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig); // target service pip PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService); // arguments PipData arguments; ReadOnlyArray <FileArtifact> fileDependencies; ReadOnlyArray <DirectoryArtifact> directoryDependencies; using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder())) { // process arguments ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody); TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral); // input file dependencies var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true); if (dependenciesArray != null) { for (int i = 0; i < dependenciesArray.Length; i++) { ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray)); } } arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); fileDependencies = ipcProcessBuilder.GetInputFilesSoFar(); directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar(); } // output AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true); if (!output.IsValid) { output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName); } // tags string[] tags = null; var tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true); if (tagsArray != null && tagsArray.Count > 0) { tags = new string[tagsArray.Count]; for (int i = 0; i < tagsArray.Count; i++) { tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray)); } } // skip materialization for files FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>(); ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true); if (skipMaterializationLiteral != null) { skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length]; for (int i = 0; i < skipMaterializationLiteral.Length; i++) { Converter.ExpectFileOrStaticDirectory( skipMaterializationLiteral[i], out var fileArtifact, out var staticDirectory, context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral)); Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null); skipMaterializationArtifacts[i] = fileArtifact.IsValid ? FileOrDirectoryArtifact.Create(fileArtifact) : FileOrDirectoryArtifact.Create(staticDirectory.Root); } } // must run on master var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true; outputFile = FileArtifact.CreateOutputFile(output); // create IPC pip and add it to the graph bool result = context.GetPipConstructionHelper().TryAddIpc( ipcClientInfo, arguments, outputFile, servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty, fileDependencies: fileDependencies, directoryDependencies: directoryDependencies, skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts), isServiceFinalization: isServiceFinalization, mustRunOnMaster: mustRunOnMaster, tags: tags, out var ipcPip); pipId = ipcPip.PipId; return(result); }
public void TestAddingAndUnifyingIpcPip() { var fragment = CreatePipGraphFragmentTest(nameof(TestAddingAndUnifyingIpcPip)); (IIpcMoniker moniker, PipId servicePipId) = TestPipGraphFragmentUtils.CreateService(fragment); var processBuilder = fragment.GetProcessBuilder(); var argumentsBuilder = new ArgumentsBuilder(processBuilder); FileArtifact outputFileToVerify = fragment.CreateOutputFile("g"); AbsolutePath outputDirectoryToVerify = fragment.CreateOutputDirectory("d").Path; argumentsBuilder .AddInputFileOption("/input:", fragment.CreateSourceFile("f")) .AddOutputFileOption("/output:", outputFileToVerify.Path) .AddOutputDirectoryOption("/outputDir:", outputDirectoryToVerify) .Finish(); (Process process, ProcessOutputs processOutputs) = fragment.ScheduleProcessBuilder(processBuilder); XAssert.IsTrue(processOutputs.TryGetOutputDirectory(outputDirectoryToVerify, out var outputDirectory)); var addFileProcessBuilder = fragment.GetIpcProcessBuilder(); new ArgumentsBuilder(addFileProcessBuilder) .AddStringOption("--command ", "addFile") .AddIpcMonikerOption("--ipcMoniker ", moniker) .AddInputFileOption("--file ", outputFileToVerify) .AddInputDirectoryOption("--directory ", outputDirectory.Root) .AddFileIdOption("--fileId ", outputFileToVerify) .AddDirectoryIdOption("--directoryId ", outputDirectory.Root) .AddVsoHashOption("--vsoHash ", outputFileToVerify) .Finish(); FileArtifact ipcOutputFileToVerify; IpcPip ipcPip = fragment.ScheduleIpcPip( moniker, servicePipId, addFileProcessBuilder, ipcOutputFileToVerify = fragment.CreateOutputFile("add"), false); var graph = SerializeAndDeserializeIndependentFragments(fragment, fragment); VerifyGraphSuccessfullyConstructed(graph); VerifyProducerExists(graph, fragment, outputFileToVerify.Path); VerifyProducerExists(graph, fragment, outputDirectoryToVerify); VerifyProducerExists(graph, fragment, ipcOutputFileToVerify); var remappedOutputFile = FileArtifact.CreateOutputFile(RemapFragmentPath(fragment, outputFileToVerify.Path)); var remappedOutputDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(RemapFragmentPath(fragment, outputDirectory.Root)); PipData expectedArguments = new ArgumentsDataBuilder(Context.StringTable) .AddStringOption("--command ", "addFile") .AddIpcMonikerOption("--ipcMoniker ", moniker) .AddPathOption("--file ", remappedOutputFile.Path) .AddPathOption("--directory ", remappedOutputDirectory.Path) .AddFileIdOption("--fileId ", remappedOutputFile) .AddDirectoryIdOption("--directoryId ", remappedOutputDirectory) .AddVsoHashOption("--vsoHash ", remappedOutputFile) .Finish(); VerifyResultingArguments(graph, fragment, ipcOutputFileToVerify, expectedArguments); }
/// <summary> /// Creates an output file artifact. /// </summary> public FileArtifact CreateOutputFile(string relative) => FileArtifact.CreateOutputFile(m_objectRoot.Combine( Context.PathTable, RelativePath.Create(Context.StringTable, relative)));
private FileArtifact OutputFile(string path) => FileArtifact.CreateOutputFile(AbsolutePath.Create(PathTable, path));