示例#1
0
        public async Task TombstoneFileDoesNotRepresentARequiredOutput()
        {
            var context = BuildXLContext.CreateInstanceForTesting();

            using (var tempFiles = new TempFileStorage(canGetFileNames: true))
            {
                var pathTable = context.PathTable;

                var outputFile     = tempFiles.GetUniqueFileName();
                var outputFilePath = AbsolutePath.Create(pathTable, outputFile);

                var renamedFile     = $"{outputFile}.renamed";
                var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);

                // Arguments to create an output file that gets immediately renamed.
                // However, both files are marked as required, even though the original one
                // is not there after the rename happens
                var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
                var outputs   = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
                    new[]
                {
                    FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
                    FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
                });

                var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
                // the move command under the console seems to have some issue with the detours policy
                // This is orthogonal to the test, and we don't care about detours at this point
                var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);

                // The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
                AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
                AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
            }
        }
示例#2
0
        /// <summary>
        /// Verifies the resulting arguments after full graph construction.
        /// </summary>
        /// <param name="graph">Resulting graph.</param>
        /// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param>
        /// <param name="outputInFragmentOrigin">Output file to identify pip.</param>
        /// <param name="expectedArguments">Expected arguments.</param>
        private void VerifyResultingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, FileArtifact outputInFragmentOrigin, PipData expectedArguments)
        {
            var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputInFragmentOrigin)));

            XAssert.IsTrue(pipId.IsValid);

            Pip pip = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip);

            XAssert.IsNotNull(pip);

            PipData actualArguments = PipData.Invalid;

            if (pip is Process process)
            {
                actualArguments = process.Arguments;
            }
            else if (pip is IpcPip ipcPip)
            {
                actualArguments = ipcPip.MessageBody;
            }
            else
            {
                XAssert.Fail("No arguments associated with pip");
            }

            string expected = expectedArguments.ToString(Context.PathTable).ToUpperInvariant();
            string actual   = actualArguments.ToString(Context.PathTable).ToUpperInvariant();

            XAssert.AreEqual(expected, actual);
        }
示例#3
0
        private ContentFingerprint CreateFingerprintForPartialSealWithMember(string fileName)
        {
            using (TestEnv env = TestEnv.CreateTestEnvWithPausedScheduler())
            {
                AbsolutePath root   = env.Paths.CreateAbsolutePath(@"\\dummyPath\Root");
                FileArtifact member = FileArtifact.CreateSourceFile(root.Combine(env.PathTable, fileName));

                var staticDirectory = env.PipConstructionHelper.SealDirectoryPartial(
                    root,
                    new[] { member });

                var pipBuilder = CreatePipBuilderWithTag(env, nameof(TestPartialSealDirectoryMembersAffectFingerprints));
                var outputPath = env.Paths.CreateAbsolutePath(@"\\dummyPath\out");
                pipBuilder.AddOutputFile(outputPath);
                pipBuilder.AddInputDirectory(staticDirectory);

                env.PipConstructionHelper.AddProcess(pipBuilder);
                var graph      = AssertSuccessGraphBuilding(env);
                var producerId = graph.TryGetProducer(FileArtifact.CreateOutputFile(outputPath));

                XAssert.IsTrue(producerId.IsValid);
                XAssert.IsTrue(graph.TryGetPipFingerprint(producerId, out ContentFingerprint fingerprint));

                return(fingerprint);
            }
        }
示例#4
0
        public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
        {
            var context = BuildXLContext.CreateInstanceForTesting();

            using (var tempFiles = new TempFileStorage(canGetFileNames: true))
            {
                var pathTable = context.PathTable;

                var outputFile       = tempFiles.GetUniqueFileName(@"outputs\");
                var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");

                var outputFilePath       = AbsolutePath.Create(pathTable, outputFile);
                var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);

                FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));

                // Arguments to create two output files, one nested under the other
                var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
                var outputs   = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
                    new[]
                {
                    FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
                    FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
                });

                var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
                var pipExecutionResult = await RunProcess(context, pip);

                // Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
                XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
            }
        }
示例#5
0
        private void DoTestRenderer(PathTable pathTable, PipFragmentRenderer renderer, string expectedHash)
        {
            // StringId
            var strValue = "my string";

            XAssert.AreEqual(strValue, renderer.Render(PipFragment.FromString(strValue, pathTable.StringTable)));

            var pathStr   = A("t", "file1.txt");
            var path      = AbsolutePath.Create(pathTable, pathStr);
            var srcFile   = FileArtifact.CreateSourceFile(path);
            var outFile   = FileArtifact.CreateOutputFile(srcFile);
            var rwFile    = outFile.CreateNextWrittenVersion();
            var rw2File   = rwFile.CreateNextWrittenVersion();
            var opaqueDir = DirectoryArtifact.CreateDirectoryArtifactForTesting(path, 0);
            var sharedDir = new DirectoryArtifact(path, 1, isSharedOpaque: true);

            // AbsolutePath
            XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(path)));
            XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(srcFile)));
            XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(outFile)));
            XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rwFile)));
            XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rw2File)));

            // VsoHash
            XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(srcFile)));
            XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(outFile)));
            XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rwFile)));
            XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rw2File)));

            XAssert.AreEqual(DirectoryId.ToString(opaqueDir), renderer.Render(PipFragment.DirectoryIdForTesting(opaqueDir)));
            XAssert.AreEqual(DirectoryId.ToString(sharedDir), renderer.Render(PipFragment.DirectoryIdForTesting(sharedDir)));
        }
示例#6
0
        /// <summary>
        /// Creates an output directory.
        /// </summary>
        protected ValueTuple <DirectoryArtifact, ReadOnlyArray <FileArtifactWithAttributes> > CreateOutputDirectory(
            AbsolutePath rootPath = default(AbsolutePath),
            RelativePath relativePathToDirectory = default(RelativePath),
            RelativePath[] relativePathToMembers = null)
        {
            var directory = CreateDirectory(rootPath, relativePathToDirectory);
            ReadOnlyArray <FileArtifactWithAttributes> members;

            if (relativePathToMembers == null || relativePathToMembers.Length == 0)
            {
                members = ReadOnlyArray <FileArtifactWithAttributes> .Empty;
            }
            else
            {
                var fullMemberNames = new FileArtifactWithAttributes[relativePathToMembers.Length];
                for (int i = 0; i < fullMemberNames.Length; ++i)
                {
                    fullMemberNames[i] = FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(directory.Path.Combine(Context.PathTable, relativePathToMembers[i])), FileExistence.Required);
                }

                members = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(fullMemberNames);
            }

            return(directory, members);
        }
示例#7
0
        /// <summary>
        /// Creates an IpcPip from a single string representing IPC operation payload.
        ///
        /// The standard output property of the created pip is set to <paramref name="workingDir"/>\stdout.txt
        /// </summary>
        internal static IpcPip CreateFromStringPayload(
            PipExecutionContext context,
            AbsolutePath workingDir,
            IpcClientInfo ipcInfo,
            string operationPayload,
            PipProvenance provenance,
            FileArtifact outputFile = default(FileArtifact),
            IEnumerable <PipId> servicePipDependencies            = null,
            IEnumerable <FileArtifact> fileDependencies           = null,
            IEnumerable <DirectoryArtifact> directoryDependencies = null,
            IEnumerable <StringId> tags = null,
            bool isServiceFinalization  = false,
            bool mustRunOnMaster        = false)
        {
            var stdoutPath = workingDir.Combine(context.PathTable, PathAtom.Create(context.StringTable, "stdout.txt"));
            var stdoutFile = outputFile.IsValid ? outputFile : FileArtifact.CreateOutputFile(stdoutPath);

            var pipDataBuilder = new PipDataBuilder(context.StringTable);

            pipDataBuilder.Add(operationPayload);

            return(new IpcPip(
                       ipcInfo,
                       arguments: pipDataBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
                       outputFile: stdoutFile,
                       servicePipDependencies: ToReadOnlyArray(servicePipDependencies),
                       fileDependencies: ToReadOnlyArray(fileDependencies),
                       directoryDependencies: ToReadOnlyArray(directoryDependencies),
                       skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty,
                       tags: ToReadOnlyArray(tags),
                       isServiceFinalization: isServiceFinalization,
                       mustRunOnMaster: mustRunOnMaster,
                       provenance: provenance));
        }
示例#8
0
        /// <summary>
        /// Creates the following process
        ///
        ///   CreateDir (sodPath)/(NestedDirSrc)
        ///   WriteFile (sodPath)/(NestedDirSrc)/(FileNameInSrc)
        ///   MoveDir   (sodPath)/(NestedDirSrc) (sodPath)/(NestedDirDest)
        ///   WriteFile (sodPath)/(NestedDirDest)/(FileNameInDest)
        /// </summary>
        private ProcessBuilder CreateMoveDirectoryProcessBuilder(AbsolutePath rootDir)
        {
            return(CreatePipBuilder(new[]
            {
                OpCreateDir(NestedDirSrc),
                OpWriteFile(NestedDirSrc, FileNameInSrc),
                OpMoveDir(NestedDirSrc, NestedDirDest),
                OpWriteFile(NestedDirDest, FileNameInDest)
            }));

            Operation OpCreateDir(string nestedDirName)
            {
                return(Operation.CreateDir(OutDir(nestedDirName), doNotInfer: true));
            }

            Operation OpWriteFile(string nestedDirName, string fileName)
            {
                var file = FileArtifact.CreateOutputFile(OutDir(nestedDirName).Path.Combine(Context.PathTable, fileName));

                return(Operation.WriteFile(file, doNotInfer: true));
            }

            Operation OpMoveDir(string srcDir, string destDir)
            {
                return(Operation.MoveDir(srcPath: OutDir(srcDir), destPath: OutDir(destDir)));
            }

            DirectoryArtifact OutDir(string nestedDirName)
            {
                return(OutputDirectory.Create(rootDir.Combine(Context.PathTable, nestedDirName)));
            }
        }
示例#9
0
        public void ToolEnumeratingFile()
        {
            var sealDirectoryPath = CreateUniqueDirectory(ObjectRoot);
            var path              = sealDirectoryPath.ToString(Context.PathTable);
            var nestedFile1       = CreateSourceFile(path);
            var nestedFile2       = CreateUniquePath("nonExistentFile", path);
            DirectoryArtifact dir = SealDirectory(sealDirectoryPath, SealDirectoryKind.SourceAllDirectories);

            Configuration.Sandbox.UnsafeSandboxConfigurationMutable.UnexpectedFileAccessesAreErrors = false;

            var ops = new Operation[]
            {
                Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(nestedFile2), doNotInfer: true, enumeratePattern: "*"),
                Operation.ReadFile(nestedFile1, doNotInfer: true),
                Operation.WriteFile(CreateOutputFileArtifact()),

                // This write will be allowed because UnexpectedFileAccessesAreErrors = false.
                // But this will make the above reporting of enumerating non-existing file as enumerating existing file.
                Operation.WriteFile(FileArtifact.CreateOutputFile(nestedFile2), doNotInfer: true)
            };

            var builder = CreatePipBuilder(ops);

            builder.AddInputDirectory(dir);
            Process pip = SchedulePipBuilder(builder).Process;

            var result = RunScheduler().AssertSuccess();

            AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 2);
            AssertWarningEventLogged(EventId.FileMonitoringWarning, count: 1);
        }
示例#10
0
        public void DBEventCountVerification()
        {
            Configuration.Logging.LogExecution = true;

            var fileOne = FileArtifact.CreateOutputFile(Combine(TestDirPath, "foo.txt"));
            var fileTwo = FileArtifact.CreateOutputFile(Combine(TestDirPath, "bar.txt"));
            var dirOne  = Path.Combine(ReadonlyRoot, "baz");

            Directory.CreateDirectory(dirOne);
            File.WriteAllText(Path.Combine(dirOne, "abc.txt"), "text");
            File.WriteAllText(Path.Combine(dirOne, "xyz.txt"), "text12");

            var pipA = CreateAndSchedulePipBuilder(new[]
            {
                Operation.WriteFile(fileOne),
            }).Process;

            var pipB = CreateAndSchedulePipBuilder(new[]
            {
                Operation.WriteFile(fileTwo),
            }).Process;

            var pipC = CreateAndSchedulePipBuilder(new[]
            {
                Operation.ReadFile(fileOne),
                Operation.WriteFile(CreateOutputFileArtifact())
            }).Process;

            var pipD = CreateAndSchedulePipBuilder(new[]
            {
                Operation.EnumerateDir(new DirectoryArtifact(AbsolutePath.Create(Context.PathTable, dirOne), 0, false)),
                Operation.WriteFile(CreateOutputFileArtifact())
            }).Process;

            var buildA      = RunScheduler().AssertSuccess();
            var analyzerRes = RunAnalyzer(buildA).AssertSuccess();

            var dataStore = new XldbDataStore(storeDirectory: OutputDirPath.ToString(Context.PathTable));

            // As per Mike's offline comment, non-zero vs zero event count tests for now until we can rent out some mac machines
            // and figure out the true reason why the windows and mac event log counts differ by so much

            // For these tests, there should be a non-zero number of events logged
            XAssert.AreNotEqual(0, dataStore.GetFileArtifactContentDecidedEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetPipExecutionPerformanceEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetDirectoryMembershipHashedEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetProcessExecutionMonitoringReportedEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetProcessFingerprintComputationEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetBuildSessionConfigurationEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetPipExecutionStepPerformanceReportedEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetPipCacheMissEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetStatusReportedEvents().Count());
            XAssert.AreNotEqual(0, dataStore.GetBxlInvocationEvents().Count());

            // For these tests, there should be no events logged
            XAssert.AreEqual(0, dataStore.GetPipExecutionDirectoryOutputsEvents().Count());
            XAssert.AreEqual(0, dataStore.GetWorkerListEvents().Count());
            XAssert.AreEqual(0, dataStore.GetDependencyViolationReportedEvents().Count());
        }
示例#11
0
        /// <nodoc />
        public void SetStandardErrorFile(AbsolutePath path)
        {
            Contract.Requires(path.IsValid);
            Contract.Assert(!m_standardErrorFile.IsValid, "Value already set");

            AddOutputFile(path, FileExistence.Required);
            m_standardErrorFile = FileArtifact.CreateOutputFile(path);
        }
        public void WritesAreRedirected()
        {
            CreateSourceAndDestinationDirectories(out string sourceDirectory, out string destinationDirectory);

            var destinationFile = Path.Combine(sourceDirectory, "output.txt");

            var operations = new Operation[] { Operation.WriteFile(FileArtifact.CreateOutputFile(AbsolutePath.Create(PathTable, destinationFile))) };

            var containerConfiguration = ContainerConfiguration.CreateConfigurationForTesting(PathTable, new[] { (sourceDirectory, destinationDirectory) });
        public async Task CorrelateMoveFileAsync()
        {
            var context   = BuildXLContext.CreateInstanceForTesting();
            var pathTable = context.PathTable;

            using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
            {
                AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source");
                AbsolutePath sourceFile      = sourceDirectory.Combine(pathTable, "SourceFile.txt");
                var          destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt");
                WriteFile(pathTable, sourceFile, "content");

                var process = CreateDetourProcess(
                    context,
                    pathTable,
                    tempFiles,
                    argumentStr: "CorrelateMoveFile",
                    inputFiles: ReadOnlyArray <FileArtifact> .Empty,
                    inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
                    outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
                        FileArtifactWithAttributes.Create(
                            FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)),
                    outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
                    untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory));

                var correlator = new Correlator(pathTable);
                SandboxedProcessPipExecutionResult result = await RunProcessAsync(
                    pathTable : pathTable,
                    ignoreSetFileInformationByHandle : false,
                    ignoreZwRenameFileInformation : false,
                    monitorNtCreate : true,
                    ignoreReparsePoints : false,
                    ignoreNonCreateFileReparsePoints : false,
                    monitorZwCreateOpenQueryFile : false,
                    context : context,
                    pip : process,
                    detoursListener : correlator,
                    errorString : out _);

                VerifyNormalSuccess(context, result);

                XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable)));

                var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
                {
                    (sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed),
                    (destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed)
                };

                VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
                correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation(
                                                 destinationFile.ToString(pathTable),
                                                 ReportedFileOperation.MoveFileWithProgressDest,
                                                 sourceFile.ToString(pathTable),
                                                 ReportedFileOperation.MoveFileWithProgressSource));
            }
        }
示例#14
0
        private static Process CreateOutputFileProcess(BuildXLContext context, TempFileStorage tempFiles, AbsolutePath outputFilePath)
        {
            // Arguments to create an output file
            var arguments = $"echo hi > {outputFilePath.ToString(context.PathTable)}";
            var outputs   = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
                new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required) });

            var pip = CreateConsoleProcessInContainer(context, tempFiles, context.PathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());

            return(pip);
        }
示例#15
0
        /// <summary>
        /// Verifies that the file/directory output by a fragment exists in the resulting graph.
        /// </summary>
        /// <param name="graph">Resulting graph.</param>
        /// <param name="fragmentOrigin">Graph fragment where the output originates.</param>
        /// <param name="outputPath">Path to output file/directory.</param>
        private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath)
        {
            AbsolutePath remappedOutputPath = RemapFragmentPath(fragmentOrigin, outputPath);
            var          pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(remappedOutputPath));

            if (!pipId.IsValid)
            {
                pipId = graph.TryGetProducer(DirectoryArtifact.CreateWithZeroPartialSealId(remappedOutputPath));
            }

            XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph");
        }
示例#16
0
        public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel)
        {
            var context = BuildXLContext.CreateInstanceForTesting();

            using (var tempFiles = new TempFileStorage(canGetFileNames: true))
            {
                var pathTable = context.PathTable;

                var outputFile       = tempFiles.GetUniqueFileName(@"fileOutputs\");
                var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\");

                var outputFilePath       = AbsolutePath.Create(pathTable, outputFile);
                var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile);

                var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}";
                var outputs   = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
                    new[]
                {
                    FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
                });

                var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy(
                    new[]
                {
                    new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories),
                }
                    );

                var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel);
                var pipExecutionResult = await RunProcess(context, pip);

                XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);

                var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs");
                var redirectedFile        = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable);

                var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs");
                var redirectedOpaqueFile        = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable);

                // Make sure outputs got redirected based on the configured isolation level
                switch (containerIsolationLevel)
                {
                case ContainerIsolationLevel.IsolateOutputFiles:
                    XAssert.IsTrue(File.Exists(redirectedFile));
                    break;

                case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories:
                case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories:
                    XAssert.IsTrue(File.Exists(redirectedOpaqueFile));
                    break;
                }
            }
        }
示例#17
0
        /// <nodoc/>
        public bool TryAssertOutputExistenceInOpaqueDirectory(DirectoryArtifact outputDirectoryArtifact, AbsolutePath outputInOpaque, out FileArtifact fileArtifact)
        {
            Contract.Requires(outputDirectoryArtifact.IsValid);
            Contract.Requires(outputInOpaque.IsValid);

            // A null pip graph is the case of, for example, /phase:evaluate. Since there is no real execution going on, let's pretend the asserted file exists
            if (PipGraph is null)
            {
                // Outputs in opaques always have rewrite count 1
                fileArtifact = FileArtifact.CreateOutputFile(outputInOpaque);
                return(true);
            }

            return(PipGraph.TryAssertOutputExistenceInOpaqueDirectory(outputDirectoryArtifact, outputInOpaque, out fileArtifact));
        }
示例#18
0
        /// <summary>
        /// Verifies that the arguments constructed in the fragment (string) matches with the one in the resulting graph.
        /// </summary>
        /// <param name="graph">Resulting graph.</param>
        /// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param>
        /// <param name="processInFragment">Process in fragments whose arguments are to be verified.</param>
        private void VerifyMatchingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, Process processInFragment)
        {
            var outputPath = processInFragment.FileOutputs.First().ToFileArtifact().Path;
            var pipId      = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath)));

            XAssert.IsTrue(pipId.IsValid);

            Process processInGraph = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip) as Process;

            XAssert.IsNotNull(processInGraph);

            string argumentsInFragment = processInFragment.Arguments.ToString(fragmentOrigin.Context.PathTable).ToUpperInvariant();
            string argumentsInGraph    = processInGraph.Arguments.ToString(Context.PathTable).ToUpperInvariant();

            XAssert.AreEqual(argumentsInFragment, argumentsInGraph);
        }
示例#19
0
        [Trait("Category", "WindowsOSOnly")] // WriteFile operation failed on MacOS; need further investigation.
        public void MoveDirectoryFailed()
        {
            // Create \temp.
            var tempDirectoryPath = CreateUniqueDirectory(root: ObjectRoot, prefix: "temp");

            // Create \temp\out.
            var tempOutputDirectoryPath = CreateUniqueDirectory(root: tempDirectoryPath, prefix: "out");

            // Specify \temp\out\f.txt.
            FileArtifact tempOutput = FileArtifact.CreateOutputFile(tempOutputDirectoryPath.Combine(Context.PathTable, "f.txt"));

            // Specify \finalOut.
            DirectoryArtifact finalDirectory = CreateOutputDirectoryArtifact(ObjectRoot);

            // Specify \finalOut\f.txt.
            AbsolutePath finalOutput = finalDirectory.Path.Combine(Context.PathTable, "f.txt");

            var ops = new Operation[]
            {
                // Write to \temp\out\f.txt.
                Operation.WriteFile(tempOutput, content: "Hello", doNotInfer: true),

                // Move \temp\out to \finalOut
                Operation.MoveDir(DirectoryArtifact.CreateWithZeroPartialSealId(tempOutputDirectoryPath), finalDirectory),
            };

            var builder = CreatePipBuilder(ops);

            // Untrack \temp.
            builder.AddUntrackedDirectoryScope(tempDirectoryPath);

            // Specify \finalOut\f.txt as output.
            builder.AddOutputFile(finalOutput);

            SchedulePipBuilder(builder);

            // This test failed because in order to move \temp\out to \finalOut, \finalOut needs to be wiped out first.
            // However, to wipe out \finalOut, one needs to have a write access to \finalOut. Currently, we only have create-directory access
            // to \finalOut.
            RunScheduler().AssertFailure();

            // Error DX0064: The test process failed to execute an operation: MoveDir (access denied)
            // Error DX0500: Disallowed write access.
            SetExpectedFailures(2, 0);
        }
示例#20
0
        [Trait("Category", "WindowsOSOnly")] // WriteFile operation failed on MacOS; need further investigation.
        public void MoveDirectory()
        {
            // Create \temp.
            var tempDirectoryPath = CreateUniqueDirectory(root: ObjectRoot, prefix: "temp");

            // Create \temp\out.
            var tempOutputDirectoryPath = CreateUniqueDirectory(root: tempDirectoryPath, prefix: "out");

            // Specify \temp\out\f.txt.
            FileArtifact tempOutput = FileArtifact.CreateOutputFile(tempOutputDirectoryPath.Combine(Context.PathTable, "f.txt"));

            // Specify \final.
            DirectoryArtifact finalDirectory = CreateOutputDirectoryArtifact(ObjectRoot);

            // Specify \final\out.
            DirectoryArtifact finalOutputDirectory = CreateOutputDirectoryArtifact(finalDirectory.Path.ToString(Context.PathTable));

            // Specify \final\out\f.txt.
            AbsolutePath finalOutput = finalOutputDirectory.Path.Combine(Context.PathTable, "f.txt");

            var ops = new Operation[]
            {
                // Write to \temp\out\f.txt.
                Operation.WriteFile(tempOutput, content: "Hello", doNotInfer: true),

                // Move \temp\out to \final\out.
                Operation.MoveDir(DirectoryArtifact.CreateWithZeroPartialSealId(tempOutputDirectoryPath), finalOutputDirectory),
            };

            var builder = CreatePipBuilder(ops);

            // Untrack \temp.
            builder.AddUntrackedDirectoryScope(tempDirectoryPath);

            // Specify \final as opaque directory.
            builder.AddOutputDirectory(finalDirectory.Path, SealDirectoryKind.Opaque);

            // Specify explicitly \final\out\f.txt as output.
            builder.AddOutputFile(finalOutput);

            SchedulePipBuilder(builder);

            RunScheduler().AssertSuccess();
        }
示例#21
0
        public void ToStringTest()
        {
            var    pt = new PathTable();
            string f  = FileArtifact.CreateSourceFile(AbsolutePath.Create(pt, A("F"))).ToString();

            XAssert.IsTrue(f.Contains("File"), f);
            XAssert.IsTrue(f.Contains("(source)"), f);

            string g = FileArtifact.CreateOutputFile(AbsolutePath.Create(pt, A("G"))).ToString();

            XAssert.IsTrue(g.Contains("File"), g);
            XAssert.IsTrue(g.Contains("(output)"), g);

            string h = FileArtifact.CreateOutputFile(AbsolutePath.Create(pt, A("H"))).CreateNextWrittenVersion().ToString();

            XAssert.IsTrue(h.Contains("File"), h);
            XAssert.IsTrue(h.Contains("(rewrite:2)"), h);

            string i = FileArtifact.Invalid.ToString();

            XAssert.IsTrue(i.Contains("Invalid"), i);
        }
示例#22
0
        public void DBCreatedAndPopulated()
        {
            Configuration.Logging.LogExecution = true;

            var file = FileArtifact.CreateOutputFile(Combine(TestDirPath, "blah.txt"));

            var pipA = CreateAndSchedulePipBuilder(new[]
            {
                Operation.WriteFile(file),
            }).Process;

            var pipB = CreateAndSchedulePipBuilder(new[]
            {
                Operation.ReadFile(file),
                Operation.WriteFile(CreateOutputFileArtifact())
            }).Process;

            var buildA = RunScheduler().AssertSuccess();

            var analyzerRes = RunAnalyzer(buildA).AssertSuccess();

            XAssert.AreNotEqual(Directory.GetFiles(OutputDirPath.ToString(Context.PathTable), "*.sst").Length, 0);
        }
示例#23
0
 private FileArtifact CreateOutputFile(string path)
 {
     return(FileArtifact.CreateOutputFile(CreateAbsolutePath(path)));
 }
示例#24
0
 /// <summary>
 /// Creates an output file artifact.
 /// </summary>
 public FileArtifact CreateOutputFile(string relative) => FileArtifact.CreateOutputFile(CreateAbsolutePath(m_objectRoot, relative));
示例#25
0
        /// <summary>
        /// Compute the SHA-256 hash for file stored in Cache. Required for Build Manifets generation.
        /// </summary>
        private async Task <Possible <ContentHash> > ComputeBuildManifestHashFromCacheAsync(BuildManifestEntry buildManifestEntry)
        {
            if (!File.Exists(buildManifestEntry.FullFilePath))
            {
                // Ensure file is materialized locally
                if (!AbsolutePath.TryCreate(m_context.PathTable, buildManifestEntry.FullFilePath, out AbsolutePath path))
                {
                    return(new Failure <string>($"Invalid absolute path: '{buildManifestEntry.FullFilePath}'"));
                }

                MaterializeFileCommand materializeCommand = new MaterializeFileCommand(FileArtifact.CreateOutputFile(path), buildManifestEntry.FullFilePath);
                IIpcResult             materializeResult  = await ExecuteMaterializeFileAsync(materializeCommand);

                if (!materializeResult.Succeeded)
                {
                    return(new Failure <string>($"Unable to materialize file: '{buildManifestEntry.FullFilePath}' with hash: '{buildManifestEntry.Hash.Serialize()}'. Failure: {materializeResult.Payload}"));
                }
            }

            return(await TryGetBuildManifestHashFromLocalFileAsync(buildManifestEntry.FullFilePath));
        }
        /// <summary>
        /// Verifies that the file output by a fragment exists in the resulting graph.
        /// </summary>
        /// <param name="graph">Resulting graph.</param>
        /// <param name="fragmentOrigin">Graph fragment where the output originates.</param>
        /// <param name="outputPath">Path to output file.</param>
        private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath)
        {
            var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath)));

            XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph");
        }
示例#27
0
        private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId)
        {
            // IpcClientInfo
            IIpcMoniker moniker          = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false);
            int?        numRetries       = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true);
            int?        retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true);
            var         clientConfig     = new ClientConfig(numRetries, retryDelayMillis);
            var         ipcClientInfo    = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig);

            // target service pip
            PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService);

            // arguments
            PipData arguments;
            ReadOnlyArray <FileArtifact>      fileDependencies;
            ReadOnlyArray <DirectoryArtifact> directoryDependencies;

            using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder()))
            {
                // process arguments
                ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody);
                TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral);

                // input file dependencies
                var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true);
                if (dependenciesArray != null)
                {
                    for (int i = 0; i < dependenciesArray.Length; i++)
                    {
                        ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray));
                    }
                }

                arguments             = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
                fileDependencies      = ipcProcessBuilder.GetInputFilesSoFar();
                directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar();
            }

            // output
            AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true);

            if (!output.IsValid)
            {
                output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName);
            }

            // tags
            string[] tags      = null;
            var      tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true);

            if (tagsArray != null && tagsArray.Count > 0)
            {
                tags = new string[tagsArray.Count];
                for (int i = 0; i < tagsArray.Count; i++)
                {
                    tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray));
                }
            }

            // skip materialization for files
            FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>();
            ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true);

            if (skipMaterializationLiteral != null)
            {
                skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length];
                for (int i = 0; i < skipMaterializationLiteral.Length; i++)
                {
                    Converter.ExpectFileOrStaticDirectory(
                        skipMaterializationLiteral[i],
                        out var fileArtifact,
                        out var staticDirectory,
                        context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral));

                    Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null);

                    skipMaterializationArtifacts[i] = fileArtifact.IsValid
                        ? FileOrDirectoryArtifact.Create(fileArtifact)
                        : FileOrDirectoryArtifact.Create(staticDirectory.Root);
                }
            }

            // must run on master
            var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true;

            outputFile = FileArtifact.CreateOutputFile(output);

            // create IPC pip and add it to the graph
            bool result = context.GetPipConstructionHelper().TryAddIpc(
                ipcClientInfo,
                arguments,
                outputFile,
                servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
                fileDependencies: fileDependencies,
                directoryDependencies: directoryDependencies,
                skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts),
                isServiceFinalization: isServiceFinalization,
                mustRunOnMaster: mustRunOnMaster,
                tags: tags,
                out var ipcPip);

            pipId = ipcPip.PipId;

            return(result);
        }
示例#28
0
        public void TestAddingAndUnifyingIpcPip()
        {
            var fragment = CreatePipGraphFragmentTest(nameof(TestAddingAndUnifyingIpcPip));

            (IIpcMoniker moniker, PipId servicePipId) = TestPipGraphFragmentUtils.CreateService(fragment);

            var          processBuilder          = fragment.GetProcessBuilder();
            var          argumentsBuilder        = new ArgumentsBuilder(processBuilder);
            FileArtifact outputFileToVerify      = fragment.CreateOutputFile("g");
            AbsolutePath outputDirectoryToVerify = fragment.CreateOutputDirectory("d").Path;

            argumentsBuilder
            .AddInputFileOption("/input:", fragment.CreateSourceFile("f"))
            .AddOutputFileOption("/output:", outputFileToVerify.Path)
            .AddOutputDirectoryOption("/outputDir:", outputDirectoryToVerify)
            .Finish();
            (Process process, ProcessOutputs processOutputs) = fragment.ScheduleProcessBuilder(processBuilder);

            XAssert.IsTrue(processOutputs.TryGetOutputDirectory(outputDirectoryToVerify, out var outputDirectory));

            var addFileProcessBuilder = fragment.GetIpcProcessBuilder();

            new ArgumentsBuilder(addFileProcessBuilder)
            .AddStringOption("--command ", "addFile")
            .AddIpcMonikerOption("--ipcMoniker ", moniker)
            .AddInputFileOption("--file ", outputFileToVerify)
            .AddInputDirectoryOption("--directory ", outputDirectory.Root)
            .AddFileIdOption("--fileId ", outputFileToVerify)
            .AddDirectoryIdOption("--directoryId ", outputDirectory.Root)
            .AddVsoHashOption("--vsoHash ", outputFileToVerify)
            .Finish();

            FileArtifact ipcOutputFileToVerify;

            IpcPip ipcPip = fragment.ScheduleIpcPip(
                moniker,
                servicePipId,
                addFileProcessBuilder,
                ipcOutputFileToVerify = fragment.CreateOutputFile("add"),
                false);

            var graph = SerializeAndDeserializeIndependentFragments(fragment, fragment);

            VerifyGraphSuccessfullyConstructed(graph);
            VerifyProducerExists(graph, fragment, outputFileToVerify.Path);
            VerifyProducerExists(graph, fragment, outputDirectoryToVerify);
            VerifyProducerExists(graph, fragment, ipcOutputFileToVerify);

            var remappedOutputFile      = FileArtifact.CreateOutputFile(RemapFragmentPath(fragment, outputFileToVerify.Path));
            var remappedOutputDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(RemapFragmentPath(fragment, outputDirectory.Root));

            PipData expectedArguments = new ArgumentsDataBuilder(Context.StringTable)
                                        .AddStringOption("--command ", "addFile")
                                        .AddIpcMonikerOption("--ipcMoniker ", moniker)
                                        .AddPathOption("--file ", remappedOutputFile.Path)
                                        .AddPathOption("--directory ", remappedOutputDirectory.Path)
                                        .AddFileIdOption("--fileId ", remappedOutputFile)
                                        .AddDirectoryIdOption("--directoryId ", remappedOutputDirectory)
                                        .AddVsoHashOption("--vsoHash ", remappedOutputFile)
                                        .Finish();

            VerifyResultingArguments(graph, fragment, ipcOutputFileToVerify, expectedArguments);
        }
示例#29
0
 /// <summary>
 /// Creates an output file artifact.
 /// </summary>
 public FileArtifact CreateOutputFile(string relative) =>
 FileArtifact.CreateOutputFile(m_objectRoot.Combine(
                                   Context.PathTable, RelativePath.Create(Context.StringTable, relative)));
示例#30
0
 private FileArtifact OutputFile(string path) => FileArtifact.CreateOutputFile(AbsolutePath.Create(PathTable, path));