private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId)
        {
            SeparateExecutableFromCommands(node.Command, out string executable, out string args);

            if (!TryFindExecutablePath(executable, out AbsolutePath exePath))
            {
                Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command);
                return(false);
            }

            FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath);

            processBuilder.Executable = prExeArtifact;
            processBuilder.AddInputFile(prExeArtifact);

            using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
            {
                var pipDataBuilder = pipDataBuilderWrapper.Instance;
                pipDataBuilder.Add(args);
                processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
            }


            if (node.ResponseFile.HasValue)
            {
                using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
                {
                    var pipDataBuilder = pipDataBuilderWrapper.Instance;
                    pipDataBuilder.Add(node.ResponseFile?.Content);
                    PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping);

                    // We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false)
                    // because that information is already in node.Command
                    var rspFileSpec = ResponseFileSpecification.Builder()
                                      .ExplicitData(responseFileData)
                                      .RequiresArgument(false)
                                      .ExplicitPath((AbsolutePath)node.ResponseFile?.Path)
                                      .Build();

                    processBuilder.SetResponseFileSpecification(rspFileSpec);
                }
            }

            // TODO: Maybe a better description. Add ninja description or change command for input/outputs
            processBuilder.ToolDescription = StringId.Create(m_context.StringTable,
                                                             I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]"));


            processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable;
            processBuilder.EnableTempDirectory();

            // Working directory - the directory containing the ninja spec file
            // Ninja generators may express paths relative to this
            processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable));

            // Untrack directories
            UntrackFilesAndDirectories(processBuilder);

            // Allow some surviving child process
            AddRequiredSurvivingChildren(processBuilder);

            // Environment variables
            SetEnvironmentVariables(processBuilder, node);
            return(true);
        }
Exemple #2
0
        private bool TryConfigureProcessBuilder(
            ProcessBuilder processBuilder,
            PipConstructionHelper pipConstructionHelper,
            ProjectWithPredictions project,
            QualifierId qualifierId,
            out AbsolutePath outputResultCacheFile,
            out string failureDetail)
        {
            outputResultCacheFile = AbsolutePath.Invalid;
            if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
            {
                failureDetail = "Failed to construct tooldefinition";
                return(false);
            }

            // Working directory - the directory where the project file lives.
            processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));

            // We allow undeclared inputs to be read
            processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;

            // Run in a container if specified
            if (m_resolverSettings.RunInContainer)
            {
                processBuilder.Options |= Process.Options.NeedsToRunInContainer;
                processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
            }

            // By default the double write policy is to allow same content double writes.
            processBuilder.DoubleWritePolicy |= m_resolverSettings.DoubleWritePolicy ?? DoubleWritePolicy.AllowSameContentDoubleWrites;

            SetUntrackedFilesAndDirectories(processBuilder);

            // Add the log directory and its corresponding files
            var          qualifier    = m_context.QualifierTable.GetQualifier(qualifierId);
            AbsolutePath logDirectory = GetLogDirectory(project, qualifier);

            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);

            if (m_resolverSettings.EnableBinLogTracing == true)
            {
                processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
            }

            // Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
            // it produces an output cache file. This file is placed on the (unique) object directory for this project
            if (m_resolverSettings.UseLegacyProjectIsolation != true)
            {
                var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
                outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
                processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
            }

            // Path to the project
            processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
            // Response file with the rest of the arguments
            var rspFileSpec = ResponseFileSpecification.Builder()
                              .AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
                              .ForceCreation(true)
                              .Prefix("@")
                              .Build();

            processBuilder.SetResponseFileSpecification(rspFileSpec);

            if (!TryAddMsBuildArguments(project, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
            {
                return(false);
            }

            // Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
            // It'll cause full cache misses if we try to hash it as an input, however, so exclude.
            processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));

            // GlobalUnsafePassthroughEnvironmentVariables
            processBuilder.SetGlobalPassthroughEnvironmentVariable(m_frontEndHost.Configuration.FrontEnd.GlobalUnsafePassthroughEnvironmentVariables, m_context.StringTable);

            // mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
            // However this process will live beyond the build.cmd or msbuild.exe call.
            // Allow the pip job object to clean the process without complaint.
            //
            // vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
            // Just let it be killed.
            // TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
            //
            // conhost.exe: This process needs a little bit more time to finish after the main process, but killing it right away
            // is inconsequential.
            //
            // All child processes: Don't wait to kill the processes.
            // CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
            processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
                PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
                PathAtom.Create(m_context.StringTable, "vctip.exe"),
                PathAtom.Create(m_context.StringTable, "conhost.exe"),
                PathAtom.Create(m_context.StringTable, "VBCSCompiler.exe"));

            // There are some cases (e.g. a 64-bit MSBuild launched as a child process from a 32-bit MSBuild instance) where
            // processes need a little bit more time to finish. Increasing the timeout does not affect job objects where no child
            // processes survive, or job object where the only surviving processes are the ones explicitly allowed to survive (which
            // are killed immediately). So overall, this non-zero timeout will only make some pips that would have failed to take a little
            // bit longer (and hopefully succeed)
            processBuilder.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(500);

            SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);

            failureDetail = string.Empty;
            return(true);
        }
Exemple #3
0
        private bool TryConfigureProcessBuilder(
            ProcessBuilder processBuilder,
            PipConstructionHelper pipConstructionHelper,
            ProjectWithPredictions project,
            QualifierId qualifierId,
            out AbsolutePath outputResultCacheFile,
            out string failureDetail)
        {
            outputResultCacheFile = AbsolutePath.Invalid;
            if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
            {
                failureDetail = "Failed to construct tooldefinition";
                return(false);
            }

            // Working directory - the directory where the project file lives.
            processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));

            // We allow undeclared inputs to be read
            processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;

            // Run in a container if specified
            if (m_resolverSettings.RunInContainer)
            {
                processBuilder.Options |= Process.Options.NeedsToRunInContainer;
                processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
            }

            // Until we can deal with double writes in a better way, this unsafe option allows the build to progress and
            // prints warnings
            processBuilder.DoubleWritePolicy |= DoubleWritePolicy.UnsafeFirstDoubleWriteWins;

            SetUntrackedFilesAndDirectories(processBuilder);

            // Add the log directory and its corresponding files
            var          qualifier    = m_context.QualifierTable.GetQualifier(qualifierId);
            AbsolutePath logDirectory = GetLogDirectory(project, qualifier);

            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
            processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);

            if (m_resolverSettings.EnableBinLogTracing == true)
            {
                processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
            }

            // Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
            // it produces an output cache file. This file is placed on the (unique) object directory for this project
            if (m_resolverSettings.UseLegacyProjectIsolation != true)
            {
                var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
                outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
                processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
            }

            // Path to the project
            processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
            // Response file with the rest of the arguments
            var rspFileSpec = ResponseFileSpecification.Builder()
                              .AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
                              .ForceCreation(true)
                              .Prefix("@")
                              .Build();

            processBuilder.SetResponseFileSpecification(rspFileSpec);

            if (!TryAddMsBuildArguments(project, qualifier, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
            {
                return(false);
            }

            // Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
            // It'll cause full cache misses if we try to hash it as an input, however, so exclude.
            processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));

            // mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
            // However this process will live beyond the build.cmd or msbuild.exe call.
            // Allow the pip job object to clean the process without complaint.
            //
            // vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
            // Just let it be killed.
            // TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
            //
            // conhost.exe: This process needs a little bit more time to finish after the main process. We shouldn't be allowing
            // this one to survive, we just need the timeout to be slightly more than zero. This will also be beneficial to other
            // arbitrary processeses that need a little bit more time. But, apparently, setting a timeout has a perf impact that is
            // being investigated. TODO: revisit this once this is fixed.
            //
            // All child processes: Don't wait to kill the processes.
            // CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
            processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
                PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
                PathAtom.Create(m_context.StringTable, "vctip.exe"),
                PathAtom.Create(m_context.StringTable, "conhost.exe"));

            processBuilder.NestedProcessTerminationTimeout = TimeSpan.Zero;

            SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);

            failureDetail = string.Empty;
            return(true);
        }