Beispiel #1
0
        public override bool ExecuteActions(List <Action> ActionsToExecute)
        {
            bool XGEResult = true;

            // Batch up XGE execution by actions with the same output event handler.
            List <Action> ActionBatch = new List <Action>();

            ActionBatch.Add(ActionsToExecute[0]);
            for (int ActionIndex = 1; ActionIndex < ActionsToExecute.Count && XGEResult; ++ActionIndex)
            {
                Action CurrentAction = ActionsToExecute[ActionIndex];
                if (CurrentAction.OutputEventHandler == ActionBatch[0].OutputEventHandler)
                {
                    ActionBatch.Add(CurrentAction);
                }
                else
                {
                    XGEResult = XGE.ExecuteActionBatch(ActionBatch);
                    ActionBatch.Clear();
                    ActionBatch.Add(CurrentAction);
                }
            }
            if (ActionBatch.Count > 0 && XGEResult)
            {
                XGEResult = XGE.ExecuteActionBatch(ActionBatch);
                ActionBatch.Clear();
            }

            return(XGEResult);
        }
Beispiel #2
0
        public static ExecutionResult ExecuteActions(List <Action> Actions)
        {
            ExecutionResult XGEResult = ExecutionResult.TasksSucceeded;

            if (Actions.Count > 0)
            {
                // Write the actions to execute to a XGE task file.
                string XGETaskFilePath = Path.Combine(BuildConfiguration.BaseIntermediatePath, "XGETasks.xml");
                WriteTaskFile(Actions, XGETaskFilePath, ProgressWriter.bWriteMarkup);

                if (BuildConfiguration.bXGEExport)
                {
                    SaveXGEFile(XGETaskFilePath);
                }
                else
                {
                    // Try to execute the XGE tasks, and if XGE is available, skip the local execution fallback.
                    if (Telemetry.IsAvailable())
                    {
                        try
                        {
                            const string BuilderKey = "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Xoreax\\IncrediBuild\\Builder";

                            string CPUUtilization = Registry.GetValue(BuilderKey, "ForceCPUCount", "").ToString();
                            string AvoidTaskExecutionOnLocalMachine     = Registry.GetValue(BuilderKey, "AvoidLocalExec", "").ToString();
                            string RestartRemoteProcessesOnLocalMachine = Registry.GetValue(BuilderKey, "AllowDoubleTargets", "").ToString();
                            string LimitMaxNumberOfCores         = Registry.GetValue(BuilderKey, "MaxHelpers", "").ToString();
                            string WriteOutputToDiskInBackground = Registry.GetValue(BuilderKey, "LazyOutputWriter_Beta", "").ToString();
                            string MaxConcurrentPDBs             = Registry.GetValue(BuilderKey, "MaxConcurrentPDBs", "").ToString();
                            string EnabledAsHelper = Registry.GetValue(BuilderKey, "LastEnabled", "").ToString();

                            Telemetry.SendEvent("XGESettings.2",
                                                "CPUUtilization", CPUUtilization,
                                                "AvoidTaskExecutionOnLocalMachine", AvoidTaskExecutionOnLocalMachine,
                                                "RestartRemoteProcessesOnLocalMachine", RestartRemoteProcessesOnLocalMachine,
                                                "LimitMaxNumberOfCores", LimitMaxNumberOfCores,
                                                "WriteOutputToDiskInBackground", WriteOutputToDiskInBackground,
                                                "MaxConcurrentPDBs", MaxConcurrentPDBs,
                                                "EnabledAsHelper", EnabledAsHelper);
                        }
                        catch
                        {
                        }
                    }
                    XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (Sender, Args) =>
                    {
                        if (Actions[0].OutputEventHandler != null)
                        {
                            Actions[0].OutputEventHandler(Sender, Args);
                        }
                        else
                        {
                            Console.WriteLine(Args.Data);
                        }
                    });
                }
            }
            return(XGEResult);
        }
Beispiel #3
0
        /// <summary>
        /// Executes a list of actions.
        /// </summary>
        public static void ExecuteActions(BuildConfiguration BuildConfiguration, List <Action> ActionsToExecute, List <TargetDescriptor> TargetDescriptors = null)
        {
            Log.TraceInformation("Execute actions num:" + ActionsToExecute.Count + " target descriptors num:" + TargetDescriptors.Count);

            if (ActionsToExecute.Count == 0)
            {
                Log.TraceInformation("Target is up to date");
            }
            else
            {
                // Figure out which executor to use
                ActionExecutor Executor;
                if (BuildConfiguration.bAllowHybridExecutor && HybridExecutor.IsAvailable())
                {
                    Executor = new HybridExecutor();
                }
                else if (BuildConfiguration.bAllowXGE && XGE.IsAvailable())
                {
                    Executor = new XGE();
                }
                else if (BuildConfiguration.bAllowDistcc)
                {
                    Executor = new Distcc();
                }
                else if (BuildConfiguration.bAllowSNDBS && SNDBS.IsAvailable())
                {
                    Executor = new SNDBS();
                }
                else if (BuildConfiguration.bAllowParallelExecutor && ParallelExecutor.IsAvailable())
                {
                    Executor = new ParallelExecutor();
                }
                else
                {
                    Executor = new LocalExecutor();
                }


                //if (BuildConfiguration.bAllowFastBuild && FastBuild_v1.IsAvailable())
                //{
                //	Executor = new FastBuild_v1(Executor, TargetDescriptors);
                //}
                if (BuildConfiguration.bAllowFastBuild && FastBuild_v2.IsAvailable())
                {
                    Executor = new FastBuild_v2(Executor, TargetDescriptors);
                }

                // Execute the build
                Stopwatch Timer = Stopwatch.StartNew();
                if (!Executor.ExecuteActions(ActionsToExecute, BuildConfiguration.bLogDetailedActionStats))
                {
                    throw new CompilationResultException(CompilationResult.OtherCompilationError);
                }
                Log.TraceInformation("Total time in {0} executor: {1:0.00} seconds", Executor.Name, Timer.Elapsed.TotalSeconds);

                // Reset the file info for all the produced items
                foreach (Action BuildAction in ActionsToExecute)
                {
                    foreach (FileItem ProducedItem in BuildAction.ProducedItems)
                    {
                        ProducedItem.ResetCachedInfo();
                    }
                }

                // Verify the link outputs were created (seems to happen with Win64 compiles)
                foreach (Action BuildAction in ActionsToExecute)
                {
                    if (BuildAction.ActionType == ActionType.Link)
                    {
                        foreach (FileItem Item in BuildAction.ProducedItems)
                        {
                            if (!Item.Exists)
                            {
                                throw new BuildException("Failed to produce item: {0}", Item.AbsolutePath);
                            }
                        }
                    }
                }
            }
        }
Beispiel #4
0
        /// <summary>
        /// Build a list of targets with a given set of makefiles.
        /// </summary>
        /// <param name="Makefiles">Makefiles created with CreateMakefiles</param>
        /// <param name="TargetDescriptors">Target descriptors</param>
        /// <param name="BuildConfiguration">Current build configuration</param>
        /// <param name="WorkingSet">The source file working set</param>
        /// <param name="Options">Additional options for the build</param>
        /// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
        /// <returns>Result from the compilation</returns>
        static void Build(TargetMakefile[] Makefiles, List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
        {
            // Export the actions for each target
            for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
            {
                TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
                foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
                {
                    Log.TraceInformation("Writing actions to {0}", WriteActionFile);
                    ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
                }
            }

            // Execute the build
            if ((Options & BuildOptions.SkipBuild) == 0)
            {
                // Make sure that none of the actions conflict with any other (producing output files differently, etc...)
                ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));

                // Check we don't exceed the nominal max path length
                using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
                {
                    ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
                }

                // Clean up any previous hot reload runs, and reapply the current state if it's already active
                for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
                {
                    HotReload.Setup(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], BuildConfiguration);
                }

                // Merge the action graphs together
                List <Action> MergedActions;
                if (TargetDescriptors.Count == 1)
                {
                    MergedActions = new List <Action>(Makefiles[0].Actions);
                }
                else
                {
                    MergedActions = MergeActionGraphs(TargetDescriptors, Makefiles);
                }

                // Gather all the prerequisite actions that are part of the targets
                HashSet <FileItem> MergedOutputItems = new HashSet <FileItem>();
                for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
                {
                    GatherOutputItems(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], MergedOutputItems);
                }

                // Link all the actions together
                ActionGraph.Link(MergedActions);

                // Get all the actions that are prerequisites for these targets. This forms the list of actions that we want executed.
                List <Action> PrerequisiteActions = ActionGraph.GatherPrerequisiteActions(MergedActions, MergedOutputItems);

                // Create the action history
                ActionHistory History = new ActionHistory();
                for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
                {
                    using (Timeline.ScopeEvent("Reading action history"))
                    {
                        TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
                        if (TargetDescriptor.ProjectFile != null)
                        {
                            History.Mount(TargetDescriptor.ProjectFile.Directory);
                        }
                    }
                }

                // Figure out which actions need to be built
                Dictionary <Action, bool> ActionToOutdatedFlag = new Dictionary <Action, bool>();
                for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
                {
                    TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];

                    // Create the dependencies cache
                    CppDependencyCache CppDependencies;
                    using (Timeline.ScopeEvent("Reading dependency cache"))
                    {
                        CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefiles[TargetIdx].TargetType, TargetDescriptor.Architecture);
                    }

                    // Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
                    if (TargetDescriptor.SpecificFilesToCompile.Count == 0)
                    {
                        ActionGraph.GatherAllOutdatedActions(PrerequisiteActions, History, ActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries);
                    }
                    else
                    {
                        foreach (FileReference SpecificFile in TargetDescriptor.SpecificFilesToCompile)
                        {
                            foreach (Action PrerequisiteAction in PrerequisiteActions.Where(x => x.PrerequisiteItems.Any(y => y.Location == SpecificFile)))
                            {
                                ActionToOutdatedFlag[PrerequisiteAction] = true;
                            }
                        }
                    }
                }

                // Link the action graph again to sort it
                List <Action> MergedActionsToExecute = ActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
                ActionGraph.Link(MergedActionsToExecute);

                // Allow hot reload to override the actions
                int HotReloadTargetIdx = -1;
                for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
                {
                    if (TargetDescriptors[Idx].HotReloadMode != HotReloadMode.Disabled)
                    {
                        if (HotReloadTargetIdx != -1)
                        {
                            throw new BuildException("Unable to perform hot reload with multiple targets.");
                        }
                        else
                        {
                            MergedActionsToExecute = HotReload.PatchActionsForTarget(BuildConfiguration, TargetDescriptors[Idx], Makefiles[Idx], PrerequisiteActions, MergedActionsToExecute);
                        }
                        HotReloadTargetIdx = Idx;
                    }
                }

                // Make sure we're not modifying any engine files
                if ((Options & BuildOptions.NoEngineChanges) != 0)
                {
                    List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
                    if (EngineChanges.Count > 0)
                    {
                        StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
                        foreach (FileItem EngineChange in EngineChanges)
                        {
                            Result.AppendFormat("\n{0}", EngineChange.FullName);
                        }
                        Result.Append("\n\nPlease rebuild from an IDE instead.");
                        Log.TraceError("{0}", Result.ToString());
                        throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
                    }
                }

                // Make sure the appropriate executor is selected
                foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
                {
                    UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
                    BuildConfiguration.bAllowXGE    &= BuildPlatform.CanUseXGE();
                    BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
                    BuildConfiguration.bAllowSNDBS  &= BuildPlatform.CanUseSNDBS();
                }

                // Delete produced items that are outdated.
                ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);

                // Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
                // items created during the build don't have the wrong command line.
                History.Save();

                // Create directories for the outdated produced items.
                ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);

                // Execute the actions
                if ((Options & BuildOptions.XGEExport) != 0)
                {
                    OutputToolchainInfo(TargetDescriptors, Makefiles);

                    // Just export to an XML file
                    using (Timeline.ScopeEvent("XGE.ExportActions()"))
                    {
                        XGE.ExportActions(MergedActionsToExecute);
                    }
                }
                else if (WriteOutdatedActionsFile != null)
                {
                    OutputToolchainInfo(TargetDescriptors, Makefiles);

                    // Write actions to an output file
                    using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
                    {
                        ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
                    }
                }
                else
                {
                    // Execute the actions
                    if (MergedActionsToExecute.Count == 0)
                    {
                        if (TargetDescriptors.Any(x => !x.bQuiet))
                        {
                            Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
                        }
                    }
                    else
                    {
                        if (TargetDescriptors.Any(x => !x.bQuiet))
                        {
                            Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
                        }

                        OutputToolchainInfo(TargetDescriptors, Makefiles);

                        using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
                        {
                            ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
                        }
                    }

                    // Run the deployment steps
                    foreach (TargetMakefile Makefile in Makefiles)
                    {
                        if (Makefile.bDeployAfterCompile)
                        {
                            TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
                            Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);

                            UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
                        }
                    }
                }
            }
        }
Beispiel #5
0
        /// <summary>
        /// Build a list of targets
        /// </summary>
        /// <param name="TargetDescriptors">Target descriptors</param>
        /// <param name="BuildConfiguration">Current build configuration</param>
        /// <param name="WorkingSet">The source file working set</param>
        /// <param name="Options">Additional options for the build</param>
        /// <param name="LiveCodingManifest">Path to write the live coding manifest to</param>
        /// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
        /// <returns>Result from the compilation</returns>
        public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference LiveCodingManifest, FileReference WriteOutdatedActionsFile)
        {
            // Create a makefile for each target
            TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count];
            for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
            {
                Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet);
            }

            // Output the Live Coding manifest
            if (LiveCodingManifest != null)
            {
                List <Action> AllActions = Makefiles.SelectMany(x => x.Actions).ToList();
                HotReload.WriteLiveCodingManifest(LiveCodingManifest, AllActions);
            }

            // Export the actions for each target
            for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
            {
                TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
                foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
                {
                    Log.TraceInformation("Writing actions to {0}", WriteActionFile);
                    ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
                }
            }

            // Execute the build
            if ((Options & BuildOptions.SkipBuild) == 0)
            {
                // Make sure that none of the actions conflict with any other (producing output files differently, etc...)
                ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));

                // Find all the actions to be executed
                HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count];
                for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
                {
                    ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]);
                }

                // If there are multiple targets being built, merge the actions together
                List <Action> MergedActionsToExecute;
                if (TargetDescriptors.Count == 1)
                {
                    MergedActionsToExecute = new List <Action>(ActionsToExecute[0]);
                }
                else
                {
                    MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute);
                }

                // Link all the actions together
                ActionGraph.Link(MergedActionsToExecute);

                // Make sure the appropriate executor is selected
                foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
                {
                    UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
                    BuildConfiguration.bAllowXGE    &= BuildPlatform.CanUseXGE();
                    BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
                    BuildConfiguration.bAllowSNDBS  &= BuildPlatform.CanUseSNDBS();
                }

                // Delete produced items that are outdated.
                ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);

                // Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
                // items created during the build don't have the wrong command line.
                ActionHistory.SaveAll();

                // Create directories for the outdated produced items.
                ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);

                // Execute the actions
                if ((Options & BuildOptions.XGEExport) != 0)
                {
                    // Just export to an XML file
                    using (Timeline.ScopeEvent("XGE.ExportActions()"))
                    {
                        XGE.ExportActions(MergedActionsToExecute);
                    }
                }
                else if (WriteOutdatedActionsFile != null)
                {
                    // Write actions to an output file
                    using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
                    {
                        ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
                    }
                }
                else
                {
                    // Execute the actions
                    if (MergedActionsToExecute.Count == 0)
                    {
                        if ((Options & BuildOptions.Quiet) == 0)
                        {
                            Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
                        }
                    }
                    else
                    {
                        if ((Options & BuildOptions.Quiet) != 0)
                        {
                            Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
                        }

                        OutputToolchainInfo(TargetDescriptors, Makefiles);

                        using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
                        {
                            ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
                        }
                    }

                    // Run the deployment steps
                    foreach (TargetMakefile Makefile in Makefiles)
                    {
                        if (Makefile.bDeployAfterCompile)
                        {
                            TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
                            Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
                            UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
                        }
                    }
                }
            }
        }
Beispiel #6
0
        public static ExecutionResult ExecuteActions(List <Action> Actions)
        {
            ExecutionResult XGEResult = ExecutionResult.TasksSucceeded;

            if (Actions.Count > 0)
            {
                // Write the actions to execute to a XGE task file.
                string XGETaskFilePath = Path.Combine(BuildConfiguration.BaseIntermediatePath, "XGETasks.xml");
                WriteTaskFile(Actions, XGETaskFilePath, ProgressWriter.bWriteMarkup);

                if (BuildConfiguration.bXGEExport)
                {
                    SaveXGEFile(XGETaskFilePath);
                }
                else
                {
                    // Try to execute the XGE tasks, and if XGE is available, skip the local execution fallback.
                    if (Telemetry.IsAvailable())
                    {
                        try
                        {
                            const string BuilderKey = "HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Xoreax\\IncrediBuild\\Builder";

                            string CPUUtilization = Registry.GetValue(BuilderKey, "ForceCPUCount", "").ToString();
                            string AvoidTaskExecutionOnLocalMachine     = Registry.GetValue(BuilderKey, "AvoidLocalExec", "").ToString();
                            string RestartRemoteProcessesOnLocalMachine = Registry.GetValue(BuilderKey, "AllowDoubleTargets", "").ToString();
                            string LimitMaxNumberOfCores         = Registry.GetValue(BuilderKey, "MaxHelpers", "").ToString();
                            string WriteOutputToDiskInBackground = Registry.GetValue(BuilderKey, "LazyOutputWriter_Beta", "").ToString();
                            string MaxConcurrentPDBs             = Registry.GetValue(BuilderKey, "MaxConcurrentPDBs", "").ToString();
                            string EnabledAsHelper = Registry.GetValue(BuilderKey, "LastEnabled", "").ToString();

                            Telemetry.SendEvent("XGESettings.2",
                                                "CPUUtilization", CPUUtilization,
                                                "AvoidTaskExecutionOnLocalMachine", AvoidTaskExecutionOnLocalMachine,
                                                "RestartRemoteProcessesOnLocalMachine", RestartRemoteProcessesOnLocalMachine,
                                                "LimitMaxNumberOfCores", LimitMaxNumberOfCores,
                                                "WriteOutputToDiskInBackground", WriteOutputToDiskInBackground,
                                                "MaxConcurrentPDBs", MaxConcurrentPDBs,
                                                "EnabledAsHelper", EnabledAsHelper);
                        }
                        catch
                        {
                        }

                        // Add a custom output handler to determine the build duration for each task and map it back to the action that generated it.
                        XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (sender, args) =>
                        {
                            // sometimes the args comes in as null
                            var match = XGEDurationRegex.Match(args.Data ?? "");

                            /*
                             *                                                              // This is considered fragile and risky parsing of undocumented XGE output, so protect this code from taking down UBT
                             *                                                              try
                             *                                                              {
                             *                                                                      // Use LINQ to evaluate the terms so we don't end up with a
                             *                                                                      // huge nested conditional expression just to verify the input.
                             *                                                                      foreach (var PCHEvent in
                             *                                                                              // first, convert the input line into an enumerable
                             *                                                                              from RegexMatch in new[] { match }
                             *                                                                              where RegexMatch.Success
                             *                                                                              let Filename = RegexMatch.Groups["Filename"].Value
                             *                                                                              // find the mapped action. (On PS4 at least, XGE appears to use the full filenane, so we need to strip it back down.
                             *                                                                              let Action = Actions.Find(a => a.StatusDescription == Path.GetFileName(Filename) && a.ActionType == ActionType.Compile)
                             *                                                                              // We should ALWAYS find the action, so maybe this should throw if we can't
                             *                                                                              where Action != null
                             *                                                                              // see if the mapped action produces a PCH file
                             *                                                                              // (there is currently no compile environment left around by which to tell absolutely, so we infer by the extension. Ugh).
                             *                                                                              let ActionProducedPCH = Action.ProducedItems.Find(fileItem => new[] { ".PCH", ".GCH" }.Contains(Path.GetExtension(fileItem.AbsolutePath).ToUpperInvariant()))
                             *                                                                              where ActionProducedPCH != null
                             *                                                                              // we found a valid PCH action and output file, so parse the duration and send an event.
                             *                                                                              let durationMatchStr = RegexMatch.Groups["Duration"].Value
                             *                                                                              // if there's no hour designator, add one so .NET can parse the time.
                             *                                                                              let durationStr = durationMatchStr.Count(c => c == ':') == 1 ? "0:" + durationMatchStr : durationMatchStr
                             *                                                                              let duration = TimeSpan.Parse(durationStr)
                             *                                                                              select new
                             *                                                                              {
                             *                                                                                      // actually use the filename here because it is coerced to be PCH.MODULE.HEADER.cpp.
                             *                                                                                      // This allows us an easy way to determine shared status (or source module) and the real header source.
                             *                                                                                      FileName = Filename,//Path.GetFileName(ActionProducedPCH.AbsolutePath),
                             *                                                                                      // Get the length from the OS as the FileItem.Length is really for when the file is used as an input,
                             *                                                                                      // so the stored length is absent for a new for or out of date at best.
                             *                                                                                      GeneratedFileLength = new FileInfo(ActionProducedPCH.AbsolutePath).Length,
                             *                                                                                      GeneratedFileDuration = duration,
                             *                                                                              })
                             *                                                                      {
                             *                                                                              // If we had a valid match for a PCH item, send an event.
                             *                                                                              Telemetry.SendEvent("PCHTime.2",
                             *                                                                                      "ExecutorType", "XGE",
                             *                                                                                      "Filename", PCHEvent.FileName,
                             *                                                                                      "FileSize", PCHEvent.GeneratedFileLength.ToString(),
                             *                                                                                      "Duration", PCHEvent.GeneratedFileDuration.TotalSeconds.ToString("0.00"));
                             *                                                                      }
                             *                                                              }
                             *                                                              catch (Exception ex)
                             *                                                              {
                             *                                                                      // report that something went wrong so we can diagnose.
                             *                                                                      Telemetry.SendEvent("PCHTimeError.2",
                             *                                                                              "OutputLine", args.Data,
                             *                                                                              "Exception", ex.ToString());
                             *                                                              }
                             */

                            // XGE outputs the duration info in a format that makes VC think it's an error file/line notation if the full filename is used.
                            // So munge it a bit so we don't confuse VC.
                            var updatedData = match.Success ? args.Data.Replace('(', '[').Replace(')', ']') : args.Data;

                            // forward the output on to the normal handler or the console like normal
                            if (Actions[0].OutputEventHandler != null)
                            {
                                DataReceivedEventArgs EventArgs = ConstructDataReceivedEventArgs(updatedData);
                                Actions[0].OutputEventHandler(sender, EventArgs);
                            }
                            else
                            {
                                Console.WriteLine(updatedData);
                            }
                        });
                    }
                    else
                    {
                        XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (Sender, Args) =>
                        {
                            if (Actions[0].OutputEventHandler != null)
                            {
                                Actions[0].OutputEventHandler(Sender, Args);
                            }
                            else
                            {
                                Console.WriteLine(Args.Data);
                            }
                        });
                    }
                }
            }
            return(XGEResult);
        }
Beispiel #7
0
 /// <summary>
 /// Gets the path for the XGE console executable
 /// </summary>
 /// <param name="OutXgConsoleExe">On success, receives the path to the XGE console executable</param>
 /// <returns>True if the path was found, false otherwise</returns>
 public static bool TryGetXgConsoleExecutable(out string OutXgConsoleExe)
 {
     return(XGE.TryGetXgConsoleExecutable(out OutXgConsoleExe));
 }
Beispiel #8
0
        public static ExecutionResult ExecuteActions(List <Action> Actions)
        {
            ExecutionResult XGEResult = ExecutionResult.TasksSucceeded;

            if (Actions.Count > 0)
            {
                // Write the actions to execute to a XGE task file.
                string XGETaskFilePath = Path.Combine(BuildConfiguration.BaseIntermediatePath, "XGETasks.xml");
                WriteTaskFile(Actions, XGETaskFilePath, ProgressWriter.bWriteMarkup);

                if (BuildConfiguration.bXGEExport)
                {
                    SaveXGEFile(XGETaskFilePath);
                }
                else
                {
                    // Try to execute the XGE tasks, and if XGE is available, skip the local execution fallback.
                    if (Telemetry.IsAvailable())
                    {
                        // Add a custom output handler to determine the build duration for each task and map it back to the action that generated it.
                        XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (sender, args) =>
                        {
                            // sometimes the args comes in as null
                            var match = XGEDurationRegex.Match(args.Data ?? "");

                            // This is considered fragile and risky parsing of undocumented XGE output, so protect this code from taking down UBT
                            try
                            {
                                // Use LINQ to evaluate the terms so we don't end up with a
                                // huge nested conditional expression just to verify the input.
                                foreach (var PCHEvent in
                                         // first, convert the input line into an enumerable
                                         from RegexMatch in new[] { match }
                                         where RegexMatch.Success
                                         let Filename = RegexMatch.Groups["Filename"].Value
                                                        // find the mapped action. (On PS4 at least, XGE appears to use the full filenane, so we need to strip it back down.
                                                        let Action = Actions.Find(a => a.StatusDescription == Path.GetFileName(Filename) && a.ActionType == ActionType.Compile)
                                                        // We should ALWAYS find the action, so maybe this should throw if we can't
                                                                     where Action != null
                                                        // see if the mapped action produces a PCH file
                                                        // (there is currently no compile environment left around by which to tell absolutely, so we infer by the extension. Ugh).
                                                                     let ActionProducedPCH = Action.ProducedItems.Find(fileItem => new[] { ".PCH", ".GCH" }.Contains(Path.GetExtension(fileItem.AbsolutePath).ToUpperInvariant()))
                                                                                             where ActionProducedPCH != null
                                                                                             // we found a valid PCH action and output file, so parse the duration and send an event.
                                                                                             let durationMatchStr = RegexMatch.Groups["Duration"].Value
                                                                                                                    // if there's no hour designator, add one so .NET can parse the time.
                                                                                                                    let durationStr = durationMatchStr.Count(c => c == ':') == 1 ? "0:" + durationMatchStr : durationMatchStr
                                                                                                                                      let duration = TimeSpan.Parse(durationStr)
                                                                                                                                                     select new
                                {
                                    // actually use the filename here because it is coerced to be PCH.MODULE.HEADER.cpp.
                                    // This allows us an easy way to determine shared status (or source module) and the real header source.
                                    FileName = Filename,                                                    //Path.GetFileName(ActionProducedPCH.AbsolutePath),
                                    // Get the length from the OS as the FileItem.Length is really for when the file is used as an input,
                                    // so the stored length is absent for a new for or out of date at best.
                                    GeneratedFileLength = new FileInfo(ActionProducedPCH.AbsolutePath).Length,
                                    GeneratedFileDuration = duration,
                                })
                                {
                                    // If we had a valid match for a PCH item, send an event.
                                    Telemetry.SendEvent("PCHTime.2",
                                                        "ExecutorType", "XGE",
                                                        "Filename", PCHEvent.FileName,
                                                        "FileSize", PCHEvent.GeneratedFileLength.ToString(),
                                                        "Duration", PCHEvent.GeneratedFileDuration.TotalSeconds.ToString("0.00"));
                                }
                            }
                            catch (Exception ex)
                            {
                                // report that something went wrong so we can diagnose.
                                Telemetry.SendEvent("PCHTimeError.2",
                                                    "OutputLine", args.Data,
                                                    "Exception", ex.ToString());
                            }

                            // XGE outputs the duration info in a format that makes VC think it's an error file/line notation if the full filename is used.
                            // So munge it a bit so we don't confuse VC.
                            var updatedData = match.Success ? args.Data.Replace('(', '[').Replace(')', ']') : args.Data;

                            // forward the output on to the normal handler or the console like normal
                            if (Actions[0].OutputEventHandler != null)
                            {
                                DataReceivedEventArgs EventArgs = ConstructDataReceivedEventArgs(updatedData);
                                Actions[0].OutputEventHandler(sender, EventArgs);
                            }
                            else
                            {
                                Console.WriteLine(updatedData);
                            }
                        });
                    }
                    else
                    {
                        XGEResult = XGE.ExecuteTaskFileWithProgressMarkup(XGETaskFilePath, Actions.Count, (Sender, Args) =>
                        {
                            if (Actions[0].OutputEventHandler != null)
                            {
                                Actions[0].OutputEventHandler(Sender, Args);
                            }
                            else
                            {
                                Console.WriteLine(Args.Data);
                            }
                        });
                    }
                }
            }
            return(XGEResult);
        }
Beispiel #9
0
        /// <summary>
        /// Build a list of targets
        /// </summary>
        /// <param name="TargetDescriptors">Target descriptors</param>
        /// <param name="BuildConfiguration">Current build configuration</param>
        /// <param name="WorkingSet">The source file working set</param>
        /// <param name="Options">Additional options for the build</param>
        /// <param name="WriteOutdatedActionsFile">Files to write the list of outdated actions to (rather than building them)</param>
        /// <returns>Result from the compilation</returns>
        public static void Build(List <TargetDescriptor> TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference WriteOutdatedActionsFile)
        {
            // Create a makefile for each target
            TargetMakefile[] Makefiles = new TargetMakefile[TargetDescriptors.Count];
            for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
            {
                Makefiles[TargetIdx] = CreateMakefile(BuildConfiguration, TargetDescriptors[TargetIdx], WorkingSet);
            }

            // Export the actions for each target
            for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
            {
                TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
                foreach (FileReference WriteActionFile in TargetDescriptor.WriteActionFiles)
                {
                    Log.TraceInformation("Writing actions to {0}", WriteActionFile);
                    ActionGraph.ExportJson(Makefiles[TargetIdx].Actions, WriteActionFile);
                }
            }

            // Execute the build
            if ((Options & BuildOptions.SkipBuild) == 0)
            {
                // Make sure that none of the actions conflict with any other (producing output files differently, etc...)
                ActionGraph.CheckForConflicts(Makefiles.SelectMany(x => x.Actions));

                // Check we don't exceed the nominal max path length
                using (Timeline.ScopeEvent("ActionGraph.CheckPathLengths"))
                {
                    ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions));
                }

                // Find all the actions to be executed
                HashSet <Action>[] ActionsToExecute = new HashSet <Action> [TargetDescriptors.Count];
                for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
                {
                    ActionsToExecute[TargetIdx] = GetActionsForTarget(BuildConfiguration, TargetDescriptors[TargetIdx], Makefiles[TargetIdx]);
                }

                // If there are multiple targets being built, merge the actions together
                List <Action> MergedActionsToExecute;
                if (TargetDescriptors.Count == 1)
                {
                    MergedActionsToExecute = new List <Action>(ActionsToExecute[0]);
                }
                else
                {
                    MergedActionsToExecute = MergeActionGraphs(TargetDescriptors, ActionsToExecute);
                }

                // Link all the actions together
                ActionGraph.Link(MergedActionsToExecute);

                // Make sure we're not modifying any engine files
                if ((Options & BuildOptions.NoEngineChanges) != 0)
                {
                    List <FileItem> EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(UnrealBuildTool.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
                    if (EngineChanges.Count > 0)
                    {
                        StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
                        foreach (FileItem EngineChange in EngineChanges)
                        {
                            Result.AppendFormat("\n{0}", EngineChange.FullName);
                        }
                        Result.Append("\n\nPlease rebuild from an IDE instead.");
                        Log.TraceError("{0}", Result.ToString());
                        throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
                    }
                }

                // Make sure the appropriate executor is selected
                foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
                {
                    UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
                    BuildConfiguration.bAllowXGE    &= BuildPlatform.CanUseXGE();
                    BuildConfiguration.bAllowDistcc &= BuildPlatform.CanUseDistcc();
                    BuildConfiguration.bAllowSNDBS  &= BuildPlatform.CanUseSNDBS();
                }

                // Delete produced items that are outdated.
                ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute);

                // Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
                // items created during the build don't have the wrong command line.
                ActionHistory.SaveAll();

                // Create directories for the outdated produced items.
                ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);

                // Execute the actions
                if ((Options & BuildOptions.XGEExport) != 0)
                {
                    OutputToolchainInfo(TargetDescriptors, Makefiles);

                    // Just export to an XML file
                    using (Timeline.ScopeEvent("XGE.ExportActions()"))
                    {
                        XGE.ExportActions(MergedActionsToExecute);
                    }
                }
                else if (WriteOutdatedActionsFile != null)
                {
                    OutputToolchainInfo(TargetDescriptors, Makefiles);

                    // Write actions to an output file
                    using (Timeline.ScopeEvent("ActionGraph.WriteActions"))
                    {
                        ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
                    }
                }
                else
                {
                    // Execute the actions
                    if (MergedActionsToExecute.Count == 0)
                    {
                        if (TargetDescriptors.Any(x => !x.bQuiet))
                        {
                            Log.TraceInformation((TargetDescriptors.Count == 1)? "Target is up to date" : "Targets are up to date");
                        }
                    }
                    else
                    {
                        if (TargetDescriptors.Any(x => !x.bQuiet))
                        {
                            Log.TraceInformation("Building {0}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
                        }

                        OutputToolchainInfo(TargetDescriptors, Makefiles);

                        using (Timeline.ScopeEvent("ActionGraph.ExecuteActions()"))
                        {
                            ActionGraph.ExecuteActions(BuildConfiguration, MergedActionsToExecute);
                        }
                    }

                    // Run the deployment steps
                    foreach (TargetMakefile Makefile in Makefiles)
                    {
                        if (Makefile.bDeployAfterCompile)
                        {
                            TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
                            Log.TraceInformation("Deploying {0} {1} {2}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);

                            UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
                        }
                    }
                }
            }
        }
 /// <summary>
 /// Check whether the given platform supports XGE
 /// </summary>
 /// <param name="Platform">Platform to check</param>
 /// <returns>True if the platform supports XGE</returns>
 public static bool CanUseXGE(UnrealTargetPlatform Platform)
 {
     return(UEBuildPlatform.IsPlatformAvailable(Platform) && UEBuildPlatform.GetBuildPlatform(Platform).CanUseXGE() && XGE.IsAvailable());
 }
 /// <summary>
 /// Tests whether this executor can be used
 /// </summary>
 /// <returns>True if the executor may be used</returns>
 public static bool IsAvailable()
 {
     return(XGE.IsAvailable() && ParallelExecutor.IsAvailable());
 }