public override void AddNodes(GUBP bp, UnrealTargetPlatform HostPlatform) { if(!bp.BranchOptions.bNoInstalledEngine) { // Find all the target platforms for this host platform. List<UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(bp, HostPlatform); // Remove any platforms that aren't available on this machine TargetPlatforms.RemoveAll(x => !bp.ActivePlatforms.Contains(x)); // Get the temp directory for stripped files for this host string StrippedDir = Path.GetFullPath(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Rocket", HostPlatform.ToString())); // Strip the host platform if (StripRocketNode.IsRequiredForPlatform(HostPlatform)) { bp.AddNode(new StripRocketToolsNode(HostPlatform, StrippedDir)); bp.AddNode(new StripRocketEditorNode(HostPlatform, StrippedDir)); } // Strip all the target platforms that are built on this host foreach (UnrealTargetPlatform TargetPlatform in TargetPlatforms) { if (GetSourceHostPlatform(bp, HostPlatform, TargetPlatform) == HostPlatform && StripRocketNode.IsRequiredForPlatform(TargetPlatform)) { bp.AddNode(new StripRocketMonolithicsNode(bp, HostPlatform, TargetPlatform, StrippedDir)); } } // Build the DDC bp.AddNode(new BuildDerivedDataCacheNode(HostPlatform, GetCookPlatforms(HostPlatform, TargetPlatforms), CurrentFeaturePacks)); // Generate a list of files that needs to be copied for each target platform bp.AddNode(new FilterRocketNode(bp, HostPlatform, TargetPlatforms, CurrentFeaturePacks, CurrentTemplates)); // Copy the install to the output directory string LocalOutputDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "Rocket", CommandUtils.GetGenericPlatformName(HostPlatform)); bp.AddNode(new GatherRocketNode(HostPlatform, TargetPlatforms, LocalOutputDir)); // Add the aggregate node for the entire install GUBP.GUBPNode PromotableNode = bp.FindNode(GUBP.SharedAggregatePromotableNode.StaticGetFullName()); PromotableNode.AddDependency(FilterRocketNode.StaticGetFullName(HostPlatform)); PromotableNode.AddDependency(BuildDerivedDataCacheNode.StaticGetFullName(HostPlatform)); // Add a node for GitHub promotions if(HostPlatform == UnrealTargetPlatform.Win64) { string GitConfigRelativePath = "Engine/Build/Git/UnrealBot.ini"; if(CommandUtils.FileExists(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, GitConfigRelativePath))) { bp.AddNode(new BuildGitPromotable(bp, HostPlatform, GitConfigRelativePath)); PromotableNode.AddDependency(BuildGitPromotable.StaticGetFullName(HostPlatform)); } } // Get the output directory for the build zips string PublishedEngineDir; if(ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP()) { PublishedEngineDir = CommandUtils.CombinePaths(CommandUtils.RootSharedTempStorageDirectory(), "Rocket", "Automated", GetBuildLabel(), CommandUtils.GetGenericPlatformName(HostPlatform)); } else { PublishedEngineDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketPublish", CommandUtils.GetGenericPlatformName(HostPlatform)); } // Publish the install to the network bp.AddNode(new PublishRocketNode(HostPlatform, LocalOutputDir, PublishedEngineDir)); bp.AddNode(new PublishRocketSymbolsNode(bp, HostPlatform, TargetPlatforms, PublishedEngineDir + "Symbols")); // Add a dependency on this being published as part of the shared promotable being labeled GUBP.SharedLabelPromotableSuccessNode LabelPromotableNode = (GUBP.SharedLabelPromotableSuccessNode)bp.FindNode(GUBP.SharedLabelPromotableSuccessNode.StaticGetFullName()); LabelPromotableNode.AddDependency(PublishRocketNode.StaticGetFullName(HostPlatform)); LabelPromotableNode.AddDependency(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform)); // Add dependencies on a promotable to do these steps too GUBP.WaitForSharedPromotionUserInput WaitForPromotionNode = (GUBP.WaitForSharedPromotionUserInput)bp.FindNode(GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(true)); WaitForPromotionNode.AddDependency(PublishRocketNode.StaticGetFullName(HostPlatform)); WaitForPromotionNode.AddDependency(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform)); // Push everything behind the promotion triggers if we're doing things on the build machines if(ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP() || bp.ParseParam("WithRocketPromotable")) { string WaitForTrigger = GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(false); GatherRocketNode GatherRocket = (GatherRocketNode)bp.FindNode(GatherRocketNode.StaticGetFullName(HostPlatform)); GatherRocket.AddDependency(WaitForTrigger); PublishRocketSymbolsNode PublishRocketSymbols = (PublishRocketSymbolsNode)bp.FindNode(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform)); PublishRocketSymbols.AddDependency(WaitForTrigger); } } }
void PrintNodes(GUBP bp, List<BuildNode> Nodes, IEnumerable<AggregateNode> Aggregates, List<TriggerNode> UnfinishedTriggers, int TimeQuantum) { AggregateNode[] MatchingAggregates = Aggregates.Where(x => x.Dependencies.All(y => Nodes.Contains(y))).ToArray(); if (MatchingAggregates.Length > 0) { Log("*********** Aggregates"); foreach (AggregateNode Aggregate in MatchingAggregates.OrderBy(x => x.Name)) { StringBuilder Note = new StringBuilder(" " + Aggregate.Name); if (Aggregate.Node.IsPromotableAggregate()) { Note.Append(" (promotable)"); } Log(Note.ToString()); } } bool bShowDependencies = bp.ParseParam("ShowDependencies"); bool AddEmailProps = bp.ParseParam("ShowEmails"); string LastControllingTrigger = ""; string LastAgentGroup = ""; Log("*********** Desired And Dependent Nodes, in order."); foreach (BuildNode NodeToDo in Nodes) { string MyControllingTrigger = NodeToDo.ControllingTriggerDotName; if (MyControllingTrigger != LastControllingTrigger) { LastControllingTrigger = MyControllingTrigger; if (MyControllingTrigger != "") { string Finished = ""; if (UnfinishedTriggers != null) { if (NodeToDo.ControllingTriggers.Length > 0 && UnfinishedTriggers.Contains(NodeToDo.ControllingTriggers.Last())) { Finished = "(not yet triggered)"; } else { Finished = "(already triggered)"; } } Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished); } } if (NodeToDo.AgentSharingGroup != LastAgentGroup && NodeToDo.AgentSharingGroup != "") { Log(" Agent Group: {0}", NodeToDo.AgentSharingGroup); } LastAgentGroup = NodeToDo.AgentSharingGroup; StringBuilder Builder = new StringBuilder(" "); if(LastAgentGroup != "") { Builder.Append(" "); } Builder.AppendFormat("{0} ({1})", NodeToDo.Name, GetTimeIntervalString(TimeQuantum << NodeToDo.FrequencyShift)); if(NodeToDo.IsComplete) { Builder.Append(" - (Completed)"); } if(NodeToDo is TriggerNode) { Builder.Append(" - (TriggerNode)"); } if(NodeToDo.IsSticky) { Builder.Append(" - (Sticky)"); } string Agent = NodeToDo.AgentRequirements; if(ParseParamValue("AgentOverride") != "" && !NodeToDo.Node.GetFullName().Contains("Mac")) { Agent = ParseParamValue("AgentOverride"); } if (!String.IsNullOrEmpty(Agent)) { Builder.AppendFormat(" [{0}]", Agent); } if(NodeToDo.AgentMemoryRequirement != 0) { Builder.AppendFormat(" [{0}gb]", NodeToDo.AgentMemoryRequirement); } if (AddEmailProps) { Builder.AppendFormat(" {0}", String.Join(" ", NodeToDo.RecipientsForFailureEmails)); } Log(Builder.ToString()); if (bShowDependencies) { foreach (BuildNode Dep in NodeToDo.Dependencies) { Log(" dep> {0}", Dep.Name); } foreach (BuildNode Dep in NodeToDo.PseudoDependencies) { Log(" pdep> {0}", Dep.Name); } } } }
public override void DoBuild(GUBP bp) { CommandUtils.CreateDirectory(SavedDir); BuildProducts = new List<string>(); List<string> ManifestFiles = new List<string>(); if(!bp.ParseParam("NoDDC")) { // Find all the projects we're interested in List<BranchInfo.BranchUProject> Projects = new List<BranchInfo.BranchUProject>(); foreach(string ProjectName in ProjectNames) { BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(ProjectName); if(!Project.Properties.bIsCodeBasedProject) { Projects.Add(Project); } } // Filter out the files we need to build DDC. Removing confidential folders can affect DDC keys, so we want to be sure that we're making DDC with a build that can use it. FileFilter Filter = new FileFilter(FileFilterType.Exclude); Filter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include); Filter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini"), "CopyEditor", HostPlatform.ToString()); Filter.Exclude("/Engine/Build/..."); Filter.Exclude("/Engine/Extras/..."); Filter.Exclude("/Engine/DerivedDataCache/..."); Filter.Exclude("/Samples/..."); Filter.Exclude("/Templates/..."); Filter.Exclude(".../Source/..."); Filter.Exclude(".../Intermediate/..."); Filter.ExcludeConfidentialPlatforms(); Filter.ExcludeConfidentialFolders(); Filter.Include("/Engine/Build/NotForLicensees/EpicInternal.txt"); Filter.Include("/Engine/Binaries/.../*DDCUtils*"); // Make sure we can use the shared DDC! // Copy everything to a temporary directory string TempDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketDDC", CommandUtils.GetGenericPlatformName(HostPlatform)); CommandUtils.DeleteDirectoryContents(TempDir); CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, TempDir, Filter, true); // Get paths to everything within the temporary directory string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform); string RelativePakPath = "Engine/DerivedDataCache/Compressed.ddp"; string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath); string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv"); // Generate DDC for all the non-code projects. We don't necessarily have editor DLLs for the code projects, but they should be the same as their blueprint counterparts. List<string> ProjectPakFiles = new List<string>(); foreach(BranchInfo.BranchUProject Project in Projects) { CommandUtils.Log("Generating DDC data for {0} on {1}", Project.GameName, TargetPlatforms); CommandUtils.DDCCommandlet(Project.FilePath, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak -ProjectOnly"); string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", Project.GameName)); CommandUtils.DeleteFile(ProjectPakFile); CommandUtils.RenameFile(OutputPakFile, ProjectPakFile); string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv"); CommandUtils.DeleteFile(ProjectCsvFile); CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile); ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile)); } // Generate DDC for the editor, and merge all the other PAK files in CommandUtils.Log("Generating DDC data for engine content on {0}", TargetPlatforms); CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak " + CommandUtils.MakePathSafeToUseWithCommandLine("-MergePaks=" + String.Join("+", ProjectPakFiles))); // Copy the DDP file to the output path string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath); CommandUtils.CopyFile(OutputPakFile, SavedPakFile); BuildProducts.Add(SavedPakFile); // Add the pak file to the list of files to copy ManifestFiles.Add(RelativePakPath); } CommandUtils.WriteAllLines(SavedManifestPath, ManifestFiles.ToArray()); BuildProducts.Add(SavedManifestPath); SaveRecordOfSuccessAndAddToBuildProducts(); }
void PrintNodes(GUBP bp, List<string> Nodes, bool LocalOnly, List<string> UnfinishedTriggers = null) { bool bShowAllChanges = bp.ParseParam("AllChanges") && GUBPNodesHistory != null; bool bShowChanges = (bp.ParseParam("Changes") && GUBPNodesHistory != null) || bShowAllChanges; bool bShowDetailedHistory = (bp.ParseParam("History") && GUBPNodesHistory != null) || bShowChanges; bool bShowDependencies = !bp.ParseParam("NoShowDependencies") && !bShowDetailedHistory; bool bShowECDependencies = bp.ParseParam("ShowECDependencies"); bool bShowHistory = !bp.ParseParam("NoHistory") && GUBPNodesHistory != null; bool AddEmailProps = bp.ParseParam("ShowEmails"); bool bShowTriggers = true; string LastControllingTrigger = "$$NoMatch"; foreach (var NodeToDo in Nodes) { string EMails = ""; if (AddEmailProps) { EMails = GetEMailListForNode(bp, NodeToDo); } if (bShowTriggers) { string MyControllingTrigger = GetControllingTriggerDotName(NodeToDo); if (MyControllingTrigger != LastControllingTrigger) { LastControllingTrigger = MyControllingTrigger; if (MyControllingTrigger != "") { string Finished = ""; if (UnfinishedTriggers != null) { string MyShortControllingTrigger = GetControllingTrigger(NodeToDo); if (UnfinishedTriggers.Contains(MyShortControllingTrigger)) { Finished = "(not yet triggered)"; } else { Finished = "(already triggered)"; } } Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished); } } } string Agent = GUBPNodes[NodeToDo].ECAgentString(); if (Agent != "") { Agent = "[" + Agent + "]"; } Log(" {0}{1}{2}{3}{4} {5}", NodeToDo, NodeIsAlreadyComplete(NodeToDo, LocalOnly) ? " - (Completed)" : "", GUBPNodes[NodeToDo].TriggerNode() ? " - (TriggerNode)" : "", GUBPNodes[NodeToDo].IsSticky() ? " - (Sticky)" : "", Agent, EMails ); if (bShowHistory && GUBPNodesHistory.ContainsKey(NodeToDo)) { var History = GUBPNodesHistory[NodeToDo]; if (bShowDetailedHistory) { PrintDetailedChanges(History, bShowAllChanges); } else { Log(" Last Success: {0}", History.LastSucceeded); Log(" Fails Since: {0}", History.FailedString); Log(" InProgress Since: {0}", History.InProgressString); } } if (bShowDependencies) { foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies) { Log(" {0}", Dep); } foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies) { Log(" {0} (pseudo)", Dep); } } if (bShowECDependencies) { foreach (var Dep in GetECDependencies(NodeToDo)) { Log(" {0}", Dep); } } } }
public override int AgentMemoryRequirement(GUBP bp) { if (bp.ParseParam("Launcher") || bp.TimeIndex != 0 && HostPlatform != UnrealTargetPlatform.Mac) { return base.AgentMemoryRequirement(bp); } return 32; }
void PrintNodes(GUBP bp, List<string> Nodes, bool LocalOnly, List<string> UnfinishedTriggers = null) { bool bShowAllChanges = bp.ParseParam("AllChanges") && GUBPNodesHistory != null; bool bShowChanges = (bp.ParseParam("Changes") && GUBPNodesHistory != null) || bShowAllChanges; bool bShowDetailedHistory = (bp.ParseParam("History") && GUBPNodesHistory != null) || bShowChanges; bool bShowDependencies = bp.ParseParam("ShowDependencies"); bool bShowDependednOn = bp.ParseParam("ShowDependedOn"); bool bShowDependentPromotions = bp.ParseParam("ShowDependentPromotions"); bool bShowECDependencies = bp.ParseParam("ShowECDependencies"); bool bShowHistory = !bp.ParseParam("NoHistory") && GUBPNodesHistory != null; bool AddEmailProps = bp.ParseParam("ShowEmails"); bool ECProc = bp.ParseParam("ShowECProc"); bool ECOnly = bp.ParseParam("ShowECOnly"); bool bShowTriggers = true; string LastControllingTrigger = ""; string LastAgentGroup = ""; foreach (var NodeToDo in Nodes) { if (ECOnly && !GUBPNodes[NodeToDo].RunInEC()) { continue; } string EMails = ""; if (AddEmailProps) { EMails = GetEMailListForNode(bp, NodeToDo, "", ""); } if (bShowTriggers) { string MyControllingTrigger = GetControllingTriggerDotName(NodeToDo); if (MyControllingTrigger != LastControllingTrigger) { LastControllingTrigger = MyControllingTrigger; if (MyControllingTrigger != "") { string Finished = ""; if (UnfinishedTriggers != null) { string MyShortControllingTrigger = GetControllingTrigger(NodeToDo); if (UnfinishedTriggers.Contains(MyShortControllingTrigger)) { Finished = "(not yet triggered)"; } else { Finished = "(already triggered)"; } } Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished); } } } if (GUBPNodes[NodeToDo].AgentSharingGroup != LastAgentGroup && GUBPNodes[NodeToDo].AgentSharingGroup != "") { Log(" Agent Group: {0}", GUBPNodes[NodeToDo].AgentSharingGroup); } LastAgentGroup = GUBPNodes[NodeToDo].AgentSharingGroup; string Agent = GUBPNodes[NodeToDo].ECAgentString(); if(ParseParamValue("AgentOverride") != "" && !GUBPNodes[NodeToDo].GetFullName().Contains("Mac")) { Agent = ParseParamValue("AgentOverride"); } if (Agent != "") { Agent = "[" + Agent + "]"; } string MemoryReq = "[" + GUBPNodes[NodeToDo].AgentMemoryRequirement(bp).ToString() + "]"; if(MemoryReq == "[0]") { MemoryReq = ""; } string FrequencyString = CISFrequencyQuantumShiftString(NodeToDo); Log(" {0}{1}{2}{3}{4}{5}{6} {7} {8}", (LastAgentGroup != "" ? " " : ""), NodeToDo, FrequencyString, NodeIsAlreadyComplete(NodeToDo, LocalOnly) ? " - (Completed)" : "", GUBPNodes[NodeToDo].TriggerNode() ? " - (TriggerNode)" : "", GUBPNodes[NodeToDo].IsSticky() ? " - (Sticky)" : "", Agent, MemoryReq, EMails, ECProc ? GUBPNodes[NodeToDo].ECProcedure() : "" ); if (bShowHistory && GUBPNodesHistory.ContainsKey(NodeToDo)) { var History = GUBPNodesHistory[NodeToDo]; if (bShowDetailedHistory) { PrintDetailedChanges(History, bShowAllChanges); } else { Log(" Last Success: {0}", History.LastSucceeded); Log(" Last Fail : {0}", History.LastFailed); Log(" Fails Since: {0}", History.FailedString); Log(" InProgress Since: {0}", History.InProgressString); } } if (bShowDependencies) { foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies) { Log(" dep> {0}", Dep); } foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies) { Log(" pdep> {0}", Dep); } foreach (var Dep in GUBPNodes[NodeToDo].CompletedDependencies) { Log(" cdep>{0}", Dep); } } if (bShowECDependencies) { foreach (var Dep in GetECDependencies(NodeToDo)) { Log(" {0}", Dep); } foreach (var Dep in GetCompletedOnlyDependencies(NodeToDo)) { Log(" compDep> {0}", Dep); } } if(bShowDependednOn) { foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependedOn) { Log(" depOn> {0}", Dep); } } if (bShowDependentPromotions) { foreach (var Dep in GUBPNodes[NodeToDo].DependentPromotions) { Log(" depPro> {0}", Dep); } } } }
public override int AgentMemoryRequirement(GUBP bp) { if (bp.ParseParam("Launcher") || bp.TimeIndex != 0) { return base.AgentMemoryRequirement(bp); } return 32; }