public override void AddNodes(GUBP bp, UnrealTargetPlatform HostPlatform)
{
if(!bp.BranchOptions.bNoInstalledEngine)
{
// Find all the target platforms for this host platform.
List<UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(bp, HostPlatform);
// Remove any platforms that aren't available on this machine
TargetPlatforms.RemoveAll(x => !bp.ActivePlatforms.Contains(x));
// Get the temp directory for stripped files for this host
string StrippedDir = Path.GetFullPath(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Rocket", HostPlatform.ToString()));
// Strip the host platform
if (StripRocketNode.IsRequiredForPlatform(HostPlatform))
{
bp.AddNode(new StripRocketToolsNode(HostPlatform, StrippedDir));
bp.AddNode(new StripRocketEditorNode(HostPlatform, StrippedDir));
}
// Strip all the target platforms that are built on this host
foreach (UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if (GetSourceHostPlatform(bp, HostPlatform, TargetPlatform) == HostPlatform && StripRocketNode.IsRequiredForPlatform(TargetPlatform))
{
bp.AddNode(new StripRocketMonolithicsNode(bp, HostPlatform, TargetPlatform, StrippedDir));
}
}
// Build the DDC
bp.AddNode(new BuildDerivedDataCacheNode(HostPlatform, GetCookPlatforms(HostPlatform, TargetPlatforms), CurrentFeaturePacks));
// Generate a list of files that needs to be copied for each target platform
bp.AddNode(new FilterRocketNode(bp, HostPlatform, TargetPlatforms, CurrentFeaturePacks, CurrentTemplates));
// Copy the install to the output directory
string LocalOutputDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "Rocket", CommandUtils.GetGenericPlatformName(HostPlatform));
bp.AddNode(new GatherRocketNode(HostPlatform, TargetPlatforms, LocalOutputDir));
// Add the aggregate node for the entire install
GUBP.GUBPNode PromotableNode = bp.FindNode(GUBP.SharedAggregatePromotableNode.StaticGetFullName());
PromotableNode.AddDependency(FilterRocketNode.StaticGetFullName(HostPlatform));
PromotableNode.AddDependency(BuildDerivedDataCacheNode.StaticGetFullName(HostPlatform));
// Add a node for GitHub promotions
if(HostPlatform == UnrealTargetPlatform.Win64)
{
string GitConfigRelativePath = "Engine/Build/Git/UnrealBot.ini";
if(CommandUtils.FileExists(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, GitConfigRelativePath)))
{
bp.AddNode(new BuildGitPromotable(bp, HostPlatform, GitConfigRelativePath));
PromotableNode.AddDependency(BuildGitPromotable.StaticGetFullName(HostPlatform));
}
}
// Get the output directory for the build zips
string PublishedEngineDir;
if(ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP())
{
PublishedEngineDir = CommandUtils.CombinePaths(CommandUtils.RootSharedTempStorageDirectory(), "Rocket", "Automated", GetBuildLabel(), CommandUtils.GetGenericPlatformName(HostPlatform));
}
else
{
PublishedEngineDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketPublish", CommandUtils.GetGenericPlatformName(HostPlatform));
}
// Publish the install to the network
bp.AddNode(new PublishRocketNode(HostPlatform, LocalOutputDir, PublishedEngineDir));
bp.AddNode(new PublishRocketSymbolsNode(bp, HostPlatform, TargetPlatforms, PublishedEngineDir + "Symbols"));
// Add a dependency on this being published as part of the shared promotable being labeled
GUBP.SharedLabelPromotableSuccessNode LabelPromotableNode = (GUBP.SharedLabelPromotableSuccessNode)bp.FindNode(GUBP.SharedLabelPromotableSuccessNode.StaticGetFullName());
LabelPromotableNode.AddDependency(PublishRocketNode.StaticGetFullName(HostPlatform));
LabelPromotableNode.AddDependency(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform));
// Add dependencies on a promotable to do these steps too
GUBP.WaitForSharedPromotionUserInput WaitForPromotionNode = (GUBP.WaitForSharedPromotionUserInput)bp.FindNode(GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(true));
WaitForPromotionNode.AddDependency(PublishRocketNode.StaticGetFullName(HostPlatform));
WaitForPromotionNode.AddDependency(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform));
// Push everything behind the promotion triggers if we're doing things on the build machines
if(ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP() || bp.ParseParam("WithRocketPromotable"))
{
string WaitForTrigger = GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(false);
GatherRocketNode GatherRocket = (GatherRocketNode)bp.FindNode(GatherRocketNode.StaticGetFullName(HostPlatform));
GatherRocket.AddDependency(WaitForTrigger);
PublishRocketSymbolsNode PublishRocketSymbols = (PublishRocketSymbolsNode)bp.FindNode(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform));
PublishRocketSymbols.AddDependency(WaitForTrigger);
}
}
}
void PrintNodes(GUBP bp, List<BuildNode> Nodes, IEnumerable<AggregateNode> Aggregates, List<TriggerNode> UnfinishedTriggers, int TimeQuantum)
{
AggregateNode[] MatchingAggregates = Aggregates.Where(x => x.Dependencies.All(y => Nodes.Contains(y))).ToArray();
if (MatchingAggregates.Length > 0)
{
Log("*********** Aggregates");
foreach (AggregateNode Aggregate in MatchingAggregates.OrderBy(x => x.Name))
{
StringBuilder Note = new StringBuilder(" " + Aggregate.Name);
if (Aggregate.Node.IsPromotableAggregate())
{
Note.Append(" (promotable)");
}
Log(Note.ToString());
}
}
bool bShowDependencies = bp.ParseParam("ShowDependencies");
bool AddEmailProps = bp.ParseParam("ShowEmails");
string LastControllingTrigger = "";
string LastAgentGroup = "";
Log("*********** Desired And Dependent Nodes, in order.");
foreach (BuildNode NodeToDo in Nodes)
{
string MyControllingTrigger = NodeToDo.ControllingTriggerDotName;
if (MyControllingTrigger != LastControllingTrigger)
{
LastControllingTrigger = MyControllingTrigger;
if (MyControllingTrigger != "")
{
string Finished = "";
if (UnfinishedTriggers != null)
{
if (NodeToDo.ControllingTriggers.Length > 0 && UnfinishedTriggers.Contains(NodeToDo.ControllingTriggers.Last()))
{
Finished = "(not yet triggered)";
}
else
{
Finished = "(already triggered)";
}
}
Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished);
}
}
if (NodeToDo.AgentSharingGroup != LastAgentGroup && NodeToDo.AgentSharingGroup != "")
{
Log(" Agent Group: {0}", NodeToDo.AgentSharingGroup);
}
LastAgentGroup = NodeToDo.AgentSharingGroup;
StringBuilder Builder = new StringBuilder(" ");
if(LastAgentGroup != "")
{
Builder.Append(" ");
}
Builder.AppendFormat("{0} ({1})", NodeToDo.Name, GetTimeIntervalString(TimeQuantum << NodeToDo.FrequencyShift));
if(NodeToDo.IsComplete)
{
Builder.Append(" - (Completed)");
}
if(NodeToDo is TriggerNode)
{
Builder.Append(" - (TriggerNode)");
}
if(NodeToDo.IsSticky)
{
Builder.Append(" - (Sticky)");
}
string Agent = NodeToDo.AgentRequirements;
if(ParseParamValue("AgentOverride") != "" && !NodeToDo.Node.GetFullName().Contains("Mac"))
{
Agent = ParseParamValue("AgentOverride");
}
if (!String.IsNullOrEmpty(Agent))
{
Builder.AppendFormat(" [{0}]", Agent);
}
if(NodeToDo.AgentMemoryRequirement != 0)
{
Builder.AppendFormat(" [{0}gb]", NodeToDo.AgentMemoryRequirement);
}
if (AddEmailProps)
{
Builder.AppendFormat(" {0}", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
}
Log(Builder.ToString());
if (bShowDependencies)
{
foreach (BuildNode Dep in NodeToDo.Dependencies)
{
Log(" dep> {0}", Dep.Name);
}
foreach (BuildNode Dep in NodeToDo.PseudoDependencies)
{
Log(" pdep> {0}", Dep.Name);
}
}
}
}
public override void DoBuild(GUBP bp)
{
CommandUtils.CreateDirectory(SavedDir);
BuildProducts = new List<string>();
List<string> ManifestFiles = new List<string>();
if(!bp.ParseParam("NoDDC"))
{
// Find all the projects we're interested in
List<BranchInfo.BranchUProject> Projects = new List<BranchInfo.BranchUProject>();
foreach(string ProjectName in ProjectNames)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(ProjectName);
if(!Project.Properties.bIsCodeBasedProject)
{
Projects.Add(Project);
}
}
// Filter out the files we need to build DDC. Removing confidential folders can affect DDC keys, so we want to be sure that we're making DDC with a build that can use it.
FileFilter Filter = new FileFilter(FileFilterType.Exclude);
Filter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
Filter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini"), "CopyEditor", HostPlatform.ToString());
Filter.Exclude("/Engine/Build/...");
Filter.Exclude("/Engine/Extras/...");
Filter.Exclude("/Engine/DerivedDataCache/...");
Filter.Exclude("/Samples/...");
Filter.Exclude("/Templates/...");
Filter.Exclude(".../Source/...");
Filter.Exclude(".../Intermediate/...");
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
Filter.Include("/Engine/Build/NotForLicensees/EpicInternal.txt");
Filter.Include("/Engine/Binaries/.../*DDCUtils*"); // Make sure we can use the shared DDC!
// Copy everything to a temporary directory
string TempDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketDDC", CommandUtils.GetGenericPlatformName(HostPlatform));
CommandUtils.DeleteDirectoryContents(TempDir);
CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, TempDir, Filter, true);
// Get paths to everything within the temporary directory
string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform);
string RelativePakPath = "Engine/DerivedDataCache/Compressed.ddp";
string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath);
string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv");
// Generate DDC for all the non-code projects. We don't necessarily have editor DLLs for the code projects, but they should be the same as their blueprint counterparts.
List<string> ProjectPakFiles = new List<string>();
foreach(BranchInfo.BranchUProject Project in Projects)
{
CommandUtils.Log("Generating DDC data for {0} on {1}", Project.GameName, TargetPlatforms);
CommandUtils.DDCCommandlet(Project.FilePath, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak -ProjectOnly");
string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", Project.GameName));
CommandUtils.DeleteFile(ProjectPakFile);
CommandUtils.RenameFile(OutputPakFile, ProjectPakFile);
string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv");
CommandUtils.DeleteFile(ProjectCsvFile);
CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile);
ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile));
}
// Generate DDC for the editor, and merge all the other PAK files in
CommandUtils.Log("Generating DDC data for engine content on {0}", TargetPlatforms);
CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak " + CommandUtils.MakePathSafeToUseWithCommandLine("-MergePaks=" + String.Join("+", ProjectPakFiles)));
// Copy the DDP file to the output path
string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath);
CommandUtils.CopyFile(OutputPakFile, SavedPakFile);
BuildProducts.Add(SavedPakFile);
// Add the pak file to the list of files to copy
ManifestFiles.Add(RelativePakPath);
}
CommandUtils.WriteAllLines(SavedManifestPath, ManifestFiles.ToArray());
BuildProducts.Add(SavedManifestPath);
SaveRecordOfSuccessAndAddToBuildProducts();
}
void PrintNodes(GUBP bp, List<string> Nodes, bool LocalOnly, List<string> UnfinishedTriggers = null)
{
bool bShowAllChanges = bp.ParseParam("AllChanges") && GUBPNodesHistory != null;
bool bShowChanges = (bp.ParseParam("Changes") && GUBPNodesHistory != null) || bShowAllChanges;
bool bShowDetailedHistory = (bp.ParseParam("History") && GUBPNodesHistory != null) || bShowChanges;
bool bShowDependencies = !bp.ParseParam("NoShowDependencies") && !bShowDetailedHistory;
bool bShowECDependencies = bp.ParseParam("ShowECDependencies");
bool bShowHistory = !bp.ParseParam("NoHistory") && GUBPNodesHistory != null;
bool AddEmailProps = bp.ParseParam("ShowEmails");
bool bShowTriggers = true;
string LastControllingTrigger = "$$NoMatch";
foreach (var NodeToDo in Nodes)
{
string EMails = "";
if (AddEmailProps)
{
EMails = GetEMailListForNode(bp, NodeToDo);
}
if (bShowTriggers)
{
string MyControllingTrigger = GetControllingTriggerDotName(NodeToDo);
if (MyControllingTrigger != LastControllingTrigger)
{
LastControllingTrigger = MyControllingTrigger;
if (MyControllingTrigger != "")
{
string Finished = "";
if (UnfinishedTriggers != null)
{
string MyShortControllingTrigger = GetControllingTrigger(NodeToDo);
if (UnfinishedTriggers.Contains(MyShortControllingTrigger))
{
Finished = "(not yet triggered)";
}
else
{
Finished = "(already triggered)";
}
}
Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished);
}
}
}
string Agent = GUBPNodes[NodeToDo].ECAgentString();
if (Agent != "")
{
Agent = "[" + Agent + "]";
}
Log(" {0}{1}{2}{3}{4} {5}", NodeToDo,
NodeIsAlreadyComplete(NodeToDo, LocalOnly) ? " - (Completed)" : "",
GUBPNodes[NodeToDo].TriggerNode() ? " - (TriggerNode)" : "",
GUBPNodes[NodeToDo].IsSticky() ? " - (Sticky)" : "",
Agent,
EMails
);
if (bShowHistory && GUBPNodesHistory.ContainsKey(NodeToDo))
{
var History = GUBPNodesHistory[NodeToDo];
if (bShowDetailedHistory)
{
PrintDetailedChanges(History, bShowAllChanges);
}
else
{
Log(" Last Success: {0}", History.LastSucceeded);
Log(" Fails Since: {0}", History.FailedString);
Log(" InProgress Since: {0}", History.InProgressString);
}
}
if (bShowDependencies)
{
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
Log(" {0}", Dep);
}
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies)
{
Log(" {0} (pseudo)", Dep);
}
}
if (bShowECDependencies)
{
foreach (var Dep in GetECDependencies(NodeToDo))
{
Log(" {0}", Dep);
}
}
}
}
public override int AgentMemoryRequirement(GUBP bp)
{
if (bp.ParseParam("Launcher") || bp.TimeIndex != 0 && HostPlatform != UnrealTargetPlatform.Mac)
{
return base.AgentMemoryRequirement(bp);
}
return 32;
}
void PrintNodes(GUBP bp, List<string> Nodes, bool LocalOnly, List<string> UnfinishedTriggers = null)
{
bool bShowAllChanges = bp.ParseParam("AllChanges") && GUBPNodesHistory != null;
bool bShowChanges = (bp.ParseParam("Changes") && GUBPNodesHistory != null) || bShowAllChanges;
bool bShowDetailedHistory = (bp.ParseParam("History") && GUBPNodesHistory != null) || bShowChanges;
bool bShowDependencies = bp.ParseParam("ShowDependencies");
bool bShowDependednOn = bp.ParseParam("ShowDependedOn");
bool bShowDependentPromotions = bp.ParseParam("ShowDependentPromotions");
bool bShowECDependencies = bp.ParseParam("ShowECDependencies");
bool bShowHistory = !bp.ParseParam("NoHistory") && GUBPNodesHistory != null;
bool AddEmailProps = bp.ParseParam("ShowEmails");
bool ECProc = bp.ParseParam("ShowECProc");
bool ECOnly = bp.ParseParam("ShowECOnly");
bool bShowTriggers = true;
string LastControllingTrigger = "";
string LastAgentGroup = "";
foreach (var NodeToDo in Nodes)
{
if (ECOnly && !GUBPNodes[NodeToDo].RunInEC())
{
continue;
}
string EMails = "";
if (AddEmailProps)
{
EMails = GetEMailListForNode(bp, NodeToDo, "", "");
}
if (bShowTriggers)
{
string MyControllingTrigger = GetControllingTriggerDotName(NodeToDo);
if (MyControllingTrigger != LastControllingTrigger)
{
LastControllingTrigger = MyControllingTrigger;
if (MyControllingTrigger != "")
{
string Finished = "";
if (UnfinishedTriggers != null)
{
string MyShortControllingTrigger = GetControllingTrigger(NodeToDo);
if (UnfinishedTriggers.Contains(MyShortControllingTrigger))
{
Finished = "(not yet triggered)";
}
else
{
Finished = "(already triggered)";
}
}
Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished);
}
}
}
if (GUBPNodes[NodeToDo].AgentSharingGroup != LastAgentGroup && GUBPNodes[NodeToDo].AgentSharingGroup != "")
{
Log(" Agent Group: {0}", GUBPNodes[NodeToDo].AgentSharingGroup);
}
LastAgentGroup = GUBPNodes[NodeToDo].AgentSharingGroup;
string Agent = GUBPNodes[NodeToDo].ECAgentString();
if(ParseParamValue("AgentOverride") != "" && !GUBPNodes[NodeToDo].GetFullName().Contains("Mac"))
{
Agent = ParseParamValue("AgentOverride");
}
if (Agent != "")
{
Agent = "[" + Agent + "]";
}
string MemoryReq = "[" + GUBPNodes[NodeToDo].AgentMemoryRequirement(bp).ToString() + "]";
if(MemoryReq == "[0]")
{
MemoryReq = "";
}
string FrequencyString = CISFrequencyQuantumShiftString(NodeToDo);
Log(" {0}{1}{2}{3}{4}{5}{6} {7} {8}",
(LastAgentGroup != "" ? " " : ""),
NodeToDo,
FrequencyString,
NodeIsAlreadyComplete(NodeToDo, LocalOnly) ? " - (Completed)" : "",
GUBPNodes[NodeToDo].TriggerNode() ? " - (TriggerNode)" : "",
GUBPNodes[NodeToDo].IsSticky() ? " - (Sticky)" : "",
Agent,
MemoryReq,
EMails,
ECProc ? GUBPNodes[NodeToDo].ECProcedure() : ""
);
if (bShowHistory && GUBPNodesHistory.ContainsKey(NodeToDo))
{
var History = GUBPNodesHistory[NodeToDo];
if (bShowDetailedHistory)
{
PrintDetailedChanges(History, bShowAllChanges);
}
else
{
Log(" Last Success: {0}", History.LastSucceeded);
Log(" Last Fail : {0}", History.LastFailed);
Log(" Fails Since: {0}", History.FailedString);
Log(" InProgress Since: {0}", History.InProgressString);
}
}
if (bShowDependencies)
{
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
Log(" dep> {0}", Dep);
}
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies)
{
Log(" pdep> {0}", Dep);
}
foreach (var Dep in GUBPNodes[NodeToDo].CompletedDependencies)
{
Log(" cdep>{0}", Dep);
}
}
if (bShowECDependencies)
{
foreach (var Dep in GetECDependencies(NodeToDo))
{
Log(" {0}", Dep);
}
foreach (var Dep in GetCompletedOnlyDependencies(NodeToDo))
{
Log(" compDep> {0}", Dep);
}
}
if(bShowDependednOn)
{
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependedOn)
{
Log(" depOn> {0}", Dep);
}
}
if (bShowDependentPromotions)
{
foreach (var Dep in GUBPNodes[NodeToDo].DependentPromotions)
{
Log(" depPro> {0}", Dep);
}
}
}
}
public override int AgentMemoryRequirement(GUBP bp)
{
if (bp.ParseParam("Launcher") || bp.TimeIndex != 0)
{
return base.AgentMemoryRequirement(bp);
}
return 32;
}