public override UE4Build.BuildAgenda GetAgenda(GUBP bp)
{
var Agenda = new UE4Build.BuildAgenda();
Agenda.DotNetSolutions.Add("Engine/Source/Programs/UnrealDocTool/APIDocTool/APIDocTool.sln");
Agenda.DotNetSolutions.Add("Engine/Source/Programs/UnrealDocTool/UnrealDocTool/UnrealDocTool.sln");
return Agenda;
}
public override UE4Build.BuildAgenda GetAgenda(GUBP bp)
{
var Agenda = new UE4Build.BuildAgenda();
Agenda.DotNetSolutions.Add("Engine/Source/Programs/UnrealDocTool/APIDocTool/APIDocTool.sln");
Agenda.DotNetSolutions.Add("Engine/Source/Programs/UnrealDocTool/UnrealDocTool/UnrealDocTool.sln");
return(Agenda);
}
public override void AddNodes(GUBP bp, UnrealTargetPlatform InHostPlatform)
{
if (InHostPlatform == UnrealTargetPlatform.Win64 && !bp.BranchOptions.bNoDocumentation)
{
bp.AddNode(new ToolsForDocumentationNode(InHostPlatform));
bp.AddNode(new CodeDocumentationNode(InHostPlatform));
bp.AddNode(new BlueprintDocumentationNode(InHostPlatform));
}
}
public override void AddNodes(GUBP bp, GUBP.GUBPBranchConfig BranchConfig, UnrealTargetPlatform InHostPlatform, List<UnrealTargetPlatform> InActivePlatforms)
{
/* if(InHostPlatform == UnrealTargetPlatform.Win64 && !BranchConfig.BranchOptions.bNoDocumentation)
{
BranchConfig.AddNode(new ToolsForDocumentationNode(BranchConfig, InHostPlatform));
BranchConfig.AddNode(new CodeDocumentationNode(InHostPlatform));
BranchConfig.AddNode(new BlueprintDocumentationNode(InHostPlatform));
}*/
}
public override void AddNodes(GUBP bp, UnrealTargetPlatform InHostPlatform)
{
if(InHostPlatform == UnrealTargetPlatform.Win64 && !bp.BranchOptions.bNoDocumentation)
{
bp.AddNode(new ToolsForDocumentationNode(InHostPlatform));
bp.AddNode(new CodeDocumentationNode(InHostPlatform));
bp.AddNode(new BlueprintDocumentationNode(InHostPlatform));
}
}
public TriggerNode(GUBP bp, GUBP.WaitForUserInput InNode) : base(bp, InNode)
{
AddSubmittersToFailureEmails = false;
StateName = InNode.GetTriggerStateName();
DescriptionText = InNode.GetTriggerDescText();
ActionText = InNode.GetTriggerActionText();
RequiresRecursiveWorkflow = InNode.TriggerRequiresRecursiveWorkflow();
}
public override void AddNodes(GUBP bp, GUBP.GUBPBranchConfig BranchConfig, UnrealTargetPlatform InHostPlatform, List <UnrealTargetPlatform> InActivePlatforms)
{
if (InHostPlatform == UnrealTargetPlatform.Win64 && !BranchConfig.BranchOptions.bNoDocumentation)
{
BranchConfig.AddNode(new ToolsForDocumentationNode(BranchConfig, InHostPlatform));
BranchConfig.AddNode(new CodeDocumentationNode(InHostPlatform));
BranchConfig.AddNode(new BlueprintDocumentationNode(InHostPlatform));
}
}
public LegacyNode(GUBP.GUBPNode InNode)
{
Name = InNode.GetFullName();
Node = InNode;
AgentRequirements = Node.ECAgentString();
AgentSharingGroup = Node.AgentSharingGroup;
AgentMemoryRequirement = Node.AgentMemoryRequirement();
TimeoutInMinutes = Node.TimeoutInMinutes();
SendSuccessEmail = Node.SendSuccessEmail();
Priority = Node.Priority();
}
public BuildDerivedDataCacheNode(GUBP.GUBPBranchConfig InBranchConfig, UnrealTargetPlatform InHostPlatform, string InTargetPlatforms, string[] InProjectNames)
: base(InHostPlatform)
{
BranchConfig = InBranchConfig;
TargetPlatforms = InTargetPlatforms;
ProjectNames = InProjectNames;
SavedDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Rocket", HostPlatform.ToString());
SavedManifestPath = CommandUtils.CombinePaths(SavedDir, "DerivedDataCacheManifest.txt");
AddDependency(GUBP.RootEditorNode.StaticGetFullName(HostPlatform));
AddDependency(GUBP.ToolsNode.StaticGetFullName(HostPlatform));
}
public LegacyNodeDefinition(GUBP InOwner, GUBP.GUBPNode InNode)
{
Owner = InOwner;
Node = InNode;
// Copy the definition from the node
Name = Node.GetFullName();
AgentRequirements = Node.ECAgentString();
AgentSharingGroup = Node.AgentSharingGroup;
AgentMemoryRequirement = Node.AgentMemoryRequirement();
TimeoutInMinutes = Node.TimeoutInMinutes();
SendSuccessEmail = Node.SendSuccessEmail();
Priority = Node.Priority();
IsSticky = Node.IsSticky();
DependsOn = String.Join(";", Node.FullNamesOfDependencies);
RunAfter = String.Join(";", Node.FullNamesOfPseudodependencies);
IsTest = Node.IsTest();
DisplayGroupName = Node.GetDisplayGroupName();
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
string InputDir = Path.GetFullPath(CommandUtils.CmdEnv.LocalRoot);
string RulesFileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini");
// Read the filter for files on this platform
FileFilter StripFilter = new FileFilter();
StripFilter.ReadRulesFromFile(RulesFileName, "StripSymbols." + TargetPlatform.ToString(), HostPlatform.ToString());
// Apply the filter to the build products
List<string> SourcePaths = new List<string>();
List<string> TargetPaths = new List<string>();
foreach(string NodeToStrip in NodesToStrip)
{
GUBP.GUBPNode Node = bp.FindNode(NodeToStrip);
foreach(string DependencyBuildProduct in Node.BuildProducts)
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(DependencyBuildProduct), InputDir);
if(StripFilter.Matches(RelativePath))
{
SourcePaths.Add(CommandUtils.CombinePaths(InputDir, RelativePath));
TargetPaths.Add(CommandUtils.CombinePaths(StrippedDir, RelativePath));
}
}
}
// Strip the files and add them to the build products
StripSymbols(TargetPlatform, SourcePaths.ToArray(), TargetPaths.ToArray());
BuildProducts.AddRange(TargetPaths);
SaveRecordOfSuccessAndAddToBuildProducts();
}
public StripRocketMonolithicsNode(GUBP bp, UnrealTargetPlatform InHostPlatform, UnrealTargetPlatform InTargetPlatform, string InStrippedDir)
: base(InHostPlatform, InTargetPlatform, InStrippedDir)
{
Project = bp.Branch.BaseEngineProject;
bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(InHostPlatform, InTargetPlatform);
GUBP.GUBPNode Node = bp.FindNode(GUBP.GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, Project, InTargetPlatform, Precompiled: bIsCodeTargetPlatform));
if(String.IsNullOrEmpty(Node.AgentSharingGroup))
{
Node.AgentSharingGroup = bp.Branch.BaseEngineProject.GameName + "_MonolithicsGroup_" + InTargetPlatform + StaticGetHostPlatformSuffix(InHostPlatform);
}
AddNodeToStrip(Node.GetFullName());
AgentSharingGroup = Node.AgentSharingGroup;
}
public override void DoBuild(GUBP bp)
{
ExecuteApiDocTool("-rebuildcode", "APIDocTool-Code");
SubmitOutputs("Code documentation", "Engine/Documentation/Builds/CodeAPI-*", "Engine/Documentation/CHM/API.chm");
base.DoBuild(bp);
}
public override int CISFrequencyQuantumShift(GUBP.GUBPBranchConfig BranchConfig)
{
return base.CISFrequencyQuantumShift(BranchConfig) + 3;
}
public override void DoBuild(GUBP bp)
{
ExecuteApiDocTool("-rebuildcode", "APIDocTool-Code");
SubmitOutputs("Code documentation", "Engine/Documentation/Builds/CodeAPI-*", "Engine/Documentation/CHM/API.chm");
base.DoBuild(bp);
}
public LegacyNode(LegacyNodeDefinition Definition)
: base(Definition)
{
Owner = Definition.Owner;
Node = Definition.Node;
}
public override void DoBuild(GUBP bp)
{
// Create a zip file containing the install
string FullZipFileName = Path.Combine(CommandUtils.CmdEnv.LocalRoot, "FullInstall" + StaticGetHostPlatformSuffix(HostPlatform) + ".zip");
CommandUtils.Log("Creating {0}...", FullZipFileName);
CommandUtils.ZipFiles(FullZipFileName, LocalDir, new FileFilter(FileFilterType.Include));
// Create a filter for the files we need just to run the editor
FileFilter EditorFilter = new FileFilter(FileFilterType.Include);
EditorFilter.Exclude("/Engine/Binaries/...");
EditorFilter.Include("/Engine/Binaries/DotNET/...");
EditorFilter.Include("/Engine/Binaries/ThirdParty/...");
EditorFilter.Include("/Engine/Binaries/" + HostPlatform.ToString() + "/...");
EditorFilter.Exclude("/Engine/Binaries/.../*.lib");
EditorFilter.Exclude("/Engine/Binaries/.../*.a");
EditorFilter.Exclude("/Engine/Extras/...");
EditorFilter.Exclude("/Engine/Source/.../Private/...");
EditorFilter.Exclude("/FeaturePacks/...");
EditorFilter.Exclude("/Samples/...");
EditorFilter.Exclude("/Templates/...");
EditorFilter.Exclude("*.pdb");
// Create a zip file containing the editor install
string EditorZipFileName = Path.Combine(CommandUtils.CmdEnv.LocalRoot, "EditorInstall" + StaticGetHostPlatformSuffix(HostPlatform) + ".zip");
CommandUtils.Log("Creating {0}...", EditorZipFileName);
CommandUtils.ZipFiles(EditorZipFileName, LocalDir, EditorFilter);
// Copy the files to their final location
CommandUtils.Log("Copying files to {0}", PublishDir);
CommandUtils.CopyFile(FullZipFileName, Path.Combine(PublishDir, Path.GetFileName(FullZipFileName)));
CommandUtils.CopyFile(EditorZipFileName, Path.Combine(PublishDir, Path.GetFileName(EditorZipFileName)));
CommandUtils.DeleteFile(FullZipFileName);
CommandUtils.DeleteFile(EditorZipFileName);
// Save a record of success
BuildProducts = new List<string>();
SaveRecordOfSuccessAndAddToBuildProducts();
}
public override void DoBuild(GUBP bp)
{
if(RocketBuild.ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP())
{
// Make a lookup for all the known debug extensions, and filter all the dependency build products against that
HashSet<string> DebugExtensions = new HashSet<string>(Platform.Platforms.Values.SelectMany(x => x.GetDebugFileExtentions()).Distinct().ToArray(), StringComparer.InvariantCultureIgnoreCase);
foreach(string InputFileName in AllDependencyBuildProducts)
{
string Extension = Path.GetExtension(InputFileName);
if(DebugExtensions.Contains(Extension) || Extension == ".exe" || Extension == ".dll") // Need all windows build products for crash reporter
{
string OutputFileName = CommandUtils.MakeRerootedFilePath(InputFileName, CommandUtils.CmdEnv.LocalRoot, SymbolsOutputDir);
if (SymbolsOutputDir.StartsWith("/Volumes"))
{
int Retry = 0;
int NumRetries = 60;
bool bCopied = false;
while (!bCopied && Retry < NumRetries)
{
if (Retry > 0)
{
CommandUtils.Log("*** Mac temp storage retry {0}", OutputFileName);
System.Threading.Thread.Sleep(1000);
}
bCopied = CommandUtils.CopyFile_NoExceptions(InputFileName, OutputFileName, true);
Retry++;
}
}
else
{
CommandUtils.CopyFile(InputFileName, OutputFileName);
}
}
}
}
// Add a dummy build product
BuildProducts = new List<string>();
SaveRecordOfSuccessAndAddToBuildProducts();
}
static void AddRuleForBuildProducts(FileFilter Filter, GUBP bp, string NodeName, FileFilterType Type)
{
GUBP.GUBPNode Node = bp.FindNode(NodeName);
if(Node == null)
{
throw new AutomationException("Couldn't find node '{0}'", NodeName);
}
Filter.AddRuleForFiles(Node.BuildProducts, CommandUtils.CmdEnv.LocalRoot, Type);
}
public override int CISFrequencyQuantumShift(GUBP bp)
{
return base.CISFrequencyQuantumShift(bp) + 2;
}
public FilterRocketNode(GUBP bp, UnrealTargetPlatform InHostPlatform, List<UnrealTargetPlatform> InTargetPlatforms, string[] InCurrentFeaturePacks, string[] InCurrentTemplates)
: base(InHostPlatform)
{
TargetPlatforms = new List<UnrealTargetPlatform>(InTargetPlatforms);
CurrentFeaturePacks = InCurrentFeaturePacks;
CurrentTemplates = InCurrentTemplates;
DepotManifestPath = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Rocket", HostPlatform.ToString(), "Filter.txt");
// Add the editor
AddDependency(GUBP.VersionFilesNode.StaticGetFullName());
AddDependency(GUBP.ToolsForCompileNode.StaticGetFullName(HostPlatform));
AddDependency(GUBP.RootEditorNode.StaticGetFullName(HostPlatform));
AddDependency(GUBP.ToolsNode.StaticGetFullName(HostPlatform));
// Add all the monolithic builds from their appropriate source host platform
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
bool bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(SourceHostPlatform, TargetPlatform);
AddDependency(GUBP.GamePlatformMonolithicsNode.StaticGetFullName(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, Precompiled: bIsCodeTargetPlatform));
}
// Also add stripped symbols for all the target platforms that require it
List<string> StrippedNodeNames = new List<string>();
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if(StripRocketNode.IsRequiredForPlatform(TargetPlatform))
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
string StripNode = StripRocketMonolithicsNode.StaticGetFullName(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, RocketBuild.IsCodeTargetPlatform(SourceHostPlatform, TargetPlatform));
AddDependency(StripNode);
StrippedNodeNames.Add(StripNode);
}
}
// Add win64 tools on Mac, to get the win64 build of UBT, UAT and IPP
if (HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
AddDependency(GUBP.ToolsNode.StaticGetFullName(UnrealTargetPlatform.Win64));
AddDependency(GUBP.ToolsForCompileNode.StaticGetFullName(UnrealTargetPlatform.Win64));
}
// Add all the feature packs
AddDependency(GUBP.MakeFeaturePacksNode.StaticGetFullName(GUBP.MakeFeaturePacksNode.GetDefaultBuildPlatform(bp)));
// Find all the host platforms we need
SourceHostPlatforms = TargetPlatforms.Select(x => RocketBuild.GetSourceHostPlatform(bp, HostPlatform, x)).Distinct().ToList();
if(!SourceHostPlatforms.Contains(HostPlatform))
{
SourceHostPlatforms.Add(HostPlatform);
}
// Add the stripped host platforms
if(StripRocketNode.IsRequiredForPlatform(HostPlatform))
{
AddDependency(StripRocketToolsNode.StaticGetFullName(HostPlatform));
StrippedNodeNames.Add(StripRocketToolsNode.StaticGetFullName(HostPlatform));
AddDependency(StripRocketEditorNode.StaticGetFullName(HostPlatform));
StrippedNodeNames.Add(StripRocketEditorNode.StaticGetFullName(HostPlatform));
}
// Set all the stripped manifest paths
foreach(string StrippedNodeName in StrippedNodeNames)
{
StrippedNodeManifestPaths.Add(StrippedNodeName, Path.Combine(Path.GetDirectoryName(DepotManifestPath), "Filter_" + StrippedNodeName + ".txt"));
}
}
public override int CISFrequencyQuantumShift(GUBP bp)
{
return(base.CISFrequencyQuantumShift(bp) + 3);
}
public override void DoBuild(GUBP bp)
{
ExecuteApiDocTool("-rebuildblueprint", "APIDocTool-Blueprint");
SubmitOutputs("Blueprint documentation", "Engine/Documentation/Builds/BlueprintAPI-*", "Engine/Documentation/CHM/BlueprintAPI.chm");
base.DoBuild(bp);
}
public BuildGitPromotable(GUBP bp, UnrealTargetPlatform HostPlatform, string InConfigRelativePath)
: base(HostPlatform)
{
ConfigRelativePath = InConfigRelativePath;
foreach(UnrealTargetPlatform OtherHostPlatform in bp.HostPlatforms)
{
AddDependency(GUBP.RootEditorNode.StaticGetFullName(OtherHostPlatform));
AddDependency(GUBP.ToolsNode.StaticGetFullName(OtherHostPlatform));
AddDependency(GUBP.InternalToolsNode.StaticGetFullName(OtherHostPlatform));
}
}
public override void AddNodes(GUBP bp, UnrealTargetPlatform HostPlatform)
{
if(!bp.BranchOptions.bNoInstalledEngine)
{
// Find all the target platforms for this host platform.
List<UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(bp, HostPlatform);
// Remove any platforms that aren't available on this machine
TargetPlatforms.RemoveAll(x => !bp.ActivePlatforms.Contains(x));
// Get the temp directory for stripped files for this host
string StrippedDir = Path.GetFullPath(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Rocket", HostPlatform.ToString()));
// Strip the host platform
if (StripRocketNode.IsRequiredForPlatform(HostPlatform))
{
bp.AddNode(new StripRocketToolsNode(HostPlatform, StrippedDir));
bp.AddNode(new StripRocketEditorNode(HostPlatform, StrippedDir));
}
// Strip all the target platforms that are built on this host
foreach (UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if (GetSourceHostPlatform(bp, HostPlatform, TargetPlatform) == HostPlatform && StripRocketNode.IsRequiredForPlatform(TargetPlatform))
{
bp.AddNode(new StripRocketMonolithicsNode(bp, HostPlatform, TargetPlatform, StrippedDir));
}
}
// Build the DDC
bp.AddNode(new BuildDerivedDataCacheNode(HostPlatform, GetCookPlatforms(HostPlatform, TargetPlatforms), CurrentFeaturePacks));
// Generate a list of files that needs to be copied for each target platform
bp.AddNode(new FilterRocketNode(bp, HostPlatform, TargetPlatforms, CurrentFeaturePacks, CurrentTemplates));
// Copy the install to the output directory
string LocalOutputDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "Rocket", CommandUtils.GetGenericPlatformName(HostPlatform));
bp.AddNode(new GatherRocketNode(HostPlatform, TargetPlatforms, LocalOutputDir));
// Add the aggregate node for the entire install
GUBP.GUBPNode PromotableNode = bp.FindNode(GUBP.SharedAggregatePromotableNode.StaticGetFullName());
PromotableNode.AddDependency(FilterRocketNode.StaticGetFullName(HostPlatform));
PromotableNode.AddDependency(BuildDerivedDataCacheNode.StaticGetFullName(HostPlatform));
// Add a node for GitHub promotions
if(HostPlatform == UnrealTargetPlatform.Win64)
{
string GitConfigRelativePath = "Engine/Build/Git/UnrealBot.ini";
if(CommandUtils.FileExists(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, GitConfigRelativePath)))
{
bp.AddNode(new BuildGitPromotable(bp, HostPlatform, GitConfigRelativePath));
PromotableNode.AddDependency(BuildGitPromotable.StaticGetFullName(HostPlatform));
}
}
// Get the output directory for the build zips
string PublishedEngineDir;
if(ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP())
{
PublishedEngineDir = CommandUtils.CombinePaths(CommandUtils.RootSharedTempStorageDirectory(), "Rocket", "Automated", GetBuildLabel(), CommandUtils.GetGenericPlatformName(HostPlatform));
}
else
{
PublishedEngineDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketPublish", CommandUtils.GetGenericPlatformName(HostPlatform));
}
// Publish the install to the network
bp.AddNode(new PublishRocketNode(HostPlatform, LocalOutputDir, PublishedEngineDir));
bp.AddNode(new PublishRocketSymbolsNode(bp, HostPlatform, TargetPlatforms, PublishedEngineDir + "Symbols"));
// Add a dependency on this being published as part of the shared promotable being labeled
GUBP.SharedLabelPromotableSuccessNode LabelPromotableNode = (GUBP.SharedLabelPromotableSuccessNode)bp.FindNode(GUBP.SharedLabelPromotableSuccessNode.StaticGetFullName());
LabelPromotableNode.AddDependency(PublishRocketNode.StaticGetFullName(HostPlatform));
LabelPromotableNode.AddDependency(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform));
// Add dependencies on a promotable to do these steps too
GUBP.WaitForSharedPromotionUserInput WaitForPromotionNode = (GUBP.WaitForSharedPromotionUserInput)bp.FindNode(GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(true));
WaitForPromotionNode.AddDependency(PublishRocketNode.StaticGetFullName(HostPlatform));
WaitForPromotionNode.AddDependency(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform));
// Push everything behind the promotion triggers if we're doing things on the build machines
if(ShouldDoSeriousThingsLikeP4CheckinAndPostToMCP() || bp.ParseParam("WithRocketPromotable"))
{
string WaitForTrigger = GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(false);
GatherRocketNode GatherRocket = (GatherRocketNode)bp.FindNode(GatherRocketNode.StaticGetFullName(HostPlatform));
GatherRocket.AddDependency(WaitForTrigger);
PublishRocketSymbolsNode PublishRocketSymbols = (PublishRocketSymbolsNode)bp.FindNode(PublishRocketSymbolsNode.StaticGetFullName(HostPlatform));
PublishRocketSymbols.AddDependency(WaitForTrigger);
}
}
}
public override void DoBuild(GUBP bp)
{
// Create a filter for all the promoted binaries
FileFilter PromotableFilter = new FileFilter();
PromotableFilter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
PromotableFilter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, ConfigRelativePath), "promotable");
PromotableFilter.ExcludeConfidentialFolders();
// Copy everything that matches the filter to the promotion folder
string PromotableFolder = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "GitPromotable");
CommandUtils.DeleteDirectoryContents(PromotableFolder);
string[] PromotableFiles = CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, PromotableFolder, PromotableFilter, bIgnoreSymlinks: true);
BuildProducts = new List<string>(PromotableFiles);
}
public TriggerNodeDefinition(GUBP InOwner, GUBP.WaitForUserInput InNode) : base(InOwner, InNode)
{
Node = InNode;
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
FileFilter Filter = new FileFilter();
// Include all the editor products
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.RootEditorNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
// Include win64 tools on Mac, to get the win64 build of UBT, UAT and IPP
if (HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
}
// Include the editor headers
UnzipAndAddRuleForHeaders(GUBP.RootEditorNode.StaticGetArchivedHeadersPath(HostPlatform), Filter, FileFilterType.Include);
// Include the build dependencies for every code platform
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if(RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform))
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
string FileListPath = GUBP.GamePlatformMonolithicsNode.StaticGetBuildDependenciesPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform);
Filter.AddRuleForFiles(UnrealBuildTool.Utils.ReadClass<UnrealBuildTool.ExternalFileList>(FileListPath).FileNames, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
UnzipAndAddRuleForHeaders(GUBP.GamePlatformMonolithicsNode.StaticGetArchivedHeadersPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform), Filter, FileFilterType.Include);
}
}
// Add the monolithic binaries
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
bool bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(SourceHostPlatform, TargetPlatform);
AddRuleForBuildProducts(Filter, bp, GUBP.GamePlatformMonolithicsNode.StaticGetFullName(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, Precompiled: bIsCodeTargetPlatform), FileFilterType.Include);
}
// Include the feature packs
foreach(string CurrentFeaturePack in CurrentFeaturePacks)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(CurrentFeaturePack);
Filter.AddRuleForFile(GUBP.MakeFeaturePacksNode.GetOutputFile(Project), CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
}
// Include all the templates
foreach (string Template in CurrentTemplates)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(Template);
Filter.Include("/" + Utils.StripBaseDirectory(Path.GetDirectoryName(Project.FilePath), CommandUtils.CmdEnv.LocalRoot).Replace('\\', '/') + "/...");
}
// Include all the standard rules
string RulesFileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini");
Filter.ReadRulesFromFile(RulesFileName, "CopyEditor", HostPlatform.ToString());
Filter.ReadRulesFromFile(RulesFileName, "CopyTargetPlatforms", HostPlatform.ToString());
// Custom rules for each target platform
foreach(UnrealTargetPlatform TargetPlaform in TargetPlatforms)
{
string SectionName = String.Format("CopyTargetPlatform.{0}", TargetPlaform.ToString());
Filter.ReadRulesFromFile(RulesFileName, SectionName, HostPlatform.ToString());
}
// Add the final exclusions for legal reasons.
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
// Run the filter on the stripped symbols, and remove those files from the copy filter
List<string> AllStrippedFiles = new List<string>();
foreach(KeyValuePair<string, string> StrippedNodeManifestPath in StrippedNodeManifestPaths)
{
List<string> StrippedFiles = new List<string>();
StripRocketNode StripNode = (StripRocketNode)bp.FindNode(StrippedNodeManifestPath.Key);
foreach(string BuildProduct in StripNode.BuildProducts)
{
if(Utils.IsFileUnderDirectory(BuildProduct, StripNode.StrippedDir))
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(BuildProduct), StripNode.StrippedDir);
if(Filter.Matches(RelativePath))
{
StrippedFiles.Add(RelativePath);
AllStrippedFiles.Add(RelativePath);
Filter.Exclude("/" + RelativePath);
}
}
}
WriteManifest(StrippedNodeManifestPath.Value, StrippedFiles);
BuildProducts.Add(StrippedNodeManifestPath.Value);
}
// Write the filtered list of depot files to disk, removing any symlinks
List<string> DepotFiles = Filter.ApplyToDirectory(CommandUtils.CmdEnv.LocalRoot, true).ToList();
WriteManifest(DepotManifestPath, DepotFiles);
BuildProducts.Add(DepotManifestPath);
// Sort the list of output files
SortedDictionary<string, bool> SortedFiles = new SortedDictionary<string,bool>(StringComparer.InvariantCultureIgnoreCase);
foreach(string DepotFile in DepotFiles)
{
SortedFiles.Add(DepotFile, false);
}
foreach(string StrippedFile in AllStrippedFiles)
{
SortedFiles.Add(StrippedFile, true);
}
// Write the list to the log
CommandUtils.Log("Files to be included in Rocket build:");
foreach(KeyValuePair<string, bool> SortedFile in SortedFiles)
{
CommandUtils.Log(" {0}{1}", SortedFile.Key, SortedFile.Value? " (stripped)" : "");
}
}
public override UE4Build.BuildAgenda GetAgenda(GUBP bp)
{
bool bAnyAdded = false;
var Agenda = new UE4Build.BuildAgenda();
if (HostPlatform == UnrealTargetPlatform.Win64)
{
bAnyAdded = true;
Agenda.DotNetProjects.AddRange(
new string[]
{
CombinePaths(@"Engine\Source\Programs\NotForLicensees\CrashReportServer\CrashReportCommon\CrashReportCommon.csproj"),
CombinePaths(@"Engine\Source\Programs\NotForLicensees\CrashReportServer\CrashReportReceiver\CrashReportReceiver.csproj"),
CombinePaths(@"Engine\Source\Programs\NotForLicensees\CrashReportServer\CrashReportProcess\CrashReportProcess.csproj"),
CombinePaths(@"Engine\Source\Programs\CrashReporter\RegisterPII\RegisterPII.csproj"),
});
Agenda.DotNetSolutions.AddRange(
new string[]
{
CombinePaths(@"Engine\Source\Programs\UnrealDocTool\UnrealDocTool\UnrealDocTool.sln"),
}
);
Agenda.ExtraDotNetFiles.AddRange(
new string[]
{
"Interop.IWshRuntimeLibrary",
"UnrealMarkdown",
"CommonUnrealMarkdown",
}
);
}
string AddArgs = "-nobuilduht -skipactionhistory -CopyAppBundleBackToDevice";
foreach (var ProgramTarget in BranchConfig.Branch.BaseEngineProject.Properties.Programs)
{
bool bInternalOnly;
bool SeparateNode;
bool CrossCompile;
if (ProgramTarget.Rules.GUBP_AlwaysBuildWithTools(HostPlatform, out bInternalOnly, out SeparateNode, out CrossCompile) && ProgramTarget.Rules.SupportsPlatform(HostPlatform) && bInternalOnly && !SeparateNode)
{
foreach (var Plat in ProgramTarget.Rules.GUBP_ToolPlatforms(HostPlatform))
{
foreach (var Config in ProgramTarget.Rules.GUBP_ToolConfigs(HostPlatform))
{
Agenda.AddTargets(new string[] { ProgramTarget.TargetName }, Plat, Config, InAddArgs: AddArgs);
bAnyAdded = true;
}
}
}
}
if (bAnyAdded)
{
return Agenda;
}
return null;
}
public override void DoBuild(GUBP bp)
{
CommandUtils.DeleteDirectoryContents(OutputDir);
// Extract the editor headers
CommandUtils.UnzipFiles(GUBP.RootEditorNode.StaticGetArchivedHeadersPath(HostPlatform), CommandUtils.CmdEnv.LocalRoot);
// Extract all the headers for code target platforms
foreach(UnrealTargetPlatform CodeTargetPlatform in CodeTargetPlatforms)
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, CodeTargetPlatform);
string ZipFileName = GUBP.GamePlatformMonolithicsNode.StaticGetArchivedHeadersPath(SourceHostPlatform, bp.Branch.BaseEngineProject, CodeTargetPlatform);
CommandUtils.UnzipFiles(ZipFileName, CommandUtils.CmdEnv.LocalRoot);
}
// Copy the depot files to the output directory
FilterRocketNode FilterNode = (FilterRocketNode)bp.FindNode(FilterRocketNode.StaticGetFullName(HostPlatform));
CopyManifestFilesToOutput(FilterNode.DepotManifestPath, CommandUtils.CmdEnv.LocalRoot, OutputDir);
// Copy the stripped files to the output directory
foreach(KeyValuePair<string, string> StrippedManifestPath in FilterNode.StrippedNodeManifestPaths)
{
StripRocketNode StripNode = (StripRocketNode)bp.FindNode(StrippedManifestPath.Key);
CopyManifestFilesToOutput(StrippedManifestPath.Value, StripNode.StrippedDir, OutputDir);
}
// Copy the DDC to the output directory
BuildDerivedDataCacheNode DerivedDataCacheNode = (BuildDerivedDataCacheNode)bp.FindNode(BuildDerivedDataCacheNode.StaticGetFullName(HostPlatform));
CopyManifestFilesToOutput(DerivedDataCacheNode.SavedManifestPath, DerivedDataCacheNode.SavedDir, OutputDir);
// Write the Rocket.txt file with the
string RocketFile = CommandUtils.CombinePaths(OutputDir, "Engine/Build/Rocket.txt");
CommandUtils.WriteAllText(RocketFile, "-installedengine -rocket");
// Create a dummy build product
BuildProducts = new List<string>();
SaveRecordOfSuccessAndAddToBuildProducts();
}
public SingleInternalToolsNode(GUBP.GUBPBranchConfig InBranchConfig, UnrealTargetPlatform InHostPlatform, SingleTargetProperties InProgramTarget)
: base(InBranchConfig, InHostPlatform)
{
SetupSingleInternalToolsNode(InProgramTarget);
}
public override void DoBuild(GUBP bp)
{
CommandUtils.CreateDirectory(SavedDir);
BuildProducts = new List<string>();
List<string> ManifestFiles = new List<string>();
if(!bp.ParseParam("NoDDC"))
{
// Find all the projects we're interested in
List<BranchInfo.BranchUProject> Projects = new List<BranchInfo.BranchUProject>();
foreach(string ProjectName in ProjectNames)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(ProjectName);
if(!Project.Properties.bIsCodeBasedProject)
{
Projects.Add(Project);
}
}
// Filter out the files we need to build DDC. Removing confidential folders can affect DDC keys, so we want to be sure that we're making DDC with a build that can use it.
FileFilter Filter = new FileFilter(FileFilterType.Exclude);
Filter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
Filter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini"), "CopyEditor", HostPlatform.ToString());
Filter.Exclude("/Engine/Build/...");
Filter.Exclude("/Engine/Extras/...");
Filter.Exclude("/Engine/DerivedDataCache/...");
Filter.Exclude("/Samples/...");
Filter.Exclude("/Templates/...");
Filter.Exclude(".../Source/...");
Filter.Exclude(".../Intermediate/...");
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
Filter.Include("/Engine/Build/NotForLicensees/EpicInternal.txt");
Filter.Include("/Engine/Binaries/.../*DDCUtils*"); // Make sure we can use the shared DDC!
// Copy everything to a temporary directory
string TempDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketDDC", CommandUtils.GetGenericPlatformName(HostPlatform));
CommandUtils.DeleteDirectoryContents(TempDir);
CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, TempDir, Filter, true);
// Get paths to everything within the temporary directory
string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform);
string RelativePakPath = "Engine/DerivedDataCache/Compressed.ddp";
string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath);
string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv");
// Generate DDC for all the non-code projects. We don't necessarily have editor DLLs for the code projects, but they should be the same as their blueprint counterparts.
List<string> ProjectPakFiles = new List<string>();
foreach(BranchInfo.BranchUProject Project in Projects)
{
CommandUtils.Log("Generating DDC data for {0} on {1}", Project.GameName, TargetPlatforms);
CommandUtils.DDCCommandlet(Project.FilePath, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak -ProjectOnly");
string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", Project.GameName));
CommandUtils.DeleteFile(ProjectPakFile);
CommandUtils.RenameFile(OutputPakFile, ProjectPakFile);
string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv");
CommandUtils.DeleteFile(ProjectCsvFile);
CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile);
ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile));
}
// Generate DDC for the editor, and merge all the other PAK files in
CommandUtils.Log("Generating DDC data for engine content on {0}", TargetPlatforms);
CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak " + CommandUtils.MakePathSafeToUseWithCommandLine("-MergePaks=" + String.Join("+", ProjectPakFiles)));
// Copy the DDP file to the output path
string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath);
CommandUtils.CopyFile(OutputPakFile, SavedPakFile);
BuildProducts.Add(SavedPakFile);
// Add the pak file to the list of files to copy
ManifestFiles.Add(RelativePakPath);
}
CommandUtils.WriteAllLines(SavedManifestPath, ManifestFiles.ToArray());
BuildProducts.Add(SavedManifestPath);
SaveRecordOfSuccessAndAddToBuildProducts();
}
public SingleInternalToolsNode(GUBP bp, GUBPBranchConfig InBranchConfig, UnrealTargetPlatform InHostPlatform, SingleTargetProperties InProgramTarget)
: base(InBranchConfig, InHostPlatform)
{
// Don't add rooteditor dependency if it isn't in the graph
var bRootEditorNodeDoesExit = BranchConfig.HasNode(RootEditorNode.StaticGetFullName(HostPlatform));
SetupSingleInternalToolsNode(InProgramTarget, !bRootEditorNodeDoesExit && BranchConfig.BranchOptions.bNoEditorDependenciesForTools);
}
public PublishRocketSymbolsNode(GUBP bp, UnrealTargetPlatform HostPlatform, IEnumerable<UnrealTargetPlatform> TargetPlatforms, string InSymbolsOutputDir)
: base(HostPlatform)
{
SymbolsOutputDir = InSymbolsOutputDir;
AddDependency(GUBP.WaitForSharedPromotionUserInput.StaticGetFullName(false));
AddDependency(GUBP.ToolsForCompileNode.StaticGetFullName(HostPlatform));
AddDependency(GUBP.RootEditorNode.StaticGetFullName(HostPlatform));
AddDependency(GUBP.ToolsNode.StaticGetFullName(HostPlatform));
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if(HostPlatform == RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform))
{
bool bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform);
AddDependency(GUBP.GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, Precompiled: bIsCodeTargetPlatform));
}
}
AgentSharingGroup = "RocketGroup" + StaticGetHostPlatformSuffix(HostPlatform);
}
public override int CISFrequencyQuantumShift(GUBP.GUBPBranchConfig BranchConfig)
{
int Result = base.CISFrequencyQuantumShift(BranchConfig) + 1;
return Result;
}
public static UnrealTargetPlatform GetSourceHostPlatform(GUBP bp, UnrealTargetPlatform HostPlatform, UnrealTargetPlatform TargetPlatform)
{
if (TargetPlatform == UnrealTargetPlatform.HTML5 && HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
return UnrealTargetPlatform.Win64;
}
if (TargetPlatform == UnrealTargetPlatform.Android && HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
return UnrealTargetPlatform.Win64;
}
if(TargetPlatform == UnrealTargetPlatform.IOS && HostPlatform == UnrealTargetPlatform.Win64 && bp.HostPlatforms.Contains(UnrealTargetPlatform.Mac))
{
return UnrealTargetPlatform.Mac;
}
return HostPlatform;
}
public override void DoBuild(GUBP bp)
{
ExecuteApiDocTool("-rebuildblueprint", "APIDocTool-Blueprint");
SubmitOutputs("Blueprint documentation", "Engine/Documentation/Builds/BlueprintAPI-*", "Engine/Documentation/CHM/BlueprintAPI.chm");
base.DoBuild(bp);
}
void PrintNodes(GUBP bp, List<BuildNode> Nodes, IEnumerable<AggregateNode> Aggregates, List<TriggerNode> UnfinishedTriggers, int TimeQuantum)
{
AggregateNode[] MatchingAggregates = Aggregates.Where(x => x.Dependencies.All(y => Nodes.Contains(y))).ToArray();
if (MatchingAggregates.Length > 0)
{
Log("*********** Aggregates");
foreach (AggregateNode Aggregate in MatchingAggregates.OrderBy(x => x.Name))
{
StringBuilder Note = new StringBuilder(" " + Aggregate.Name);
if (Aggregate.Node.IsPromotableAggregate())
{
Note.Append(" (promotable)");
}
Log(Note.ToString());
}
}
bool bShowDependencies = bp.ParseParam("ShowDependencies");
bool AddEmailProps = bp.ParseParam("ShowEmails");
string LastControllingTrigger = "";
string LastAgentGroup = "";
Log("*********** Desired And Dependent Nodes, in order.");
foreach (BuildNode NodeToDo in Nodes)
{
string MyControllingTrigger = NodeToDo.ControllingTriggerDotName;
if (MyControllingTrigger != LastControllingTrigger)
{
LastControllingTrigger = MyControllingTrigger;
if (MyControllingTrigger != "")
{
string Finished = "";
if (UnfinishedTriggers != null)
{
if (NodeToDo.ControllingTriggers.Length > 0 && UnfinishedTriggers.Contains(NodeToDo.ControllingTriggers.Last()))
{
Finished = "(not yet triggered)";
}
else
{
Finished = "(already triggered)";
}
}
Log(" Controlling Trigger: {0} {1}", MyControllingTrigger, Finished);
}
}
if (NodeToDo.AgentSharingGroup != LastAgentGroup && NodeToDo.AgentSharingGroup != "")
{
Log(" Agent Group: {0}", NodeToDo.AgentSharingGroup);
}
LastAgentGroup = NodeToDo.AgentSharingGroup;
StringBuilder Builder = new StringBuilder(" ");
if(LastAgentGroup != "")
{
Builder.Append(" ");
}
Builder.AppendFormat("{0} ({1})", NodeToDo.Name, GetTimeIntervalString(TimeQuantum << NodeToDo.FrequencyShift));
if(NodeToDo.IsComplete)
{
Builder.Append(" - (Completed)");
}
if(NodeToDo is TriggerNode)
{
Builder.Append(" - (TriggerNode)");
}
if(NodeToDo.IsSticky)
{
Builder.Append(" - (Sticky)");
}
string Agent = NodeToDo.AgentRequirements;
if(ParseParamValue("AgentOverride") != "" && !NodeToDo.Node.GetFullName().Contains("Mac"))
{
Agent = ParseParamValue("AgentOverride");
}
if (!String.IsNullOrEmpty(Agent))
{
Builder.AppendFormat(" [{0}]", Agent);
}
if(NodeToDo.AgentMemoryRequirement != 0)
{
Builder.AppendFormat(" [{0}gb]", NodeToDo.AgentMemoryRequirement);
}
if (AddEmailProps)
{
Builder.AppendFormat(" {0}", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
}
Log(Builder.ToString());
if (bShowDependencies)
{
foreach (BuildNode Dep in NodeToDo.Dependencies)
{
Log(" dep> {0}", Dep.Name);
}
foreach (BuildNode Dep in NodeToDo.PseudoDependencies)
{
Log(" pdep> {0}", Dep.Name);
}
}
}
}
public ToolsForDocumentationNode(GUBP.GUBPBranchConfig InBranchConfig, UnrealTargetPlatform InHostPlatform)
: base(InBranchConfig, InHostPlatform, false)
{
AgentSharingGroup = "Documentation" + StaticGetHostPlatformSuffix(InHostPlatform);
}
