public virtual void RetrieveBuildProducts(string SharedStorageDir)
{
CommandUtils.Log("***** Retrieving GUBP Node {0}", Name);
try
{
BuildProducts = TempStorage.RetrieveFromTempStorage(SharedStorageDir, Name);
}
catch (Exception Ex)
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", Name);
}
}
public virtual void RetrieveBuildProducts(TempStorageNodeInfo TempStorageNodeInfo)
{
CommandUtils.Log("***** Retrieving GUBP Node {0} from {1}", Name, TempStorageNodeInfo.GetRelativeDirectory());
try
{
BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, CommandUtils.CmdEnv.LocalRoot);
}
catch (Exception Ex)
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", Name);
}
}
/// <summary>
/// Find all the nodes in the graph which are already completed
/// </summary>
/// <param name="Graph">The graph instance</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
HashSet <Node> FindCompletedNodes(Graph Graph, TempStorage Storage)
{
HashSet <Node> CompletedNodes = new HashSet <Node>();
foreach (Node Node in Graph.Agents.SelectMany(x => x.Nodes))
{
if (Storage.IsComplete(Node.Name))
{
CompletedNodes.Add(Node);
}
}
return(CompletedNodes);
}
/// <summary>
/// Builds all the nodes in the graph
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph instance</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <returns>True if everything built successfully</returns>
bool BuildAllNodes(JobContext Job, Graph Graph, TempStorage Storage)
{
// Build a flat list of nodes to execute, in order
Node[] NodesToExecute = Graph.Agents.SelectMany(x => x.Nodes).ToArray();
// Check the integrity of any local nodes that have been completed. It's common to run formal builds locally between regular development builds, so we may have
// stale local state. Rather than failing later, detect and clean them up now.
HashSet <Node> CleanedNodes = new HashSet <Node>();
foreach (Node NodeToExecute in NodesToExecute)
{
if (NodeToExecute.InputDependencies.Any(x => CleanedNodes.Contains(x)) || !Storage.CheckLocalIntegrity(NodeToExecute.Name, NodeToExecute.Outputs.Select(x => x.TagName)))
{
Storage.CleanLocalNode(NodeToExecute.Name);
CleanedNodes.Add(NodeToExecute);
}
}
// Execute them in order
int NodeIdx = 0;
foreach (Node NodeToExecute in NodesToExecute)
{
LogInformation("****** [{0}/{1}] {2}", ++NodeIdx, NodesToExecute.Length, NodeToExecute.Name);
if (!Storage.IsComplete(NodeToExecute.Name))
{
LogInformation("");
if (!BuildNode(Job, Graph, NodeToExecute, Storage, false))
{
return(false);
}
LogInformation("");
}
}
return(true);
}
public void SaveStatus(BuildNode NodeToDo, string Suffix, string NodeStoreName, bool bSaveSharedTempStorage, string GameNameIfAny, string JobStepIDForFailure = null)
{
string Contents = "Just a status record: " + Suffix;
if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine)
{
try
{
Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true);
}
catch (Exception Ex)
{
CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to get properties for jobstep to save them.");
CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex));
}
}
string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", NodeToDo.Name + Suffix + ".log");
CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess));
CommandUtils.WriteAllText(RecordOfSuccess, Contents);
TempStorage.StoreToTempStorage(CommandUtils.CmdEnv, NodeStoreName + Suffix, new List <string> {
RecordOfSuccess
}, !bSaveSharedTempStorage, GameNameIfAny);
}
public void SaveStatus(TempStorageNodeInfo TempStorageNodeInfo, bool bSaveSharedTempStorage, string JobStepIDForFailure = null)
{
string Contents = "Just a status record: " + TempStorageNodeInfo.NodeStorageName;
if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine)
{
try
{
Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true);
}
catch (Exception Ex)
{
CommandUtils.LogWarning("Failed to get properties for jobstep to save them.");
CommandUtils.LogWarning(LogUtils.FormatException(Ex));
}
}
string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", TempStorageNodeInfo.NodeStorageName + ".log");
CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess));
CommandUtils.WriteAllText(RecordOfSuccess, Contents);
TempStorage.StoreToTempStorage(TempStorageNodeInfo, new List <string> {
RecordOfSuccess
}, !bSaveSharedTempStorage, CommandUtils.CmdEnv.LocalRoot);
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
if (ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
string TargetNames = ParseParamValue("Target", null);
if (TargetNames == null)
{
LogError("Missing -Target= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string[] TriggerNames = ParseParamValue("Trigger", "").Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
bool bClean = ParseParam("Clean");
bool bListOnly = ParseParam("ListOnly");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
GraphPrintOptions PrintOptions = 0;
if (ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if (ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List <string> CleanNodes = new List <string>();
foreach (string NodeList in ParseParamValues("CleanNode"))
{
foreach (string NodeName in NodeList.Split('+'))
{
CleanNodes.Add(NodeName);
}
}
// Read any environment variables
Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (DictionaryEntry Entry in Environment.GetEnvironmentVariables())
{
DefaultProperties[Entry.Key.ToString()] = Entry.Value.ToString();
}
// Add any additional custom parameters from the command line (of the form -Set:X=Y)
foreach (string Param in Params)
{
const string Prefix = "set:";
if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if (EqualsIdx >= 0)
{
DefaultProperties[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Set up the standard properties which build scripts might need
DefaultProperties["Branch"] = P4Enabled? P4Env.BuildRootP4 : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled? P4Env.BuildRootEscaped : "Unknown";
DefaultProperties["Change"] = P4Enabled? P4Env.Changelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
// Find all the tasks from the loaded assemblies
Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();
if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return(ExitCode.Error_Unknown);
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if (SchemaFileName != null)
{
Schema.Export(new FileReference(SchemaFileName));
}
// Read the script from disk
Graph Graph;
if (!ScriptReader.TryRead(new FileReference(ScriptFileName), DefaultProperties, Schema, out Graph))
{
return(ExitCode.Error_Unknown);
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if (bClean)
{
Storage.CleanLocal();
}
foreach (string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet <Node> TargetNodes = new HashSet <Node>();
foreach (string TargetName in TargetNames.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if (!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Nodes);
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Find the triggers which are explicitly activated, and all of its upstream triggers.
HashSet <ManualTrigger> Triggers = new HashSet <ManualTrigger>();
foreach (string TriggerName in TriggerNames)
{
ManualTrigger Trigger;
if (!Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
while (Trigger != null)
{
Triggers.Add(Trigger);
Trigger = Trigger.Parent;
}
}
if (bSkipTriggers)
{
Triggers.UnionWith(Graph.NameToTrigger.Values);
}
// If we're just building a single node, find it
Node SingleNode = null;
if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return(ExitCode.Error_Unknown);
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system.
if (SingleNode == null)
{
IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == null || Triggers.Contains(x.EnclosingTrigger));
foreach (GraphDiagnostic Diagnostic in Diagnostics)
{
if (Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return(ExitCode.Error_Unknown);
}
}
// Execute the command
if (bListOnly)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
}
else if (ExportFileName != null)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Triggers, CompletedNodes);
}
else if (SingleNode != null)
{
if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return(ExitCode.Error_Unknown);
}
}
else
{
if (!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return(ExitCode.Error_Unknown);
}
}
return(ExitCode.Success);
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
// Create a mapping from tag name to the files it contains, and seed it with invalid entries for everything in the graph
Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();
foreach (NodeOutput Output in Graph.Groups.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs))
{
TagNameToFileSet[Output.Name] = null;
}
// Fetch all the input dependencies for this node, and fill in the tag names with those files
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
foreach (NodeOutput Input in Node.Inputs)
{
TempStorageManifest Manifest = Storage.Retreive(Input.ProducingNode.Name, Input.Name);
TagNameToFileSet[Input.Name] = new HashSet <FileReference>(Manifest.Files.Select(x => x.ToFileReference(RootDir)));
}
// Add placeholder outputs for the current node
foreach (NodeOutput Output in Node.Outputs)
{
TagNameToFileSet[Output.Name] = new HashSet <FileReference>();
}
// Execute the node
if (bWithBanner)
{
Console.WriteLine();
CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
}
if (!Node.Build(Job, TagNameToFileSet))
{
return(false);
}
if (bWithBanner)
{
CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Determine all the outputs which are required to be copied to temp storage (because they're referenced by nodes in another agent group)
HashSet <NodeOutput> ReferencedOutputs = new HashSet <NodeOutput>();
foreach (AgentGroup Group in Graph.Groups)
{
bool bSameGroup = Group.Nodes.Contains(Node);
foreach (Node OtherNode in Group.Nodes)
{
if (!bSameGroup || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
ReferencedOutputs.UnionWith(OtherNode.Inputs);
}
}
}
// Publish all the outputs
foreach (NodeOutput Output in Node.Outputs)
{
Storage.Archive(Node.Name, Output.Name, TagNameToFileSet[Output.Name].ToArray(), ReferencedOutputs.Contains(Output));
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return(true);
}
/// <summary>
/// Builds all the nodes in the graph
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph instance</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <returns>True if everything built successfully</returns>
bool BuildAllNodes(JobContext Job, Graph Graph, TempStorage Storage)
{
// Build a flat list of nodes to execute, in order
Node[] NodesToExecute = Graph.Agents.SelectMany(x => x.Nodes).ToArray();
// Check the integrity of any local nodes that have been completed. It's common to run formal builds locally between regular development builds, so we may have
// stale local state. Rather than failing later, detect and clean them up now.
HashSet<Node> CleanedNodes = new HashSet<Node>();
foreach(Node NodeToExecute in NodesToExecute)
{
if(NodeToExecute.InputDependencies.Any(x => CleanedNodes.Contains(x)) || !Storage.CheckLocalIntegrity(NodeToExecute.Name, NodeToExecute.Outputs.Select(x => x.TagName)))
{
Storage.CleanLocalNode(NodeToExecute.Name);
CleanedNodes.Add(NodeToExecute);
}
}
// Execute them in order
int NodeIdx = 0;
foreach(Node NodeToExecute in NodesToExecute)
{
Log("****** [{0}/{1}] {2}", ++NodeIdx, NodesToExecute.Length, NodeToExecute.Name);
if(!Storage.IsComplete(NodeToExecute.Name))
{
Log("");
if(!BuildNode(Job, Graph, NodeToExecute, Storage, false))
{
return false;
}
Log("");
}
}
return true;
}
/// <summary>
/// Run the automated tests
/// </summary>
public override void ExecuteBuild()
{
// Get all the shared directories
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local");
CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared");
CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName);
CommandUtils.DeleteDirectoryContents(SharedDir.FullName);
DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working");
CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
// Create the temp storage object
TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true);
// Create a working directory, and copy some source files into it
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime");
if(!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true))
{
throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName);
}
// Save the default output
Dictionary<FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f');
TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false);
Dictionary<FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i');
TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true);
// Check both outputs are still ok
TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null);
CheckManifest(WorkingDir, DefaultManifest, DefaultOutput);
TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifest, NamedOutput);
// Delete local temp storage and the working directory and try again
CommandUtils.Log("Clearing local folders...");
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
// First output should fail
CommandUtils.Log("Checking default manifest is now unavailable...");
bool bGotManifest = false;
try
{
TempStore.Retreive("TestNode", null);
}
catch
{
bGotManifest = false;
}
if(bGotManifest)
{
throw new AutomationException("Did not expect shared temp storage manifest to exist");
}
// Second one should be fine
TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput);
}
public virtual void ArchiveBuildProducts(string SharedStorageDir, bool bWriteToSharedStorage)
{
TempStorage.StoreToTempStorage(Name, BuildProducts, SharedStorageDir, bWriteToSharedStorage);
}
/// <summary>
/// Run the automated tests
/// </summary>
public override void ExecuteBuild()
{
// Get all the shared directories
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local");
CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared");
CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName);
CommandUtils.DeleteDirectoryContents(SharedDir.FullName);
DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working");
CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
// Create the temp storage object
TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true);
// Create a working directory, and copy some source files into it
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime");
if (!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true))
{
throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName);
}
// Save the default output
Dictionary <FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f');
TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false);
Dictionary <FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i');
TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true);
// Check both outputs are still ok
TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null);
CheckManifest(WorkingDir, DefaultManifest, DefaultOutput);
TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifest, NamedOutput);
// Delete local temp storage and the working directory and try again
CommandUtils.Log("Clearing local folders...");
CommandUtils.DeleteDirectoryContents(WorkingDir.FullName);
CommandUtils.DeleteDirectoryContents(LocalDir.FullName);
// First output should fail
CommandUtils.Log("Checking default manifest is now unavailable...");
bool bGotManifest = false;
try
{
TempStore.Retreive("TestNode", null);
}
catch
{
bGotManifest = false;
}
if (bGotManifest)
{
throw new AutomationException("Did not expect shared temp storage manifest to exist");
}
// Second one should be fine
TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput");
CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput);
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bClearHistory = ParseParam("Clean") || ParseParam("ClearHistory");
bool bListOnly = ParseParam("ListOnly");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if(ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if(ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List<string> CleanNodes = new List<string>();
foreach(string NodeList in ParseParamValues("CleanNode"))
{
foreach(string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary<string, string> DefaultProperties = new Dictionary<string,string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.BuildRootP4 : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? P4Env.BuildRootEscaped : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Build", "Build.version").FullName, out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary<string, string> Arguments = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if(Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if(EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary<string, ScriptTask> NameToTask = new Dictionary<string,ScriptTask>();
if(!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return ExitCode.Error_Unknown;
}
// Generate documentation
if(DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return ExitCode.Success;
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if(SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
Log("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if(ScriptFileName == null)
{
return ExitCode.Success;
}
}
// Check there was a script specified
if(ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return ExitCode.Error_Unknown;
}
// Read the script from disk
Graph Graph;
if(!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return ExitCode.Error_Unknown;
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if(bClearHistory)
{
Storage.CleanLocal();
}
foreach(string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet<Node> TargetNodes = new HashSet<Node>();
if(TargetNames == null)
{
if(!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return ExitCode.Error_Unknown;
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach(string TargetName in TargetNames.Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if(!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return ExitCode.Error_Unknown;
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if(TokenSignature != null)
{
// Find all the lock files
HashSet<FileReference> RequiredTokens = new HashSet<FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if(SingleNodeName == null)
{
CommandUtils.Log("Required tokens:");
foreach(Node Node in TargetNodes)
{
foreach(FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.Log(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List<FileReference> CreatedTokens = new List<FileReference>();
if(!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary<FileReference, string> MissingTokens = new Dictionary<FileReference, string>();
foreach(FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if(CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if(MissingTokens.Count > 0)
{
if(bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet<Node> SkipNodes = new HashSet<Node>();
foreach(IGrouping<string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach(FileReference MissingToken in MissingTokensForBuild)
{
foreach(Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
Log(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if(SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
Log("Remaining target nodes:");
foreach(Node TargetNode in TargetNodes)
{
Log(" {0}", TargetNode);
}
if(TargetNodes.Count == 0)
{
Log(" None.");
}
}
}
else
{
foreach(KeyValuePair<FileReference, string> Pair in MissingTokens)
{
List<Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach(FileReference CreatedToken in CreatedTokens)
{
CreatedToken.Delete();
}
return ExitCode.Error_Unknown;
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet<ManualTrigger> SkipTriggers = new HashSet<ManualTrigger>();
if(bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach(string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if(!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return ExitCode.Error_Unknown;
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if(TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return ExitCode.Error_Unknown;
}
// If we're just building a single node, find it
Node SingleNode = null;
if(SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return ExitCode.Error_Unknown;
}
// If we just want to show the contents of the graph, do so and exit.
if(bListOnly)
{
HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
return ExitCode.Success;
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if(SingleNode == null)
{
IEnumerable<GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach(GraphDiagnostic Diagnostic in Diagnostics)
{
if(Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if(Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return ExitCode.Error_Unknown;
}
}
// Execute the command
if(ExportFileName != null)
{
HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
}
else if(SingleNode != null)
{
if(!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return ExitCode.Error_Unknown;
}
}
else
{
if(!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return ExitCode.Error_Unknown;
}
}
return ExitCode.Success;
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
// Create the mapping of tag names to file sets
Dictionary<string, HashSet<FileReference>> TagNameToFileSet = new Dictionary<string,HashSet<FileReference>>();
// Read all the input tags for this node, and build a list of referenced input storage blocks
HashSet<TempStorageBlock> InputStorageBlocks = new HashSet<TempStorageBlock>();
foreach(NodeOutput Input in Node.Inputs)
{
TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
InputStorageBlocks.UnionWith(FileList.Blocks);
}
// Read all the input storage blocks, keeping track of which block each file came from
Dictionary<FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary<FileReference, TempStorageBlock>();
foreach(TempStorageBlock InputStorageBlock in InputStorageBlocks)
{
TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
foreach(FileReference File in Manifest.Files.Select(x => x.ToFileReference(RootDir)))
{
TempStorageBlock CurrentStorageBlock;
if(FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock))
{
LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
}
FileToStorageBlock[File] = InputStorageBlock;
}
}
// Add placeholder outputs for the current node
foreach(NodeOutput Output in Node.Outputs)
{
TagNameToFileSet.Add(Output.TagName, new HashSet<FileReference>());
}
// Execute the node
if(bWithBanner)
{
Console.WriteLine();
CommandUtils.Log("========== Starting: {0} ==========", Node.Name);
}
if(!Node.Build(Job, TagNameToFileSet))
{
return false;
}
if(bWithBanner)
{
CommandUtils.Log("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
HashSet<FileReference> ReferencedOutputFiles = new HashSet<FileReference>();
foreach(Agent Agent in Graph.Agents)
{
bool bSameAgent = Agent.Nodes.Contains(Node);
foreach(Node OtherNode in Agent.Nodes)
{
if(!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
foreach(NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
{
ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
}
}
}
}
// Find a block name for all new outputs
Dictionary<FileReference, string> FileToOutputName = new Dictionary<FileReference, string>();
foreach(NodeOutput Output in Node.Outputs)
{
HashSet<FileReference> Files = TagNameToFileSet[Output.TagName];
foreach(FileReference File in Files)
{
if(!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
{
if(Output == Node.DefaultOutput)
{
if(!FileToOutputName.ContainsKey(File))
{
FileToOutputName[File] = "";
}
}
else
{
string OutputName;
if(FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
{
FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
}
else
{
FileToOutputName[File] = Output.TagName.Substring(1);
}
}
}
}
}
// Invert the dictionary to make a mapping of storage block to the files each contains
Dictionary<string, HashSet<FileReference>> OutputStorageBlockToFiles = new Dictionary<string, HashSet<FileReference>>();
foreach(KeyValuePair<FileReference, string> Pair in FileToOutputName)
{
HashSet<FileReference> Files;
if(!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
{
Files = new HashSet<FileReference>();
OutputStorageBlockToFiles.Add(Pair.Value, Files);
}
Files.Add(Pair.Key);
}
// Write all the storage blocks, and update the mapping from file to storage block
foreach(KeyValuePair<string, HashSet<FileReference>> Pair in OutputStorageBlockToFiles)
{
TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
foreach(FileReference File in Pair.Value)
{
FileToStorageBlock.Add(File, OutputBlock);
}
Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
}
// Publish all the output tags
foreach(NodeOutput Output in Node.Outputs)
{
HashSet<FileReference> Files = TagNameToFileSet[Output.TagName];
HashSet<TempStorageBlock> StorageBlocks = new HashSet<TempStorageBlock>();
foreach(FileReference File in Files)
{
TempStorageBlock StorageBlock;
if(FileToStorageBlock.TryGetValue(File, out StorageBlock))
{
StorageBlocks.Add(StorageBlock);
}
}
Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return true;
}
/// <summary>
/// Build a node
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param>
/// <param name="Node">The node to build</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
/// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param>
/// <returns>True if the node built successfully, false otherwise.</returns>
bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner)
{
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
// Create the mapping of tag names to file sets
Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >();
// Read all the input tags for this node, and build a list of referenced input storage blocks
HashSet <TempStorageBlock> InputStorageBlocks = new HashSet <TempStorageBlock>();
foreach (NodeOutput Input in Node.Inputs)
{
TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName);
TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir);
InputStorageBlocks.UnionWith(FileList.Blocks);
}
// Read the manifests for all the input storage blocks
Dictionary <TempStorageBlock, TempStorageManifest> InputManifests = new Dictionary <TempStorageBlock, TempStorageManifest>();
foreach (TempStorageBlock InputStorageBlock in InputStorageBlocks)
{
TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName);
InputManifests[InputStorageBlock] = Manifest;
}
// Read all the input storage blocks, keeping track of which block each file came from
Dictionary <FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary <FileReference, TempStorageBlock>();
foreach (KeyValuePair <TempStorageBlock, TempStorageManifest> Pair in InputManifests)
{
TempStorageBlock InputStorageBlock = Pair.Key;
foreach (FileReference File in Pair.Value.Files.Select(x => x.ToFileReference(RootDir)))
{
TempStorageBlock CurrentStorageBlock;
if (FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock) && !TempStorage.IsDuplicateBuildProduct(File))
{
LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock);
}
FileToStorageBlock[File] = InputStorageBlock;
}
}
// Add placeholder outputs for the current node
foreach (NodeOutput Output in Node.Outputs)
{
TagNameToFileSet.Add(Output.TagName, new HashSet <FileReference>());
}
// Execute the node
if (bWithBanner)
{
Console.WriteLine();
CommandUtils.LogInformation("========== Starting: {0} ==========", Node.Name);
}
if (!Node.Build(Job, TagNameToFileSet))
{
return(false);
}
if (bWithBanner)
{
CommandUtils.LogInformation("========== Finished: {0} ==========", Node.Name);
Console.WriteLine();
}
// Check that none of the inputs have been clobbered
Dictionary <string, string> ModifiedFiles = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (TempStorageFile File in InputManifests.Values.SelectMany(x => x.Files))
{
string Message;
if (!ModifiedFiles.ContainsKey(File.RelativePath) && !File.Compare(CommandUtils.RootDirectory, out Message))
{
ModifiedFiles.Add(File.RelativePath, Message);
}
}
if (ModifiedFiles.Count > 0)
{
throw new AutomationException("Build {0} from a previous step have been modified:\n{1}", (ModifiedFiles.Count == 1)? "product" : "products", String.Join("\n", ModifiedFiles.Select(x => x.Value)));
}
// Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent)
HashSet <FileReference> ReferencedOutputFiles = new HashSet <FileReference>();
foreach (Agent Agent in Graph.Agents)
{
bool bSameAgent = Agent.Nodes.Contains(Node);
foreach (Node OtherNode in Agent.Nodes)
{
if (!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger)
{
foreach (NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node))
{
ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]);
}
}
}
}
// Find a block name for all new outputs
Dictionary <FileReference, string> FileToOutputName = new Dictionary <FileReference, string>();
foreach (NodeOutput Output in Node.Outputs)
{
HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
foreach (FileReference File in Files)
{
if (!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir))
{
if (Output == Node.DefaultOutput)
{
if (!FileToOutputName.ContainsKey(File))
{
FileToOutputName[File] = "";
}
}
else
{
string OutputName;
if (FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0)
{
FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1));
}
else
{
FileToOutputName[File] = Output.TagName.Substring(1);
}
}
}
}
}
// Invert the dictionary to make a mapping of storage block to the files each contains
Dictionary <string, HashSet <FileReference> > OutputStorageBlockToFiles = new Dictionary <string, HashSet <FileReference> >();
foreach (KeyValuePair <FileReference, string> Pair in FileToOutputName)
{
HashSet <FileReference> Files;
if (!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files))
{
Files = new HashSet <FileReference>();
OutputStorageBlockToFiles.Add(Pair.Value, Files);
}
Files.Add(Pair.Key);
}
// Write all the storage blocks, and update the mapping from file to storage block
foreach (KeyValuePair <string, HashSet <FileReference> > Pair in OutputStorageBlockToFiles)
{
TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key);
foreach (FileReference File in Pair.Value)
{
FileToStorageBlock.Add(File, OutputBlock);
}
Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x)));
}
// Publish all the output tags
foreach (NodeOutput Output in Node.Outputs)
{
HashSet <FileReference> Files = TagNameToFileSet[Output.TagName];
HashSet <TempStorageBlock> StorageBlocks = new HashSet <TempStorageBlock>();
foreach (FileReference File in Files)
{
TempStorageBlock StorageBlock;
if (FileToStorageBlock.TryGetValue(File, out StorageBlock))
{
StorageBlocks.Add(StorageBlock);
}
}
Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray());
}
// Mark the node as succeeded
Storage.MarkAsComplete(Node.Name);
return(true);
}
public virtual void ArchiveBuildProducts(TempStorageNodeInfo TempStorageNodeInfo, bool bLocalOnly)
{
TempStorage.StoreToTempStorage(TempStorageNodeInfo, BuildProducts, bLocalOnly, CommandUtils.CmdEnv.LocalRoot);
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bResume = SingleNodeName != null || ParseParam("Resume");
bool bListOnly = ParseParam("ListOnly");
bool bShowDiagnostics = ParseParam("ShowDiagnostics");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if (ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if (ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List <string> CleanNodes = new List <string>();
foreach (string NodeList in ParseParamValues("CleanNode"))
{
foreach (string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.Branch : "Unknown";
DefaultProperties["Depot"] = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
DefaultProperties["RestrictedFolderNames"] = String.Join(";", RestrictedFolders.Names);
DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x)));
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if (EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();
if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return(ExitCode.Error_Unknown);
}
// Generate documentation
if (DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return(ExitCode.Success);
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if (SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if (ScriptFileName == null)
{
return(ExitCode.Success);
}
}
// Check there was a script specified
if (ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
// Read the script from disk
Graph Graph;
if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return(ExitCode.Error_Unknown);
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if (!bResume)
{
Storage.CleanLocal();
}
foreach (string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet <Node> TargetNodes = new HashSet <Node>();
if (TargetNames == null)
{
if (!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if (!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if (TokenSignature != null)
{
// Find all the lock files
HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if (SingleNodeName == null)
{
CommandUtils.LogInformation("Required tokens:");
foreach (Node Node in TargetNodes)
{
foreach (FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.LogInformation(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List <FileReference> CreatedTokens = new List <FileReference>();
if (!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
foreach (FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if (CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if (MissingTokens.Count > 0)
{
if (bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet <Node> SkipNodes = new HashSet <Node>();
foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach (FileReference MissingToken in MissingTokensForBuild)
{
foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
LogInformation(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if (SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
LogInformation("Remaining target nodes:");
foreach (Node TargetNode in TargetNodes)
{
LogInformation(" {0}", TargetNode);
}
if (TargetNodes.Count == 0)
{
LogInformation(" None.");
}
}
}
else
{
foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
{
List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach (FileReference CreatedToken in CreatedTokens)
{
FileReference.Delete(CreatedToken);
}
return(ExitCode.Error_Unknown);
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();
if (bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach (string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName);
LogInformation("Writing {0}...", PreprocessedFileLocation);
Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
return(ExitCode.Success);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
// If we're just building a single node, find it
Node SingleNode = null;
if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return(ExitCode.Error_Unknown);
}
// If we just want to show the contents of the graph, do so and exit.
if (bListOnly)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if (SingleNode == null && (!bListOnly || bShowDiagnostics))
{
IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach (GraphDiagnostic Diagnostic in Diagnostics)
{
if (Diagnostic.EventType == LogEventType.Console)
{
CommandUtils.LogInformation(Diagnostic.Message);
}
else if (Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return(ExitCode.Error_Unknown);
}
}
// Export the graph to a file
if (ExportFileName != null)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
return(ExitCode.Success);
}
// Execute the command
if (!bListOnly)
{
if (SingleNode != null)
{
if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return(ExitCode.Error_Unknown);
}
}
else
{
if (!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return(ExitCode.Error_Unknown);
}
}
}
return(ExitCode.Success);
}
/// <summary>
/// Find all the nodes in the graph which are already completed
/// </summary>
/// <param name="Graph">The graph instance</param>
/// <param name="Storage">The temp storage backend which stores the shared state</param>
HashSet<Node> FindCompletedNodes(Graph Graph, TempStorage Storage)
{
HashSet<Node> CompletedNodes = new HashSet<Node>();
foreach(Node Node in Graph.Agents.SelectMany(x => x.Nodes))
{
if(Storage.IsComplete(Node.Name))
{
CompletedNodes.Add(Node);
}
}
return CompletedNodes;
}