/// <summary>
/// Checks whether this trigger is downstream of another
/// </summary>
/// <param name="Other">The parent trigger to check</param>
/// <returns>True if the trigger is downstream of the given trigger</returns>
public bool IsDownstreamFrom(ManualTrigger Other)
{
for(ManualTrigger Ancestor = Parent; Ancestor != null; Ancestor = Ancestor.Parent)
{
if(Ancestor == Other)
{
return true;
}
}
return false;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="InName">The name of this node</param>
/// <param name="InInputs">Inputs that this node depends on</param>
/// <param name="InOutputNames">Names of the outputs that this node produces</param>
/// <param name="InInputDependencies">Nodes which this node is dependent on for its inputs</param>
/// <param name="InOrderDependencies">Nodes which this node needs to run after. Should include all input dependencies.</param>
/// <param name="InControllingTrigger">The trigger which this node is behind</param>
/// <param name="InRequiredTokens">Optional tokens which must be required for this node to run</param>
public Node(string InName, NodeOutput[] InInputs, string[] InOutputNames, Node[] InInputDependencies, Node[] InOrderDependencies, ManualTrigger InControllingTrigger, FileReference[] InRequiredTokens)
{
Name = InName;
Inputs = InInputs;
List <NodeOutput> AllOutputs = new List <NodeOutput>();
AllOutputs.Add(new NodeOutput(this, "#" + Name));
AllOutputs.AddRange(InOutputNames.Where(x => String.Compare(x, Name, StringComparison.InvariantCultureIgnoreCase) != 0).Select(x => new NodeOutput(this, x)));
Outputs = AllOutputs.ToArray();
InputDependencies = InInputDependencies;
OrderDependencies = InOrderDependencies;
ControllingTrigger = InControllingTrigger;
RequiredTokens = InRequiredTokens;
}
/// <summary>
/// Writes this agent group out to a file, filtering nodes by a controlling trigger
/// </summary>
/// <param name="Writer">The XML writer to output to</param>
/// <param name="ControllingTrigger">The controlling trigger to filter by</param>
public void Write(XmlWriter Writer, ManualTrigger ControllingTrigger)
{
if (Nodes.Any(x => x.ControllingTrigger == ControllingTrigger))
{
Writer.WriteStartElement("Agent");
Writer.WriteAttributeString("Name", Name);
Writer.WriteAttributeString("Type", String.Join(";", PossibleTypes));
foreach (Node Node in Nodes)
{
if (Node.ControllingTrigger == ControllingTrigger)
{
Node.Write(Writer);
}
}
Writer.WriteEndElement();
}
}
/// <summary>
/// Writes this agent group out to a file, filtering nodes by a controlling trigger
/// </summary>
/// <param name="Writer">The XML writer to output to</param>
/// <param name="ControllingTrigger">The controlling trigger to filter by</param>
public void Write(XmlWriter Writer, ManualTrigger ControllingTrigger)
{
if (Nodes.Any(x => x.ControllingTrigger == ControllingTrigger))
{
Writer.WriteStartElement("Agent");
Writer.WriteAttributeString("Name", Name);
Writer.WriteAttributeString("Type", String.Join(";", PossibleTypes));
foreach (Node Node in Nodes)
{
if (Node.ControllingTrigger == ControllingTrigger)
{
Node.Write(Writer);
}
}
Writer.WriteEndElement();
}
}
/// <summary>
/// Checks whether this node is downstream of the given trigger
/// </summary>
/// <param name="Trigger">The trigger to check</param>
/// <returns>True if the node is downstream of the trigger, false otherwise</returns>
public bool IsControlledBy(ManualTrigger Trigger)
{
return(Trigger == null || ControllingTrigger == Trigger || (ControllingTrigger != null && ControllingTrigger.IsDownstreamFrom(Trigger)));
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="InName">The name of this node</param>
/// <param name="InInputs">Inputs that this node depends on</param>
/// <param name="InOutputNames">Names of the outputs that this node produces</param>
/// <param name="InInputDependencies">Nodes which this node is dependent on for its inputs</param>
/// <param name="InOrderDependencies">Nodes which this node needs to run after. Should include all input dependencies.</param>
/// <param name="InControllingTrigger">The trigger which this node is behind</param>
public Node(string InName, NodeOutput[] InInputs, string[] InOutputNames, Node[] InInputDependencies, Node[] InOrderDependencies, ManualTrigger InControllingTrigger)
{
Name = InName;
Inputs = InInputs;
Outputs = InOutputNames.Select(x => new NodeOutput(this, x)).ToArray();
InputDependencies = InInputDependencies;
OrderDependencies = InOrderDependencies;
ControllingTrigger = InControllingTrigger;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="InParent">The parent trigger</param>
/// <param name="InName">Name of this trigger</param>
public ManualTrigger(ManualTrigger InParent, string InName)
{
Parent = InParent;
Name = InName;
}
/// <summary>
/// Reads a warning from the given element, evaluates the condition on it, and writes it to the log if the condition passes.
/// </summary>
/// <param name="Element">Xml element to read the definition from</param>
/// <param name="EventType">The diagnostic event type</param>
/// <param name="EnclosingObject">The enclosing object instance</param>
void ReadDiagnostic(ScriptElement Element, LogEventType EventType, Node EnclosingNode, AgentGroup EnclosingGroup, ManualTrigger EnclosingTrigger)
{
if (EvaluateCondition(Element))
{
string Message = ReadAttribute(Element, "Message");
GraphDiagnostic Diagnostic = new GraphDiagnostic();
Diagnostic.EventType = EventType;
Diagnostic.Message = String.Format("{0}({1}): {2}", Element.File.FullName, Element.LineNumber, Message);
Diagnostic.EnclosingNode = EnclosingNode;
Diagnostic.EnclosingGroup = EnclosingGroup;
Diagnostic.EnclosingTrigger = EnclosingTrigger;
Graph.Diagnostics.Add(Diagnostic);
}
}
/// <summary>
/// Reads the definition for a node, and adds it to the given agent group
/// </summary>
/// <param name="Element">Xml element to read the definition from</param>
/// <param name="Group">Group for the node to be added to</param>
/// <param name="ControllingTrigger">The controlling trigger for this node</param>
void ReadNode(ScriptElement Element, AgentGroup Group, ManualTrigger ControllingTrigger)
{
string Name;
if (EvaluateCondition(Element) && TryReadObjectName(Element, out Name))
{
string[] RequiresNames = ReadListAttribute(Element, "Requires");
string[] ProducesNames = ReadListAttribute(Element, "Produces");
string[] AfterNames = ReadListAttribute(Element, "After");
// Resolve all the inputs we depend on
HashSet <NodeOutput> Inputs = ResolveInputReferences(Element, RequiresNames);
// Gather up all the input dependencies, and check they're all upstream of the current node
HashSet <Node> InputDependencies = new HashSet <Node>();
foreach (Node InputDependency in Inputs.Select(x => x.ProducingNode).Distinct())
{
if (InputDependency.ControllingTrigger != null && InputDependency.ControllingTrigger != ControllingTrigger && !InputDependency.ControllingTrigger.IsUpstreamFrom(ControllingTrigger))
{
LogError(Element, "'{0}' is dependent on '{1}', which is behind a different controlling trigger ({2})", Name, InputDependency.Name, InputDependency.ControllingTrigger.QualifiedName);
}
else
{
InputDependencies.Add(InputDependency);
}
}
// Recursively include all their dependencies too
foreach (Node InputDependency in InputDependencies.ToArray())
{
InputDependencies.UnionWith(InputDependency.InputDependencies);
}
// Add the name of the node itself to the list of outputs.
List <string> OutputNames = new List <string>();
foreach (string ProducesName in ProducesNames)
{
if (ProducesName.StartsWith("#"))
{
OutputNames.Add(ProducesName.Substring(1));
}
else
{
LogError(Element, "Output tag names must begin with a '#' character ('{0}')", ProducesName);
}
}
OutputNames.Add(Name);
// Gather up all the order dependencies
HashSet <Node> OrderDependencies = new HashSet <Node>(InputDependencies);
OrderDependencies.UnionWith(ResolveReferences(Element, AfterNames));
// Recursively include all their order dependencies too
foreach (Node OrderDependency in OrderDependencies.ToArray())
{
OrderDependencies.UnionWith(OrderDependency.OrderDependencies);
}
// Check that we're not dependent on anything completing that is declared after the initial declaration of this group.
int GroupIdx = Graph.Groups.IndexOf(Group);
for (int Idx = GroupIdx + 1; Idx < Graph.Groups.Count; Idx++)
{
foreach (Node Node in Graph.Groups[Idx].Nodes.Where(x => OrderDependencies.Contains(x)))
{
LogError(Element, "Node '{0}' has a dependency on '{1}', which was declared after the initial definition of '{2}'.", Name, Node.Name, Group.Name);
}
}
// Construct and register the node
if (CheckNameIsUnique(Element, Name))
{
// Add it to the node lookup
Node NewNode = new Node(Name, Inputs.ToArray(), OutputNames.ToArray(), InputDependencies.ToArray(), OrderDependencies.ToArray(), ControllingTrigger);
Graph.NameToNode.Add(Name, NewNode);
// Register each of the outputs as a reference to this node
foreach (NodeOutput Output in NewNode.Outputs)
{
if (Output.Name == Name || CheckNameIsUnique(Element, Output.Name))
{
Graph.NameToNodeOutput.Add(Output.Name, Output);
}
}
// Add all the tasks
EnterScope();
foreach (ScriptElement ChildElement in Element.ChildNodes.OfType <ScriptElement>())
{
switch (ChildElement.Name)
{
case "Property":
ReadProperty(ChildElement);
break;
case "Local":
ReadLocalProperty(ChildElement);
break;
case "Warning":
ReadDiagnostic(ChildElement, LogEventType.Warning, NewNode, Group, ControllingTrigger);
break;
case "Error":
ReadDiagnostic(ChildElement, LogEventType.Error, NewNode, Group, ControllingTrigger);
break;
default:
ReadTask(ChildElement, NewNode.Tasks);
break;
}
}
LeaveScope();
// Add it to the current agent group
Group.Nodes.Add(NewNode);
}
}
}
/// <summary>
/// Export the build graph to a Json file, for parallel execution by the build system
/// </summary>
/// <param name="File">Output file to write</param>
/// <param name="ActivatedTriggers">Set of triggers which have been activated</param>
/// <param name="CompletedNodes">Set of nodes which have been completed</param>
public void Export(FileReference File, HashSet <ManualTrigger> ActivatedTriggers, HashSet <Node> CompletedNodes)
{
// Find all the nodes which we're actually going to execute. We'll use this to filter the graph.
HashSet <Node> NodesToExecute = new HashSet <Node>();
foreach (Node Node in Groups.SelectMany(x => x.Nodes))
{
if (!CompletedNodes.Contains(Node))
{
if (Node.ControllingTrigger == null || ActivatedTriggers.Contains(Node.ControllingTrigger))
{
NodesToExecute.Add(Node);
}
}
}
// Open the output file
using (JsonWriter JsonWriter = new JsonWriter(File.FullName))
{
JsonWriter.WriteObjectStart();
// Write all the agent groups
JsonWriter.WriteArrayStart("Groups");
foreach (AgentGroup Group in Groups)
{
Node[] Nodes = Group.Nodes.Where(x => NodesToExecute.Contains(x)).ToArray();
if (Nodes.Length > 0)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Group.Name);
JsonWriter.WriteArrayStart("Agent Types");
foreach (string AgentType in Group.PossibleTypes)
{
JsonWriter.WriteValue(AgentType);
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteArrayStart("Nodes");
foreach (Node Node in Nodes)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Node.Name);
JsonWriter.WriteValue("DependsOn", String.Join(";", Node.GetDirectOrderDependencies().Where(x => !CompletedNodes.Contains(x))));
JsonWriter.WriteObjectStart("Notify");
JsonWriter.WriteValue("Default", String.Join(";", Node.NotifyUsers));
JsonWriter.WriteValue("Submitters", String.Join(";", Node.NotifySubmitters));
JsonWriter.WriteValue("Warnings", Node.bNotifyOnWarnings);
JsonWriter.WriteObjectEnd();
JsonWriter.WriteObjectEnd();
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
// Write all the triggers
JsonWriter.WriteArrayStart("Triggers");
foreach (ManualTrigger Trigger in NameToTrigger.Values)
{
if (!ActivatedTriggers.Contains(Trigger) && NodesToExecute.Any(x => x.ControllingTrigger == Trigger.Parent))
{
// Find all the nodes that this trigger is dependent on
HashSet <Node> Dependencies = new HashSet <Node>();
foreach (Node Node in Groups.SelectMany(x => x.Nodes))
{
for (ManualTrigger ControllingTrigger = Node.ControllingTrigger; ControllingTrigger != null; ControllingTrigger = ControllingTrigger.Parent)
{
if (ControllingTrigger == Trigger)
{
Dependencies.UnionWith(Node.OrderDependencies.Where(x => x.ControllingTrigger != Trigger && NodesToExecute.Contains(x)));
break;
}
}
}
// Reduce that list to the smallest subset of direct dependencies
HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies);
foreach (Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
// Write out the object
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Trigger.Name);
JsonWriter.WriteValue("AllDependencies", String.Join(";", Groups.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", Dependencies.Where(x => DirectDependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("Notify", String.Join(";", Trigger.NotifyUsers));
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
/// <summary>
/// Reads the definition for an agent group.
/// </summary>
/// <param name="Element">Xml element to read the definition from</param>
void ReadTrigger(ScriptElement Element)
{
string[] QualifiedName;
if (EvaluateCondition(Element) && TryReadQualifiedObjectName(Element, out QualifiedName))
{
// Validate all the parent triggers
ManualTrigger ParentTrigger = null;
for (int Idx = 0; Idx < QualifiedName.Length - 1; Idx++)
{
ManualTrigger NextTrigger;
if (!Graph.NameToTrigger.TryGetValue(QualifiedName[Idx], out NextTrigger))
{
LogError(Element, "Unknown trigger '{0}'", QualifiedName[Idx]);
return;
}
if (NextTrigger.Parent != ParentTrigger)
{
LogError(Element, "Qualified name of trigger '{0}' is '{1}'", NextTrigger.Name, NextTrigger.QualifiedName);
return;
}
ParentTrigger = NextTrigger;
}
// Get the name of the new trigger
string Name = QualifiedName[QualifiedName.Length - 1];
// Create the new trigger
ManualTrigger Trigger;
if (!Graph.NameToTrigger.TryGetValue(Name, out Trigger))
{
Trigger = new ManualTrigger(ParentTrigger, Name);
Graph.NameToTrigger.Add(Name, Trigger);
}
else if (Trigger.Parent != ParentTrigger)
{
LogError(Element, "Conflicting parent for '{0}' - previously declared as '{1}', now '{2}'", Name, Trigger.QualifiedName, new ManualTrigger(ParentTrigger, Name).QualifiedName);
return;
}
// Read the root BuildGraph element
EnterScope();
foreach (ScriptElement ChildElement in Element.ChildNodes.OfType <ScriptElement>())
{
switch (ChildElement.Name)
{
case "Property":
ReadProperty(ChildElement);
break;
case "Local":
ReadLocalProperty(ChildElement);
break;
case "Agent":
ReadAgent(ChildElement, Trigger);
break;
case "Aggregate":
ReadAggregate(ChildElement);
break;
case "Notifier":
ReadNotifier(ChildElement);
break;
case "Warning":
ReadDiagnostic(ChildElement, LogEventType.Warning, null, null, Trigger);
break;
case "Error":
ReadDiagnostic(ChildElement, LogEventType.Error, null, null, Trigger);
break;
default:
LogError(ChildElement, "Invalid element '{0}'", ChildElement.Name);
break;
}
}
LeaveScope();
}
}
/// <summary>
/// Checks whether this node is behind the given trigger
/// </summary>
/// <param name="Trigger">The trigger to check</param>
/// <returns>True if the node is directly or indirectly behind the given trigger, false otherwise</returns>
public bool IsBehind(ManualTrigger Trigger)
{
for(ManualTrigger OtherTrigger = ControllingTrigger; OtherTrigger != Trigger; OtherTrigger = OtherTrigger.Parent)
{
if(OtherTrigger == null)
{
return false;
}
}
return true;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="InName">The name of this node</param>
/// <param name="InInputs">Inputs that this node depends on</param>
/// <param name="InOutputNames">Names of the outputs that this node produces</param>
/// <param name="InInputDependencies">Nodes which this node is dependent on for its inputs</param>
/// <param name="InOrderDependencies">Nodes which this node needs to run after. Should include all input dependencies.</param>
/// <param name="InControllingTrigger">The trigger which this node is behind</param>
/// <param name="InRequiredTokens">Optional tokens which must be required for this node to run</param>
public Node(string InName, NodeOutput[] InInputs, string[] InOutputNames, Node[] InInputDependencies, Node[] InOrderDependencies, ManualTrigger InControllingTrigger, FileReference[] InRequiredTokens)
{
Name = InName;
Inputs = InInputs;
List<NodeOutput> AllOutputs = new List<NodeOutput>();
AllOutputs.Add(new NodeOutput(this, "#" + Name));
AllOutputs.AddRange(InOutputNames.Where(x => String.Compare(x, Name, StringComparison.InvariantCultureIgnoreCase) != 0).Select(x => new NodeOutput(this, x)));
Outputs = AllOutputs.ToArray();
InputDependencies = InInputDependencies;
OrderDependencies = InOrderDependencies;
ControllingTrigger = InControllingTrigger;
RequiredTokens = InRequiredTokens;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="InParent">The parent trigger</param>
/// <param name="InName">Name of this trigger</param>
public ManualTrigger(ManualTrigger InParent, string InName)
{
Parent = InParent;
Name = InName;
}
/// <summary>
/// Main entry point for the BuildGraph command
/// </summary>
public override ExitCode Execute()
{
// Parse the command line parameters
string ScriptFileName = ParseParamValue("Script", null);
string TargetNames = ParseParamValue("Target", null);
string DocumentationFileName = ParseParamValue("Documentation", null);
string SchemaFileName = ParseParamValue("Schema", null);
string ExportFileName = ParseParamValue("Export", null);
string PreprocessedFileName = ParseParamValue("Preprocess", null);
string SharedStorageDir = ParseParamValue("SharedStorageDir", null);
string SingleNodeName = ParseParamValue("SingleNode", null);
string TriggerName = ParseParamValue("Trigger", null);
string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
bool bSkipTriggers = ParseParam("SkipTriggers");
string TokenSignature = ParseParamValue("TokenSignature", null);
bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens");
bool bResume = SingleNodeName != null || ParseParam("Resume");
bool bListOnly = ParseParam("ListOnly");
bool bShowDiagnostics = ParseParam("ShowDiagnostics");
bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine;
bool bPublicTasksOnly = ParseParam("PublicTasksOnly");
string ReportName = ParseParamValue("ReportName", null);
GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions;
if (ParseParam("ShowDeps"))
{
PrintOptions |= GraphPrintOptions.ShowDependencies;
}
if (ParseParam("ShowNotifications"))
{
PrintOptions |= GraphPrintOptions.ShowNotifications;
}
// Parse any specific nodes to clean
List <string> CleanNodes = new List <string>();
foreach (string NodeList in ParseParamValues("CleanNode"))
{
foreach (string NodeName in NodeList.Split('+', ';'))
{
CleanNodes.Add(NodeName);
}
}
// Set up the standard properties which build scripts might need
Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
DefaultProperties["Branch"] = P4Enabled ? P4Env.Branch : "Unknown";
DefaultProperties["Depot"] = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown";
DefaultProperties["EscapedBranch"] = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown";
DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0";
DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0";
DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName;
DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false";
DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString();
DefaultProperties["RestrictedFolderNames"] = String.Join(";", RestrictedFolders.Names);
DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x)));
// Attempt to read existing Build Version information
BuildVersion Version;
if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
{
DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString();
DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString();
DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString();
DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString();
}
// Add any additional custom arguments from the command line (of the form -Set:X=Y)
Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string Param in Params)
{
const string Prefix = "set:";
if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase))
{
int EqualsIdx = Param.IndexOf('=');
if (EqualsIdx >= 0)
{
Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1);
}
else
{
LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length));
}
}
}
// Find all the tasks from the loaded assemblies
Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>();
if (!FindAvailableTasks(NameToTask, bPublicTasksOnly))
{
return(ExitCode.Error_Unknown);
}
// Generate documentation
if (DocumentationFileName != null)
{
GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName));
return(ExitCode.Success);
}
// Create a schema for the given tasks
ScriptSchema Schema = new ScriptSchema(NameToTask);
if (SchemaFileName != null)
{
FileReference FullSchemaFileName = new FileReference(SchemaFileName);
LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName);
Schema.Export(FullSchemaFileName);
if (ScriptFileName == null)
{
return(ExitCode.Success);
}
}
// Check there was a script specified
if (ScriptFileName == null)
{
LogError("Missing -Script= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
// Read the script from disk
Graph Graph;
if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph))
{
return(ExitCode.Error_Unknown);
}
// Create the temp storage handler
DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot);
TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage);
if (!bResume)
{
Storage.CleanLocal();
}
foreach (string CleanNode in CleanNodes)
{
Storage.CleanLocalNode(CleanNode);
}
// Convert the supplied target references into nodes
HashSet <Node> TargetNodes = new HashSet <Node>();
if (TargetNames == null)
{
if (!bListOnly)
{
LogError("Missing -Target= parameter for BuildGraph");
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
}
else
{
foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
{
Node[] Nodes;
if (!Graph.TryResolveReference(TargetName, out Nodes))
{
LogError("Target '{0}' is not in graph", TargetName);
return(ExitCode.Error_Unknown);
}
TargetNodes.UnionWith(Nodes);
}
}
// Try to acquire tokens for all the target nodes we want to build
if (TokenSignature != null)
{
// Find all the lock files
HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens));
// List out all the required tokens
if (SingleNodeName == null)
{
CommandUtils.LogInformation("Required tokens:");
foreach (Node Node in TargetNodes)
{
foreach (FileReference RequiredToken in Node.RequiredTokens)
{
CommandUtils.LogInformation(" '{0}' requires {1}", Node, RequiredToken);
}
}
}
// Try to create all the lock files
List <FileReference> CreatedTokens = new List <FileReference>();
if (!bListOnly)
{
CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature)));
}
// Find all the tokens that we don't have
Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>();
foreach (FileReference RequiredToken in RequiredTokens)
{
string CurrentOwner = ReadTokenFile(RequiredToken);
if (CurrentOwner != null && CurrentOwner != TokenSignature)
{
MissingTokens.Add(RequiredToken, CurrentOwner);
}
}
// If we want to skip all the nodes with missing locks, adjust the target nodes to account for it
if (MissingTokens.Count > 0)
{
if (bSkipTargetsWithoutTokens)
{
// Find all the nodes we're going to skip
HashSet <Node> SkipNodes = new HashSet <Node>();
foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key))
{
LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key);
foreach (FileReference MissingToken in MissingTokensForBuild)
{
foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x)))
{
LogInformation(" {0}", SkipNode);
}
}
}
// Write a list of everything left over
if (SkipNodes.Count > 0)
{
TargetNodes.ExceptWith(SkipNodes);
LogInformation("Remaining target nodes:");
foreach (Node TargetNode in TargetNodes)
{
LogInformation(" {0}", TargetNode);
}
if (TargetNodes.Count == 0)
{
LogInformation(" None.");
}
}
}
else
{
foreach (KeyValuePair <FileReference, string> Pair in MissingTokens)
{
List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList();
LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value);
}
foreach (FileReference CreatedToken in CreatedTokens)
{
FileReference.Delete(CreatedToken);
}
return(ExitCode.Error_Unknown);
}
}
}
// Cull the graph to include only those nodes
Graph.Select(TargetNodes);
// Collapse any triggers in the graph which are marked to be skipped
HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>();
if (bSkipTriggers)
{
SkipTriggers.UnionWith(Graph.NameToTrigger.Values);
}
else
{
foreach (string SkipTriggerName in SkipTriggerNames)
{
ManualTrigger SkipTrigger;
if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
SkipTriggers.Add(SkipTrigger);
}
}
Graph.SkipTriggers(SkipTriggers);
// If a report for the whole build was requested, insert it into the graph
if (ReportName != null)
{
Report NewReport = new Report(ReportName);
NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes));
Graph.NameToReport.Add(ReportName, NewReport);
}
// Write out the preprocessed script
if (PreprocessedFileName != null)
{
FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName);
LogInformation("Writing {0}...", PreprocessedFileLocation);
Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null);
return(ExitCode.Success);
}
// Find the triggers which we are explicitly running.
ManualTrigger Trigger = null;
if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger))
{
LogError("Couldn't find trigger '{0}'", TriggerName);
return(ExitCode.Error_Unknown);
}
// If we're just building a single node, find it
Node SingleNode = null;
if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode))
{
LogError("Node '{0}' is not in the trimmed graph", SingleNodeName);
return(ExitCode.Error_Unknown);
}
// If we just want to show the contents of the graph, do so and exit.
if (bListOnly)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
}
// Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file.
if (SingleNode == null && (!bListOnly || bShowDiagnostics))
{
IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger);
foreach (GraphDiagnostic Diagnostic in Diagnostics)
{
if (Diagnostic.EventType == LogEventType.Console)
{
CommandUtils.LogInformation(Diagnostic.Message);
}
else if (Diagnostic.EventType == LogEventType.Warning)
{
CommandUtils.LogWarning(Diagnostic.Message);
}
else
{
CommandUtils.LogError(Diagnostic.Message);
}
}
if (Diagnostics.Any(x => x.EventType == LogEventType.Error))
{
return(ExitCode.Error_Unknown);
}
}
// Export the graph to a file
if (ExportFileName != null)
{
HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage);
Graph.Print(CompletedNodes, PrintOptions);
Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes);
return(ExitCode.Success);
}
// Execute the command
if (!bListOnly)
{
if (SingleNode != null)
{
if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true))
{
return(ExitCode.Error_Unknown);
}
}
else
{
if (!BuildAllNodes(new JobContext(this), Graph, Storage))
{
return(ExitCode.Error_Unknown);
}
}
}
return(ExitCode.Success);
}
/// <summary>
/// Reads the definition for an agent group.
/// </summary>
/// <param name="Element">Xml element to read the definition from</param>
/// <param name="Trigger">The controlling trigger for nodes in this group</param>
void ReadAgent(ScriptElement Element, ManualTrigger Trigger)
{
string Name;
if (EvaluateCondition(Element) && TryReadObjectName(Element, out Name))
{
// Read the valid agent types. This may be omitted if we're continuing an existing group.
string[] Types = ReadListAttribute(Element, "Type");
// Create the group object, or continue an existing one
AgentGroup Group;
if (NameToGroup.TryGetValue(Name, out Group))
{
if (Types.Length > 0 && Group.PossibleTypes.Length > 0)
{
string[] NewTypes = Group.PossibleTypes.Intersect(Types, StringComparer.InvariantCultureIgnoreCase).ToArray();
if (NewTypes.Length == 0)
{
LogError(Element, "No common agent types with previous agent definition");
}
Group.PossibleTypes = NewTypes;
}
}
else
{
if (Types.Length == 0)
{
LogError(Element, "Missing agent type for group '{0}'", Name);
}
Group = new AgentGroup(Name, Types);
NameToGroup.Add(Name, Group);
Graph.Groups.Add(Group);
}
// Process all the child elements.
EnterScope();
foreach (ScriptElement ChildElement in Element.ChildNodes.OfType <ScriptElement>())
{
switch (ChildElement.Name)
{
case "Property":
ReadProperty(ChildElement);
break;
case "Local":
ReadLocalProperty(ChildElement);
break;
case "Node":
ReadNode(ChildElement, Group, Trigger);
break;
case "Aggregate":
ReadAggregate(ChildElement);
break;
case "Warning":
ReadDiagnostic(ChildElement, LogEventType.Warning, null, Group, Trigger);
break;
case "Error":
ReadDiagnostic(ChildElement, LogEventType.Error, null, Group, Trigger);
break;
default:
LogError(ChildElement, "Unexpected element type '{0}'", ChildElement.Name);
break;
}
}
LeaveScope();
}
}
/// <summary>
/// Export the build graph to a Json file, for parallel execution by the build system
/// </summary>
/// <param name="File">Output file to write</param>
/// <param name="Trigger">The trigger whose nodes to run. Null for the default nodes.</param>
/// <param name="CompletedNodes">Set of nodes which have been completed</param>
public void Export(FileReference File, ManualTrigger Trigger, HashSet <Node> CompletedNodes)
{
// Find all the nodes which we're actually going to execute. We'll use this to filter the graph.
HashSet <Node> NodesToExecute = new HashSet <Node>();
foreach (Node Node in Agents.SelectMany(x => x.Nodes))
{
if (!CompletedNodes.Contains(Node) && Node.IsBehind(Trigger))
{
NodesToExecute.Add(Node);
}
}
// Open the output file
using (JsonWriter JsonWriter = new JsonWriter(File.FullName))
{
JsonWriter.WriteObjectStart();
// Write all the agents
JsonWriter.WriteArrayStart("Groups");
foreach (Agent Agent in Agents)
{
Node[] Nodes = Agent.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray();
if (Nodes.Length > 0)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Agent.Name);
JsonWriter.WriteArrayStart("Agent Types");
foreach (string AgentType in Agent.PossibleTypes)
{
JsonWriter.WriteValue(AgentType);
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteArrayStart("Nodes");
foreach (Node Node in Nodes)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Node.Name);
JsonWriter.WriteValue("DependsOn", String.Join(";", Node.GetDirectOrderDependencies().Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger)));
JsonWriter.WriteValue("RunEarly", Node.bRunEarly);
JsonWriter.WriteObjectStart("Notify");
JsonWriter.WriteValue("Default", String.Join(";", Node.NotifyUsers));
JsonWriter.WriteValue("Submitters", String.Join(";", Node.NotifySubmitters));
JsonWriter.WriteValue("Warnings", Node.bNotifyOnWarnings);
JsonWriter.WriteObjectEnd();
JsonWriter.WriteObjectEnd();
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
// Write all the badges
JsonWriter.WriteArrayStart("Badges");
foreach (Badge Badge in Badges)
{
Node[] Dependencies = Badge.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray();
if (Dependencies.Length > 0)
{
// Reduce that list to the smallest subset of direct dependencies
HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies);
foreach (Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Badge.Name);
if (!String.IsNullOrEmpty(Badge.Project))
{
JsonWriter.WriteValue("Project", Badge.Project);
}
if (Badge.Change != 0)
{
JsonWriter.WriteValue("Change", Badge.Change);
}
JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name)));
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
// Write all the triggers and reports.
JsonWriter.WriteArrayStart("Reports");
foreach (Report Report in NameToReport.Values)
{
Node[] Dependencies = Report.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray();
if (Dependencies.Length > 0)
{
// Reduce that list to the smallest subset of direct dependencies
HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies);
foreach (Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Report.Name);
JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name)));
JsonWriter.WriteValue("Notify", String.Join(";", Report.NotifyUsers));
JsonWriter.WriteValue("IsTrigger", false);
JsonWriter.WriteObjectEnd();
}
}
foreach (ManualTrigger DownstreamTrigger in NameToTrigger.Values)
{
if (DownstreamTrigger.Parent == Trigger)
{
// Find all the nodes that this trigger is dependent on
HashSet <Node> Dependencies = new HashSet <Node>();
foreach (Node NodeToExecute in NodesToExecute)
{
if (NodeToExecute.IsBehind(DownstreamTrigger))
{
Dependencies.UnionWith(NodeToExecute.OrderDependencies.Where(x => x.ControllingTrigger == Trigger));
}
}
// Reduce that list to the smallest subset of direct dependencies
HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies);
foreach (Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
// Write out the object
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", DownstreamTrigger.Name);
JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", Dependencies.Where(x => DirectDependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("Notify", String.Join(";", DownstreamTrigger.NotifyUsers));
JsonWriter.WriteValue("IsTrigger", true);
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
/// <summary>
/// Export the build graph to a Json file, for parallel execution by the build system
/// </summary>
/// <param name="File">Output file to write</param>
/// <param name="Trigger">The trigger whose nodes to run. Null for the default nodes.</param>
/// <param name="CompletedNodes">Set of nodes which have been completed</param>
public void Export(FileReference File, ManualTrigger Trigger, HashSet<Node> CompletedNodes)
{
// Find all the nodes which we're actually going to execute. We'll use this to filter the graph.
HashSet<Node> NodesToExecute = new HashSet<Node>();
foreach(Node Node in Agents.SelectMany(x => x.Nodes))
{
if(!CompletedNodes.Contains(Node) && Node.IsBehind(Trigger))
{
NodesToExecute.Add(Node);
}
}
// Open the output file
using(JsonWriter JsonWriter = new JsonWriter(File.FullName))
{
JsonWriter.WriteObjectStart();
// Write all the agents
JsonWriter.WriteArrayStart("Groups");
foreach(Agent Agent in Agents)
{
Node[] Nodes = Agent.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray();
if(Nodes.Length > 0)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Agent.Name);
JsonWriter.WriteArrayStart("Agent Types");
foreach(string AgentType in Agent.PossibleTypes)
{
JsonWriter.WriteValue(AgentType);
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteArrayStart("Nodes");
foreach(Node Node in Nodes)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Node.Name);
JsonWriter.WriteValue("DependsOn", String.Join(";", Node.GetDirectOrderDependencies().Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger)));
JsonWriter.WriteObjectStart("Notify");
JsonWriter.WriteValue("Default", String.Join(";", Node.NotifyUsers));
JsonWriter.WriteValue("Submitters", String.Join(";", Node.NotifySubmitters));
JsonWriter.WriteValue("Warnings", Node.bNotifyOnWarnings);
JsonWriter.WriteObjectEnd();
JsonWriter.WriteObjectEnd();
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
// Write all the badges
JsonWriter.WriteArrayStart("Badges");
foreach (Badge Badge in Badges)
{
Node[] Dependencies = Badge.Nodes.Where(x => NodesToExecute.Contains(x)).ToArray();
if (Dependencies.Length > 0)
{
// Reduce that list to the smallest subset of direct dependencies
HashSet<Node> DirectDependencies = new HashSet<Node>(Dependencies);
foreach (Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Badge.Name);
if (!String.IsNullOrEmpty(Badge.Project))
{
JsonWriter.WriteValue("Project", Badge.Project);
}
JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name)));
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
// Write all the triggers and reports.
JsonWriter.WriteArrayStart("Reports");
foreach (Report Report in NameToReport.Values)
{
Node[] Dependencies = Report.Nodes.Where(x => NodesToExecute.Contains(x)).ToArray();
if (Dependencies.Length > 0)
{
// Reduce that list to the smallest subset of direct dependencies
HashSet<Node> DirectDependencies = new HashSet<Node>(Dependencies);
foreach (Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Report.Name);
JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name)));
JsonWriter.WriteValue("Notify", String.Join(";", Report.NotifyUsers));
JsonWriter.WriteValue("IsTrigger", false);
JsonWriter.WriteObjectEnd();
}
}
foreach (ManualTrigger DownstreamTrigger in NameToTrigger.Values)
{
if(DownstreamTrigger.Parent == Trigger)
{
// Find all the nodes that this trigger is dependent on
HashSet<Node> Dependencies = new HashSet<Node>();
foreach(Node NodeToExecute in NodesToExecute)
{
if(NodeToExecute.IsBehind(DownstreamTrigger))
{
Dependencies.UnionWith(NodeToExecute.OrderDependencies.Where(x => x.ControllingTrigger == Trigger));
}
}
// Reduce that list to the smallest subset of direct dependencies
HashSet<Node> DirectDependencies = new HashSet<Node>(Dependencies);
foreach(Node Dependency in Dependencies)
{
DirectDependencies.ExceptWith(Dependency.OrderDependencies);
}
// Write out the object
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", DownstreamTrigger.Name);
JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("DirectDependencies", String.Join(";", Dependencies.Where(x => DirectDependencies.Contains(x)).Select(x => x.Name)));
JsonWriter.WriteValue("Notify", String.Join(";", DownstreamTrigger.NotifyUsers));
JsonWriter.WriteValue("IsTrigger", true);
JsonWriter.WriteObjectEnd();
}
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
