/// <summary> /// Checks whether this trigger is downstream of another /// </summary> /// <param name="Other">The parent trigger to check</param> /// <returns>True if the trigger is downstream of the given trigger</returns> public bool IsDownstreamFrom(ManualTrigger Other) { for(ManualTrigger Ancestor = Parent; Ancestor != null; Ancestor = Ancestor.Parent) { if(Ancestor == Other) { return true; } } return false; }
/// <summary> /// Constructor /// </summary> /// <param name="InName">The name of this node</param> /// <param name="InInputs">Inputs that this node depends on</param> /// <param name="InOutputNames">Names of the outputs that this node produces</param> /// <param name="InInputDependencies">Nodes which this node is dependent on for its inputs</param> /// <param name="InOrderDependencies">Nodes which this node needs to run after. Should include all input dependencies.</param> /// <param name="InControllingTrigger">The trigger which this node is behind</param> /// <param name="InRequiredTokens">Optional tokens which must be required for this node to run</param> public Node(string InName, NodeOutput[] InInputs, string[] InOutputNames, Node[] InInputDependencies, Node[] InOrderDependencies, ManualTrigger InControllingTrigger, FileReference[] InRequiredTokens) { Name = InName; Inputs = InInputs; List <NodeOutput> AllOutputs = new List <NodeOutput>(); AllOutputs.Add(new NodeOutput(this, "#" + Name)); AllOutputs.AddRange(InOutputNames.Where(x => String.Compare(x, Name, StringComparison.InvariantCultureIgnoreCase) != 0).Select(x => new NodeOutput(this, x))); Outputs = AllOutputs.ToArray(); InputDependencies = InInputDependencies; OrderDependencies = InOrderDependencies; ControllingTrigger = InControllingTrigger; RequiredTokens = InRequiredTokens; }
/// <summary> /// Writes this agent group out to a file, filtering nodes by a controlling trigger /// </summary> /// <param name="Writer">The XML writer to output to</param> /// <param name="ControllingTrigger">The controlling trigger to filter by</param> public void Write(XmlWriter Writer, ManualTrigger ControllingTrigger) { if (Nodes.Any(x => x.ControllingTrigger == ControllingTrigger)) { Writer.WriteStartElement("Agent"); Writer.WriteAttributeString("Name", Name); Writer.WriteAttributeString("Type", String.Join(";", PossibleTypes)); foreach (Node Node in Nodes) { if (Node.ControllingTrigger == ControllingTrigger) { Node.Write(Writer); } } Writer.WriteEndElement(); } }
/// <summary> /// Writes this agent group out to a file, filtering nodes by a controlling trigger /// </summary> /// <param name="Writer">The XML writer to output to</param> /// <param name="ControllingTrigger">The controlling trigger to filter by</param> public void Write(XmlWriter Writer, ManualTrigger ControllingTrigger) { if (Nodes.Any(x => x.ControllingTrigger == ControllingTrigger)) { Writer.WriteStartElement("Agent"); Writer.WriteAttributeString("Name", Name); Writer.WriteAttributeString("Type", String.Join(";", PossibleTypes)); foreach (Node Node in Nodes) { if (Node.ControllingTrigger == ControllingTrigger) { Node.Write(Writer); } } Writer.WriteEndElement(); } }
/// <summary> /// Checks whether this node is downstream of the given trigger /// </summary> /// <param name="Trigger">The trigger to check</param> /// <returns>True if the node is downstream of the trigger, false otherwise</returns> public bool IsControlledBy(ManualTrigger Trigger) { return(Trigger == null || ControllingTrigger == Trigger || (ControllingTrigger != null && ControllingTrigger.IsDownstreamFrom(Trigger))); }
/// <summary> /// Constructor /// </summary> /// <param name="InName">The name of this node</param> /// <param name="InInputs">Inputs that this node depends on</param> /// <param name="InOutputNames">Names of the outputs that this node produces</param> /// <param name="InInputDependencies">Nodes which this node is dependent on for its inputs</param> /// <param name="InOrderDependencies">Nodes which this node needs to run after. Should include all input dependencies.</param> /// <param name="InControllingTrigger">The trigger which this node is behind</param> public Node(string InName, NodeOutput[] InInputs, string[] InOutputNames, Node[] InInputDependencies, Node[] InOrderDependencies, ManualTrigger InControllingTrigger) { Name = InName; Inputs = InInputs; Outputs = InOutputNames.Select(x => new NodeOutput(this, x)).ToArray(); InputDependencies = InInputDependencies; OrderDependencies = InOrderDependencies; ControllingTrigger = InControllingTrigger; }
/// <summary> /// Constructor /// </summary> /// <param name="InParent">The parent trigger</param> /// <param name="InName">Name of this trigger</param> public ManualTrigger(ManualTrigger InParent, string InName) { Parent = InParent; Name = InName; }
/// <summary> /// Reads a warning from the given element, evaluates the condition on it, and writes it to the log if the condition passes. /// </summary> /// <param name="Element">Xml element to read the definition from</param> /// <param name="EventType">The diagnostic event type</param> /// <param name="EnclosingObject">The enclosing object instance</param> void ReadDiagnostic(ScriptElement Element, LogEventType EventType, Node EnclosingNode, AgentGroup EnclosingGroup, ManualTrigger EnclosingTrigger) { if (EvaluateCondition(Element)) { string Message = ReadAttribute(Element, "Message"); GraphDiagnostic Diagnostic = new GraphDiagnostic(); Diagnostic.EventType = EventType; Diagnostic.Message = String.Format("{0}({1}): {2}", Element.File.FullName, Element.LineNumber, Message); Diagnostic.EnclosingNode = EnclosingNode; Diagnostic.EnclosingGroup = EnclosingGroup; Diagnostic.EnclosingTrigger = EnclosingTrigger; Graph.Diagnostics.Add(Diagnostic); } }
/// <summary> /// Reads the definition for a node, and adds it to the given agent group /// </summary> /// <param name="Element">Xml element to read the definition from</param> /// <param name="Group">Group for the node to be added to</param> /// <param name="ControllingTrigger">The controlling trigger for this node</param> void ReadNode(ScriptElement Element, AgentGroup Group, ManualTrigger ControllingTrigger) { string Name; if (EvaluateCondition(Element) && TryReadObjectName(Element, out Name)) { string[] RequiresNames = ReadListAttribute(Element, "Requires"); string[] ProducesNames = ReadListAttribute(Element, "Produces"); string[] AfterNames = ReadListAttribute(Element, "After"); // Resolve all the inputs we depend on HashSet <NodeOutput> Inputs = ResolveInputReferences(Element, RequiresNames); // Gather up all the input dependencies, and check they're all upstream of the current node HashSet <Node> InputDependencies = new HashSet <Node>(); foreach (Node InputDependency in Inputs.Select(x => x.ProducingNode).Distinct()) { if (InputDependency.ControllingTrigger != null && InputDependency.ControllingTrigger != ControllingTrigger && !InputDependency.ControllingTrigger.IsUpstreamFrom(ControllingTrigger)) { LogError(Element, "'{0}' is dependent on '{1}', which is behind a different controlling trigger ({2})", Name, InputDependency.Name, InputDependency.ControllingTrigger.QualifiedName); } else { InputDependencies.Add(InputDependency); } } // Recursively include all their dependencies too foreach (Node InputDependency in InputDependencies.ToArray()) { InputDependencies.UnionWith(InputDependency.InputDependencies); } // Add the name of the node itself to the list of outputs. List <string> OutputNames = new List <string>(); foreach (string ProducesName in ProducesNames) { if (ProducesName.StartsWith("#")) { OutputNames.Add(ProducesName.Substring(1)); } else { LogError(Element, "Output tag names must begin with a '#' character ('{0}')", ProducesName); } } OutputNames.Add(Name); // Gather up all the order dependencies HashSet <Node> OrderDependencies = new HashSet <Node>(InputDependencies); OrderDependencies.UnionWith(ResolveReferences(Element, AfterNames)); // Recursively include all their order dependencies too foreach (Node OrderDependency in OrderDependencies.ToArray()) { OrderDependencies.UnionWith(OrderDependency.OrderDependencies); } // Check that we're not dependent on anything completing that is declared after the initial declaration of this group. int GroupIdx = Graph.Groups.IndexOf(Group); for (int Idx = GroupIdx + 1; Idx < Graph.Groups.Count; Idx++) { foreach (Node Node in Graph.Groups[Idx].Nodes.Where(x => OrderDependencies.Contains(x))) { LogError(Element, "Node '{0}' has a dependency on '{1}', which was declared after the initial definition of '{2}'.", Name, Node.Name, Group.Name); } } // Construct and register the node if (CheckNameIsUnique(Element, Name)) { // Add it to the node lookup Node NewNode = new Node(Name, Inputs.ToArray(), OutputNames.ToArray(), InputDependencies.ToArray(), OrderDependencies.ToArray(), ControllingTrigger); Graph.NameToNode.Add(Name, NewNode); // Register each of the outputs as a reference to this node foreach (NodeOutput Output in NewNode.Outputs) { if (Output.Name == Name || CheckNameIsUnique(Element, Output.Name)) { Graph.NameToNodeOutput.Add(Output.Name, Output); } } // Add all the tasks EnterScope(); foreach (ScriptElement ChildElement in Element.ChildNodes.OfType <ScriptElement>()) { switch (ChildElement.Name) { case "Property": ReadProperty(ChildElement); break; case "Local": ReadLocalProperty(ChildElement); break; case "Warning": ReadDiagnostic(ChildElement, LogEventType.Warning, NewNode, Group, ControllingTrigger); break; case "Error": ReadDiagnostic(ChildElement, LogEventType.Error, NewNode, Group, ControllingTrigger); break; default: ReadTask(ChildElement, NewNode.Tasks); break; } } LeaveScope(); // Add it to the current agent group Group.Nodes.Add(NewNode); } } }
/// <summary> /// Export the build graph to a Json file, for parallel execution by the build system /// </summary> /// <param name="File">Output file to write</param> /// <param name="ActivatedTriggers">Set of triggers which have been activated</param> /// <param name="CompletedNodes">Set of nodes which have been completed</param> public void Export(FileReference File, HashSet <ManualTrigger> ActivatedTriggers, HashSet <Node> CompletedNodes) { // Find all the nodes which we're actually going to execute. We'll use this to filter the graph. HashSet <Node> NodesToExecute = new HashSet <Node>(); foreach (Node Node in Groups.SelectMany(x => x.Nodes)) { if (!CompletedNodes.Contains(Node)) { if (Node.ControllingTrigger == null || ActivatedTriggers.Contains(Node.ControllingTrigger)) { NodesToExecute.Add(Node); } } } // Open the output file using (JsonWriter JsonWriter = new JsonWriter(File.FullName)) { JsonWriter.WriteObjectStart(); // Write all the agent groups JsonWriter.WriteArrayStart("Groups"); foreach (AgentGroup Group in Groups) { Node[] Nodes = Group.Nodes.Where(x => NodesToExecute.Contains(x)).ToArray(); if (Nodes.Length > 0) { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Group.Name); JsonWriter.WriteArrayStart("Agent Types"); foreach (string AgentType in Group.PossibleTypes) { JsonWriter.WriteValue(AgentType); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteArrayStart("Nodes"); foreach (Node Node in Nodes) { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Node.Name); JsonWriter.WriteValue("DependsOn", String.Join(";", Node.GetDirectOrderDependencies().Where(x => !CompletedNodes.Contains(x)))); JsonWriter.WriteObjectStart("Notify"); JsonWriter.WriteValue("Default", String.Join(";", Node.NotifyUsers)); JsonWriter.WriteValue("Submitters", String.Join(";", Node.NotifySubmitters)); JsonWriter.WriteValue("Warnings", Node.bNotifyOnWarnings); JsonWriter.WriteObjectEnd(); JsonWriter.WriteObjectEnd(); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); // Write all the triggers JsonWriter.WriteArrayStart("Triggers"); foreach (ManualTrigger Trigger in NameToTrigger.Values) { if (!ActivatedTriggers.Contains(Trigger) && NodesToExecute.Any(x => x.ControllingTrigger == Trigger.Parent)) { // Find all the nodes that this trigger is dependent on HashSet <Node> Dependencies = new HashSet <Node>(); foreach (Node Node in Groups.SelectMany(x => x.Nodes)) { for (ManualTrigger ControllingTrigger = Node.ControllingTrigger; ControllingTrigger != null; ControllingTrigger = ControllingTrigger.Parent) { if (ControllingTrigger == Trigger) { Dependencies.UnionWith(Node.OrderDependencies.Where(x => x.ControllingTrigger != Trigger && NodesToExecute.Contains(x))); break; } } } // Reduce that list to the smallest subset of direct dependencies HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies); foreach (Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } // Write out the object JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Trigger.Name); JsonWriter.WriteValue("AllDependencies", String.Join(";", Groups.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", Dependencies.Where(x => DirectDependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("Notify", String.Join(";", Trigger.NotifyUsers)); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } }
/// <summary> /// Reads the definition for an agent group. /// </summary> /// <param name="Element">Xml element to read the definition from</param> void ReadTrigger(ScriptElement Element) { string[] QualifiedName; if (EvaluateCondition(Element) && TryReadQualifiedObjectName(Element, out QualifiedName)) { // Validate all the parent triggers ManualTrigger ParentTrigger = null; for (int Idx = 0; Idx < QualifiedName.Length - 1; Idx++) { ManualTrigger NextTrigger; if (!Graph.NameToTrigger.TryGetValue(QualifiedName[Idx], out NextTrigger)) { LogError(Element, "Unknown trigger '{0}'", QualifiedName[Idx]); return; } if (NextTrigger.Parent != ParentTrigger) { LogError(Element, "Qualified name of trigger '{0}' is '{1}'", NextTrigger.Name, NextTrigger.QualifiedName); return; } ParentTrigger = NextTrigger; } // Get the name of the new trigger string Name = QualifiedName[QualifiedName.Length - 1]; // Create the new trigger ManualTrigger Trigger; if (!Graph.NameToTrigger.TryGetValue(Name, out Trigger)) { Trigger = new ManualTrigger(ParentTrigger, Name); Graph.NameToTrigger.Add(Name, Trigger); } else if (Trigger.Parent != ParentTrigger) { LogError(Element, "Conflicting parent for '{0}' - previously declared as '{1}', now '{2}'", Name, Trigger.QualifiedName, new ManualTrigger(ParentTrigger, Name).QualifiedName); return; } // Read the root BuildGraph element EnterScope(); foreach (ScriptElement ChildElement in Element.ChildNodes.OfType <ScriptElement>()) { switch (ChildElement.Name) { case "Property": ReadProperty(ChildElement); break; case "Local": ReadLocalProperty(ChildElement); break; case "Agent": ReadAgent(ChildElement, Trigger); break; case "Aggregate": ReadAggregate(ChildElement); break; case "Notifier": ReadNotifier(ChildElement); break; case "Warning": ReadDiagnostic(ChildElement, LogEventType.Warning, null, null, Trigger); break; case "Error": ReadDiagnostic(ChildElement, LogEventType.Error, null, null, Trigger); break; default: LogError(ChildElement, "Invalid element '{0}'", ChildElement.Name); break; } } LeaveScope(); } }
/// <summary> /// Checks whether this node is behind the given trigger /// </summary> /// <param name="Trigger">The trigger to check</param> /// <returns>True if the node is directly or indirectly behind the given trigger, false otherwise</returns> public bool IsBehind(ManualTrigger Trigger) { for(ManualTrigger OtherTrigger = ControllingTrigger; OtherTrigger != Trigger; OtherTrigger = OtherTrigger.Parent) { if(OtherTrigger == null) { return false; } } return true; }
/// <summary> /// Constructor /// </summary> /// <param name="InName">The name of this node</param> /// <param name="InInputs">Inputs that this node depends on</param> /// <param name="InOutputNames">Names of the outputs that this node produces</param> /// <param name="InInputDependencies">Nodes which this node is dependent on for its inputs</param> /// <param name="InOrderDependencies">Nodes which this node needs to run after. Should include all input dependencies.</param> /// <param name="InControllingTrigger">The trigger which this node is behind</param> /// <param name="InRequiredTokens">Optional tokens which must be required for this node to run</param> public Node(string InName, NodeOutput[] InInputs, string[] InOutputNames, Node[] InInputDependencies, Node[] InOrderDependencies, ManualTrigger InControllingTrigger, FileReference[] InRequiredTokens) { Name = InName; Inputs = InInputs; List<NodeOutput> AllOutputs = new List<NodeOutput>(); AllOutputs.Add(new NodeOutput(this, "#" + Name)); AllOutputs.AddRange(InOutputNames.Where(x => String.Compare(x, Name, StringComparison.InvariantCultureIgnoreCase) != 0).Select(x => new NodeOutput(this, x))); Outputs = AllOutputs.ToArray(); InputDependencies = InInputDependencies; OrderDependencies = InOrderDependencies; ControllingTrigger = InControllingTrigger; RequiredTokens = InRequiredTokens; }
/// <summary> /// Constructor /// </summary> /// <param name="InParent">The parent trigger</param> /// <param name="InName">Name of this trigger</param> public ManualTrigger(ManualTrigger InParent, string InName) { Parent = InParent; Name = InName; }
/// <summary> /// Main entry point for the BuildGraph command /// </summary> public override ExitCode Execute() { // Parse the command line parameters string ScriptFileName = ParseParamValue("Script", null); string TargetNames = ParseParamValue("Target", null); string DocumentationFileName = ParseParamValue("Documentation", null); string SchemaFileName = ParseParamValue("Schema", null); string ExportFileName = ParseParamValue("Export", null); string PreprocessedFileName = ParseParamValue("Preprocess", null); string SharedStorageDir = ParseParamValue("SharedStorageDir", null); string SingleNodeName = ParseParamValue("SingleNode", null); string TriggerName = ParseParamValue("Trigger", null); string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); bool bSkipTriggers = ParseParam("SkipTriggers"); string TokenSignature = ParseParamValue("TokenSignature", null); bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens"); bool bResume = SingleNodeName != null || ParseParam("Resume"); bool bListOnly = ParseParam("ListOnly"); bool bShowDiagnostics = ParseParam("ShowDiagnostics"); bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine; bool bPublicTasksOnly = ParseParam("PublicTasksOnly"); string ReportName = ParseParamValue("ReportName", null); GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions; if (ParseParam("ShowDeps")) { PrintOptions |= GraphPrintOptions.ShowDependencies; } if (ParseParam("ShowNotifications")) { PrintOptions |= GraphPrintOptions.ShowNotifications; } // Parse any specific nodes to clean List <string> CleanNodes = new List <string>(); foreach (string NodeList in ParseParamValues("CleanNode")) { foreach (string NodeName in NodeList.Split('+', ';')) { CleanNodes.Add(NodeName); } } // Set up the standard properties which build scripts might need Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); DefaultProperties["Branch"] = P4Enabled ? P4Env.Branch : "Unknown"; DefaultProperties["Depot"] = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown"; DefaultProperties["EscapedBranch"] = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown"; DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0"; DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0"; DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName; DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false"; DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString(); DefaultProperties["RestrictedFolderNames"] = String.Join(";", RestrictedFolders.Names); DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x))); // Attempt to read existing Build Version information BuildVersion Version; if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) { DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString(); DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString(); DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString(); DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString(); } // Add any additional custom arguments from the command line (of the form -Set:X=Y) Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (string Param in Params) { const string Prefix = "set:"; if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase)) { int EqualsIdx = Param.IndexOf('='); if (EqualsIdx >= 0) { Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1); } else { LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length)); } } } // Find all the tasks from the loaded assemblies Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>(); if (!FindAvailableTasks(NameToTask, bPublicTasksOnly)) { return(ExitCode.Error_Unknown); } // Generate documentation if (DocumentationFileName != null) { GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName)); return(ExitCode.Success); } // Create a schema for the given tasks ScriptSchema Schema = new ScriptSchema(NameToTask); if (SchemaFileName != null) { FileReference FullSchemaFileName = new FileReference(SchemaFileName); LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName); Schema.Export(FullSchemaFileName); if (ScriptFileName == null) { return(ExitCode.Success); } } // Check there was a script specified if (ScriptFileName == null) { LogError("Missing -Script= parameter for BuildGraph"); return(ExitCode.Error_Unknown); } // Read the script from disk Graph Graph; if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph)) { return(ExitCode.Error_Unknown); } // Create the temp storage handler DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage); if (!bResume) { Storage.CleanLocal(); } foreach (string CleanNode in CleanNodes) { Storage.CleanLocalNode(CleanNode); } // Convert the supplied target references into nodes HashSet <Node> TargetNodes = new HashSet <Node>(); if (TargetNames == null) { if (!bListOnly) { LogError("Missing -Target= parameter for BuildGraph"); return(ExitCode.Error_Unknown); } TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); } else { foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())) { Node[] Nodes; if (!Graph.TryResolveReference(TargetName, out Nodes)) { LogError("Target '{0}' is not in graph", TargetName); return(ExitCode.Error_Unknown); } TargetNodes.UnionWith(Nodes); } } // Try to acquire tokens for all the target nodes we want to build if (TokenSignature != null) { // Find all the lock files HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens)); // List out all the required tokens if (SingleNodeName == null) { CommandUtils.LogInformation("Required tokens:"); foreach (Node Node in TargetNodes) { foreach (FileReference RequiredToken in Node.RequiredTokens) { CommandUtils.LogInformation(" '{0}' requires {1}", Node, RequiredToken); } } } // Try to create all the lock files List <FileReference> CreatedTokens = new List <FileReference>(); if (!bListOnly) { CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature))); } // Find all the tokens that we don't have Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>(); foreach (FileReference RequiredToken in RequiredTokens) { string CurrentOwner = ReadTokenFile(RequiredToken); if (CurrentOwner != null && CurrentOwner != TokenSignature) { MissingTokens.Add(RequiredToken, CurrentOwner); } } // If we want to skip all the nodes with missing locks, adjust the target nodes to account for it if (MissingTokens.Count > 0) { if (bSkipTargetsWithoutTokens) { // Find all the nodes we're going to skip HashSet <Node> SkipNodes = new HashSet <Node>(); foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key)) { LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key); foreach (FileReference MissingToken in MissingTokensForBuild) { foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x))) { LogInformation(" {0}", SkipNode); } } } // Write a list of everything left over if (SkipNodes.Count > 0) { TargetNodes.ExceptWith(SkipNodes); LogInformation("Remaining target nodes:"); foreach (Node TargetNode in TargetNodes) { LogInformation(" {0}", TargetNode); } if (TargetNodes.Count == 0) { LogInformation(" None."); } } } else { foreach (KeyValuePair <FileReference, string> Pair in MissingTokens) { List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList(); LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value); } foreach (FileReference CreatedToken in CreatedTokens) { FileReference.Delete(CreatedToken); } return(ExitCode.Error_Unknown); } } } // Cull the graph to include only those nodes Graph.Select(TargetNodes); // Collapse any triggers in the graph which are marked to be skipped HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>(); if (bSkipTriggers) { SkipTriggers.UnionWith(Graph.NameToTrigger.Values); } else { foreach (string SkipTriggerName in SkipTriggerNames) { ManualTrigger SkipTrigger; if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return(ExitCode.Error_Unknown); } SkipTriggers.Add(SkipTrigger); } } Graph.SkipTriggers(SkipTriggers); // If a report for the whole build was requested, insert it into the graph if (ReportName != null) { Report NewReport = new Report(ReportName); NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); Graph.NameToReport.Add(ReportName, NewReport); } // Write out the preprocessed script if (PreprocessedFileName != null) { FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName); LogInformation("Writing {0}...", PreprocessedFileLocation); Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null); return(ExitCode.Success); } // Find the triggers which we are explicitly running. ManualTrigger Trigger = null; if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return(ExitCode.Error_Unknown); } // If we're just building a single node, find it Node SingleNode = null; if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode)) { LogError("Node '{0}' is not in the trimmed graph", SingleNodeName); return(ExitCode.Error_Unknown); } // If we just want to show the contents of the graph, do so and exit. if (bListOnly) { HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); } // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file. if (SingleNode == null && (!bListOnly || bShowDiagnostics)) { IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger); foreach (GraphDiagnostic Diagnostic in Diagnostics) { if (Diagnostic.EventType == LogEventType.Console) { CommandUtils.LogInformation(Diagnostic.Message); } else if (Diagnostic.EventType == LogEventType.Warning) { CommandUtils.LogWarning(Diagnostic.Message); } else { CommandUtils.LogError(Diagnostic.Message); } } if (Diagnostics.Any(x => x.EventType == LogEventType.Error)) { return(ExitCode.Error_Unknown); } } // Export the graph to a file if (ExportFileName != null) { HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes); return(ExitCode.Success); } // Execute the command if (!bListOnly) { if (SingleNode != null) { if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true)) { return(ExitCode.Error_Unknown); } } else { if (!BuildAllNodes(new JobContext(this), Graph, Storage)) { return(ExitCode.Error_Unknown); } } } return(ExitCode.Success); }
/// <summary> /// Reads the definition for an agent group. /// </summary> /// <param name="Element">Xml element to read the definition from</param> /// <param name="Trigger">The controlling trigger for nodes in this group</param> void ReadAgent(ScriptElement Element, ManualTrigger Trigger) { string Name; if (EvaluateCondition(Element) && TryReadObjectName(Element, out Name)) { // Read the valid agent types. This may be omitted if we're continuing an existing group. string[] Types = ReadListAttribute(Element, "Type"); // Create the group object, or continue an existing one AgentGroup Group; if (NameToGroup.TryGetValue(Name, out Group)) { if (Types.Length > 0 && Group.PossibleTypes.Length > 0) { string[] NewTypes = Group.PossibleTypes.Intersect(Types, StringComparer.InvariantCultureIgnoreCase).ToArray(); if (NewTypes.Length == 0) { LogError(Element, "No common agent types with previous agent definition"); } Group.PossibleTypes = NewTypes; } } else { if (Types.Length == 0) { LogError(Element, "Missing agent type for group '{0}'", Name); } Group = new AgentGroup(Name, Types); NameToGroup.Add(Name, Group); Graph.Groups.Add(Group); } // Process all the child elements. EnterScope(); foreach (ScriptElement ChildElement in Element.ChildNodes.OfType <ScriptElement>()) { switch (ChildElement.Name) { case "Property": ReadProperty(ChildElement); break; case "Local": ReadLocalProperty(ChildElement); break; case "Node": ReadNode(ChildElement, Group, Trigger); break; case "Aggregate": ReadAggregate(ChildElement); break; case "Warning": ReadDiagnostic(ChildElement, LogEventType.Warning, null, Group, Trigger); break; case "Error": ReadDiagnostic(ChildElement, LogEventType.Error, null, Group, Trigger); break; default: LogError(ChildElement, "Unexpected element type '{0}'", ChildElement.Name); break; } } LeaveScope(); } }
/// <summary> /// Export the build graph to a Json file, for parallel execution by the build system /// </summary> /// <param name="File">Output file to write</param> /// <param name="Trigger">The trigger whose nodes to run. Null for the default nodes.</param> /// <param name="CompletedNodes">Set of nodes which have been completed</param> public void Export(FileReference File, ManualTrigger Trigger, HashSet <Node> CompletedNodes) { // Find all the nodes which we're actually going to execute. We'll use this to filter the graph. HashSet <Node> NodesToExecute = new HashSet <Node>(); foreach (Node Node in Agents.SelectMany(x => x.Nodes)) { if (!CompletedNodes.Contains(Node) && Node.IsBehind(Trigger)) { NodesToExecute.Add(Node); } } // Open the output file using (JsonWriter JsonWriter = new JsonWriter(File.FullName)) { JsonWriter.WriteObjectStart(); // Write all the agents JsonWriter.WriteArrayStart("Groups"); foreach (Agent Agent in Agents) { Node[] Nodes = Agent.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray(); if (Nodes.Length > 0) { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Agent.Name); JsonWriter.WriteArrayStart("Agent Types"); foreach (string AgentType in Agent.PossibleTypes) { JsonWriter.WriteValue(AgentType); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteArrayStart("Nodes"); foreach (Node Node in Nodes) { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Node.Name); JsonWriter.WriteValue("DependsOn", String.Join(";", Node.GetDirectOrderDependencies().Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger))); JsonWriter.WriteValue("RunEarly", Node.bRunEarly); JsonWriter.WriteObjectStart("Notify"); JsonWriter.WriteValue("Default", String.Join(";", Node.NotifyUsers)); JsonWriter.WriteValue("Submitters", String.Join(";", Node.NotifySubmitters)); JsonWriter.WriteValue("Warnings", Node.bNotifyOnWarnings); JsonWriter.WriteObjectEnd(); JsonWriter.WriteObjectEnd(); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); // Write all the badges JsonWriter.WriteArrayStart("Badges"); foreach (Badge Badge in Badges) { Node[] Dependencies = Badge.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray(); if (Dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies); foreach (Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Badge.Name); if (!String.IsNullOrEmpty(Badge.Project)) { JsonWriter.WriteValue("Project", Badge.Project); } if (Badge.Change != 0) { JsonWriter.WriteValue("Change", Badge.Change); } JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name))); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); // Write all the triggers and reports. JsonWriter.WriteArrayStart("Reports"); foreach (Report Report in NameToReport.Values) { Node[] Dependencies = Report.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray(); if (Dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies); foreach (Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Report.Name); JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name))); JsonWriter.WriteValue("Notify", String.Join(";", Report.NotifyUsers)); JsonWriter.WriteValue("IsTrigger", false); JsonWriter.WriteObjectEnd(); } } foreach (ManualTrigger DownstreamTrigger in NameToTrigger.Values) { if (DownstreamTrigger.Parent == Trigger) { // Find all the nodes that this trigger is dependent on HashSet <Node> Dependencies = new HashSet <Node>(); foreach (Node NodeToExecute in NodesToExecute) { if (NodeToExecute.IsBehind(DownstreamTrigger)) { Dependencies.UnionWith(NodeToExecute.OrderDependencies.Where(x => x.ControllingTrigger == Trigger)); } } // Reduce that list to the smallest subset of direct dependencies HashSet <Node> DirectDependencies = new HashSet <Node>(Dependencies); foreach (Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } // Write out the object JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", DownstreamTrigger.Name); JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", Dependencies.Where(x => DirectDependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("Notify", String.Join(";", DownstreamTrigger.NotifyUsers)); JsonWriter.WriteValue("IsTrigger", true); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } }
/// <summary> /// Export the build graph to a Json file, for parallel execution by the build system /// </summary> /// <param name="File">Output file to write</param> /// <param name="Trigger">The trigger whose nodes to run. Null for the default nodes.</param> /// <param name="CompletedNodes">Set of nodes which have been completed</param> public void Export(FileReference File, ManualTrigger Trigger, HashSet<Node> CompletedNodes) { // Find all the nodes which we're actually going to execute. We'll use this to filter the graph. HashSet<Node> NodesToExecute = new HashSet<Node>(); foreach(Node Node in Agents.SelectMany(x => x.Nodes)) { if(!CompletedNodes.Contains(Node) && Node.IsBehind(Trigger)) { NodesToExecute.Add(Node); } } // Open the output file using(JsonWriter JsonWriter = new JsonWriter(File.FullName)) { JsonWriter.WriteObjectStart(); // Write all the agents JsonWriter.WriteArrayStart("Groups"); foreach(Agent Agent in Agents) { Node[] Nodes = Agent.Nodes.Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger).ToArray(); if(Nodes.Length > 0) { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Agent.Name); JsonWriter.WriteArrayStart("Agent Types"); foreach(string AgentType in Agent.PossibleTypes) { JsonWriter.WriteValue(AgentType); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteArrayStart("Nodes"); foreach(Node Node in Nodes) { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Node.Name); JsonWriter.WriteValue("DependsOn", String.Join(";", Node.GetDirectOrderDependencies().Where(x => NodesToExecute.Contains(x) && x.ControllingTrigger == Trigger))); JsonWriter.WriteObjectStart("Notify"); JsonWriter.WriteValue("Default", String.Join(";", Node.NotifyUsers)); JsonWriter.WriteValue("Submitters", String.Join(";", Node.NotifySubmitters)); JsonWriter.WriteValue("Warnings", Node.bNotifyOnWarnings); JsonWriter.WriteObjectEnd(); JsonWriter.WriteObjectEnd(); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); // Write all the badges JsonWriter.WriteArrayStart("Badges"); foreach (Badge Badge in Badges) { Node[] Dependencies = Badge.Nodes.Where(x => NodesToExecute.Contains(x)).ToArray(); if (Dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet<Node> DirectDependencies = new HashSet<Node>(Dependencies); foreach (Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Badge.Name); if (!String.IsNullOrEmpty(Badge.Project)) { JsonWriter.WriteValue("Project", Badge.Project); } JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name))); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); // Write all the triggers and reports. JsonWriter.WriteArrayStart("Reports"); foreach (Report Report in NameToReport.Values) { Node[] Dependencies = Report.Nodes.Where(x => NodesToExecute.Contains(x)).ToArray(); if (Dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet<Node> DirectDependencies = new HashSet<Node>(Dependencies); foreach (Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", Report.Name); JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", DirectDependencies.Select(x => x.Name))); JsonWriter.WriteValue("Notify", String.Join(";", Report.NotifyUsers)); JsonWriter.WriteValue("IsTrigger", false); JsonWriter.WriteObjectEnd(); } } foreach (ManualTrigger DownstreamTrigger in NameToTrigger.Values) { if(DownstreamTrigger.Parent == Trigger) { // Find all the nodes that this trigger is dependent on HashSet<Node> Dependencies = new HashSet<Node>(); foreach(Node NodeToExecute in NodesToExecute) { if(NodeToExecute.IsBehind(DownstreamTrigger)) { Dependencies.UnionWith(NodeToExecute.OrderDependencies.Where(x => x.ControllingTrigger == Trigger)); } } // Reduce that list to the smallest subset of direct dependencies HashSet<Node> DirectDependencies = new HashSet<Node>(Dependencies); foreach(Node Dependency in Dependencies) { DirectDependencies.ExceptWith(Dependency.OrderDependencies); } // Write out the object JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Name", DownstreamTrigger.Name); JsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => Dependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("DirectDependencies", String.Join(";", Dependencies.Where(x => DirectDependencies.Contains(x)).Select(x => x.Name))); JsonWriter.WriteValue("Notify", String.Join(";", DownstreamTrigger.NotifyUsers)); JsonWriter.WriteValue("IsTrigger", true); JsonWriter.WriteObjectEnd(); } } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } }