public virtual void RetrieveBuildProducts(string SharedStorageDir) { CommandUtils.Log("***** Retrieving GUBP Node {0}", Name); try { BuildProducts = TempStorage.RetrieveFromTempStorage(SharedStorageDir, Name); } catch (Exception Ex) { throw new AutomationException(Ex, "Build Products cannot be found for node {0}", Name); } }
public virtual void RetrieveBuildProducts(TempStorageNodeInfo TempStorageNodeInfo) { CommandUtils.Log("***** Retrieving GUBP Node {0} from {1}", Name, TempStorageNodeInfo.GetRelativeDirectory()); try { BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, CommandUtils.CmdEnv.LocalRoot); } catch (Exception Ex) { throw new AutomationException(Ex, "Build Products cannot be found for node {0}", Name); } }
/// <summary> /// Find all the nodes in the graph which are already completed /// </summary> /// <param name="Graph">The graph instance</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> HashSet <Node> FindCompletedNodes(Graph Graph, TempStorage Storage) { HashSet <Node> CompletedNodes = new HashSet <Node>(); foreach (Node Node in Graph.Agents.SelectMany(x => x.Nodes)) { if (Storage.IsComplete(Node.Name)) { CompletedNodes.Add(Node); } } return(CompletedNodes); }
/// <summary> /// Builds all the nodes in the graph /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph instance</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> /// <returns>True if everything built successfully</returns> bool BuildAllNodes(JobContext Job, Graph Graph, TempStorage Storage) { // Build a flat list of nodes to execute, in order Node[] NodesToExecute = Graph.Agents.SelectMany(x => x.Nodes).ToArray(); // Check the integrity of any local nodes that have been completed. It's common to run formal builds locally between regular development builds, so we may have // stale local state. Rather than failing later, detect and clean them up now. HashSet <Node> CleanedNodes = new HashSet <Node>(); foreach (Node NodeToExecute in NodesToExecute) { if (NodeToExecute.InputDependencies.Any(x => CleanedNodes.Contains(x)) || !Storage.CheckLocalIntegrity(NodeToExecute.Name, NodeToExecute.Outputs.Select(x => x.TagName))) { Storage.CleanLocalNode(NodeToExecute.Name); CleanedNodes.Add(NodeToExecute); } } // Execute them in order int NodeIdx = 0; foreach (Node NodeToExecute in NodesToExecute) { LogInformation("****** [{0}/{1}] {2}", ++NodeIdx, NodesToExecute.Length, NodeToExecute.Name); if (!Storage.IsComplete(NodeToExecute.Name)) { LogInformation(""); if (!BuildNode(Job, Graph, NodeToExecute, Storage, false)) { return(false); } LogInformation(""); } } return(true); }
public void SaveStatus(BuildNode NodeToDo, string Suffix, string NodeStoreName, bool bSaveSharedTempStorage, string GameNameIfAny, string JobStepIDForFailure = null) { string Contents = "Just a status record: " + Suffix; if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine) { try { Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true); } catch (Exception Ex) { CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to get properties for jobstep to save them."); CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } } string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", NodeToDo.Name + Suffix + ".log"); CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess)); CommandUtils.WriteAllText(RecordOfSuccess, Contents); TempStorage.StoreToTempStorage(CommandUtils.CmdEnv, NodeStoreName + Suffix, new List <string> { RecordOfSuccess }, !bSaveSharedTempStorage, GameNameIfAny); }
public void SaveStatus(TempStorageNodeInfo TempStorageNodeInfo, bool bSaveSharedTempStorage, string JobStepIDForFailure = null) { string Contents = "Just a status record: " + TempStorageNodeInfo.NodeStorageName; if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine) { try { Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true); } catch (Exception Ex) { CommandUtils.LogWarning("Failed to get properties for jobstep to save them."); CommandUtils.LogWarning(LogUtils.FormatException(Ex)); } } string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", TempStorageNodeInfo.NodeStorageName + ".log"); CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess)); CommandUtils.WriteAllText(RecordOfSuccess, Contents); TempStorage.StoreToTempStorage(TempStorageNodeInfo, new List <string> { RecordOfSuccess }, !bSaveSharedTempStorage, CommandUtils.CmdEnv.LocalRoot); }
/// <summary> /// Main entry point for the BuildGraph command /// </summary> public override ExitCode Execute() { // Parse the command line parameters string ScriptFileName = ParseParamValue("Script", null); if (ScriptFileName == null) { LogError("Missing -Script= parameter for BuildGraph"); return(ExitCode.Error_Unknown); } string TargetNames = ParseParamValue("Target", null); if (TargetNames == null) { LogError("Missing -Target= parameter for BuildGraph"); return(ExitCode.Error_Unknown); } string SchemaFileName = ParseParamValue("Schema", null); string ExportFileName = ParseParamValue("Export", null); string SharedStorageDir = ParseParamValue("SharedStorageDir", null); string SingleNodeName = ParseParamValue("SingleNode", null); string[] TriggerNames = ParseParamValue("Trigger", "").Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); bool bSkipTriggers = ParseParam("SkipTriggers"); bool bClean = ParseParam("Clean"); bool bListOnly = ParseParam("ListOnly"); bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine; bool bPublicTasksOnly = ParseParam("PublicTasksOnly"); GraphPrintOptions PrintOptions = 0; if (ParseParam("ShowDeps")) { PrintOptions |= GraphPrintOptions.ShowDependencies; } if (ParseParam("ShowNotifications")) { PrintOptions |= GraphPrintOptions.ShowNotifications; } // Parse any specific nodes to clean List <string> CleanNodes = new List <string>(); foreach (string NodeList in ParseParamValues("CleanNode")) { foreach (string NodeName in NodeList.Split('+')) { CleanNodes.Add(NodeName); } } // Read any environment variables Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (DictionaryEntry Entry in Environment.GetEnvironmentVariables()) { DefaultProperties[Entry.Key.ToString()] = Entry.Value.ToString(); } // Add any additional custom parameters from the command line (of the form -Set:X=Y) foreach (string Param in Params) { const string Prefix = "set:"; if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase)) { int EqualsIdx = Param.IndexOf('='); if (EqualsIdx >= 0) { DefaultProperties[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1); } else { LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length)); } } } // Set up the standard properties which build scripts might need DefaultProperties["Branch"] = P4Enabled? P4Env.BuildRootP4 : "Unknown"; DefaultProperties["EscapedBranch"] = P4Enabled? P4Env.BuildRootEscaped : "Unknown"; DefaultProperties["Change"] = P4Enabled? P4Env.Changelist.ToString() : "0"; DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName; DefaultProperties["IsBuildMachine"] = IsBuildMachine? "true" : "false"; DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString(); // Find all the tasks from the loaded assemblies Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>(); if (!FindAvailableTasks(NameToTask, bPublicTasksOnly)) { return(ExitCode.Error_Unknown); } // Create a schema for the given tasks ScriptSchema Schema = new ScriptSchema(NameToTask); if (SchemaFileName != null) { Schema.Export(new FileReference(SchemaFileName)); } // Read the script from disk Graph Graph; if (!ScriptReader.TryRead(new FileReference(ScriptFileName), DefaultProperties, Schema, out Graph)) { return(ExitCode.Error_Unknown); } // Create the temp storage handler DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage); if (bClean) { Storage.CleanLocal(); } foreach (string CleanNode in CleanNodes) { Storage.CleanLocalNode(CleanNode); } // Convert the supplied target references into nodes HashSet <Node> TargetNodes = new HashSet <Node>(); foreach (string TargetName in TargetNames.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())) { Node[] Nodes; if (!Graph.TryResolveReference(TargetName, out Nodes)) { LogError("Target '{0}' is not in graph", TargetName); return(ExitCode.Error_Unknown); } TargetNodes.UnionWith(Nodes); } // Cull the graph to include only those nodes Graph.Select(TargetNodes); // Find the triggers which are explicitly activated, and all of its upstream triggers. HashSet <ManualTrigger> Triggers = new HashSet <ManualTrigger>(); foreach (string TriggerName in TriggerNames) { ManualTrigger Trigger; if (!Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return(ExitCode.Error_Unknown); } while (Trigger != null) { Triggers.Add(Trigger); Trigger = Trigger.Parent; } } if (bSkipTriggers) { Triggers.UnionWith(Graph.NameToTrigger.Values); } // If we're just building a single node, find it Node SingleNode = null; if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode)) { LogError("Node '{0}' is not in the trimmed graph", SingleNodeName); return(ExitCode.Error_Unknown); } // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system. if (SingleNode == null) { IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == null || Triggers.Contains(x.EnclosingTrigger)); foreach (GraphDiagnostic Diagnostic in Diagnostics) { if (Diagnostic.EventType == LogEventType.Warning) { CommandUtils.LogWarning(Diagnostic.Message); } else { CommandUtils.LogError(Diagnostic.Message); } } if (Diagnostics.Any(x => x.EventType == LogEventType.Error)) { return(ExitCode.Error_Unknown); } } // Execute the command if (bListOnly) { HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); } else if (ExportFileName != null) { HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); Graph.Export(new FileReference(ExportFileName), Triggers, CompletedNodes); } else if (SingleNode != null) { if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true)) { return(ExitCode.Error_Unknown); } } else { if (!BuildAllNodes(new JobContext(this), Graph, Storage)) { return(ExitCode.Error_Unknown); } } return(ExitCode.Success); }
/// <summary> /// Build a node /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param> /// <param name="Node">The node to build</param> /// <returns>True if the node built successfully, false otherwise.</returns> bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner) { // Create a mapping from tag name to the files it contains, and seed it with invalid entries for everything in the graph Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >(); foreach (NodeOutput Output in Graph.Groups.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs)) { TagNameToFileSet[Output.Name] = null; } // Fetch all the input dependencies for this node, and fill in the tag names with those files DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); foreach (NodeOutput Input in Node.Inputs) { TempStorageManifest Manifest = Storage.Retreive(Input.ProducingNode.Name, Input.Name); TagNameToFileSet[Input.Name] = new HashSet <FileReference>(Manifest.Files.Select(x => x.ToFileReference(RootDir))); } // Add placeholder outputs for the current node foreach (NodeOutput Output in Node.Outputs) { TagNameToFileSet[Output.Name] = new HashSet <FileReference>(); } // Execute the node if (bWithBanner) { Console.WriteLine(); CommandUtils.Log("========== Starting: {0} ==========", Node.Name); } if (!Node.Build(Job, TagNameToFileSet)) { return(false); } if (bWithBanner) { CommandUtils.Log("========== Finished: {0} ==========", Node.Name); Console.WriteLine(); } // Determine all the outputs which are required to be copied to temp storage (because they're referenced by nodes in another agent group) HashSet <NodeOutput> ReferencedOutputs = new HashSet <NodeOutput>(); foreach (AgentGroup Group in Graph.Groups) { bool bSameGroup = Group.Nodes.Contains(Node); foreach (Node OtherNode in Group.Nodes) { if (!bSameGroup || Node.ControllingTrigger != OtherNode.ControllingTrigger) { ReferencedOutputs.UnionWith(OtherNode.Inputs); } } } // Publish all the outputs foreach (NodeOutput Output in Node.Outputs) { Storage.Archive(Node.Name, Output.Name, TagNameToFileSet[Output.Name].ToArray(), ReferencedOutputs.Contains(Output)); } // Mark the node as succeeded Storage.MarkAsComplete(Node.Name); return(true); }
/// <summary> /// Builds all the nodes in the graph /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph instance</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> /// <returns>True if everything built successfully</returns> bool BuildAllNodes(JobContext Job, Graph Graph, TempStorage Storage) { // Build a flat list of nodes to execute, in order Node[] NodesToExecute = Graph.Agents.SelectMany(x => x.Nodes).ToArray(); // Check the integrity of any local nodes that have been completed. It's common to run formal builds locally between regular development builds, so we may have // stale local state. Rather than failing later, detect and clean them up now. HashSet<Node> CleanedNodes = new HashSet<Node>(); foreach(Node NodeToExecute in NodesToExecute) { if(NodeToExecute.InputDependencies.Any(x => CleanedNodes.Contains(x)) || !Storage.CheckLocalIntegrity(NodeToExecute.Name, NodeToExecute.Outputs.Select(x => x.TagName))) { Storage.CleanLocalNode(NodeToExecute.Name); CleanedNodes.Add(NodeToExecute); } } // Execute them in order int NodeIdx = 0; foreach(Node NodeToExecute in NodesToExecute) { Log("****** [{0}/{1}] {2}", ++NodeIdx, NodesToExecute.Length, NodeToExecute.Name); if(!Storage.IsComplete(NodeToExecute.Name)) { Log(""); if(!BuildNode(Job, Graph, NodeToExecute, Storage, false)) { return false; } Log(""); } } return true; }
/// <summary> /// Run the automated tests /// </summary> public override void ExecuteBuild() { // Get all the shared directories DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local"); CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared"); CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName); CommandUtils.DeleteDirectoryContents(SharedDir.FullName); DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working"); CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); // Create the temp storage object TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true); // Create a working directory, and copy some source files into it DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime"); if(!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true)) { throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName); } // Save the default output Dictionary<FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f'); TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false); Dictionary<FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i'); TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true); // Check both outputs are still ok TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null); CheckManifest(WorkingDir, DefaultManifest, DefaultOutput); TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifest, NamedOutput); // Delete local temp storage and the working directory and try again CommandUtils.Log("Clearing local folders..."); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); // First output should fail CommandUtils.Log("Checking default manifest is now unavailable..."); bool bGotManifest = false; try { TempStore.Retreive("TestNode", null); } catch { bGotManifest = false; } if(bGotManifest) { throw new AutomationException("Did not expect shared temp storage manifest to exist"); } // Second one should be fine TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput); }
public virtual void ArchiveBuildProducts(string SharedStorageDir, bool bWriteToSharedStorage) { TempStorage.StoreToTempStorage(Name, BuildProducts, SharedStorageDir, bWriteToSharedStorage); }
/// <summary> /// Run the automated tests /// </summary> public override void ExecuteBuild() { // Get all the shared directories DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); DirectoryReference LocalDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Local"); CommandUtils.CreateDirectory_NoExceptions(LocalDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); DirectoryReference SharedDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Shared"); CommandUtils.CreateDirectory_NoExceptions(SharedDir.FullName); CommandUtils.DeleteDirectoryContents(SharedDir.FullName); DirectoryReference WorkingDir = DirectoryReference.Combine(RootDir, "Engine", "Saved", "TestTempStorage-Working"); CommandUtils.CreateDirectory_NoExceptions(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); // Create the temp storage object TempStorage TempStore = new TempStorage(WorkingDir, LocalDir, SharedDir, true); // Create a working directory, and copy some source files into it DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Engine", "Source", "Runtime"); if (!CommandUtils.CopyDirectory_NoExceptions(SourceDir.FullName, WorkingDir.FullName, true)) { throw new AutomationException("Couldn't copy {0} to {1}", SourceDir.FullName, WorkingDir.FullName); } // Save the default output Dictionary <FileReference, DateTime> DefaultOutput = SelectFiles(WorkingDir, 'a', 'f'); TempStore.Archive("TestNode", null, DefaultOutput.Keys.ToArray(), false); Dictionary <FileReference, DateTime> NamedOutput = SelectFiles(WorkingDir, 'g', 'i'); TempStore.Archive("TestNode", "NamedOutput", NamedOutput.Keys.ToArray(), true); // Check both outputs are still ok TempStorageManifest DefaultManifest = TempStore.Retreive("TestNode", null); CheckManifest(WorkingDir, DefaultManifest, DefaultOutput); TempStorageManifest NamedManifest = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifest, NamedOutput); // Delete local temp storage and the working directory and try again CommandUtils.Log("Clearing local folders..."); CommandUtils.DeleteDirectoryContents(WorkingDir.FullName); CommandUtils.DeleteDirectoryContents(LocalDir.FullName); // First output should fail CommandUtils.Log("Checking default manifest is now unavailable..."); bool bGotManifest = false; try { TempStore.Retreive("TestNode", null); } catch { bGotManifest = false; } if (bGotManifest) { throw new AutomationException("Did not expect shared temp storage manifest to exist"); } // Second one should be fine TempStorageManifest NamedManifestFromShared = TempStore.Retreive("TestNode", "NamedOutput"); CheckManifest(WorkingDir, NamedManifestFromShared, NamedOutput); }
/// <summary> /// Main entry point for the BuildGraph command /// </summary> public override ExitCode Execute() { // Parse the command line parameters string ScriptFileName = ParseParamValue("Script", null); string TargetNames = ParseParamValue("Target", null); string DocumentationFileName = ParseParamValue("Documentation", null); string SchemaFileName = ParseParamValue("Schema", null); string ExportFileName = ParseParamValue("Export", null); string PreprocessedFileName = ParseParamValue("Preprocess", null); string SharedStorageDir = ParseParamValue("SharedStorageDir", null); string SingleNodeName = ParseParamValue("SingleNode", null); string TriggerName = ParseParamValue("Trigger", null); string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); bool bSkipTriggers = ParseParam("SkipTriggers"); string TokenSignature = ParseParamValue("TokenSignature", null); bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens"); bool bClearHistory = ParseParam("Clean") || ParseParam("ClearHistory"); bool bListOnly = ParseParam("ListOnly"); bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine; bool bPublicTasksOnly = ParseParam("PublicTasksOnly"); string ReportName = ParseParamValue("ReportName", null); GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions; if(ParseParam("ShowDeps")) { PrintOptions |= GraphPrintOptions.ShowDependencies; } if(ParseParam("ShowNotifications")) { PrintOptions |= GraphPrintOptions.ShowNotifications; } // Parse any specific nodes to clean List<string> CleanNodes = new List<string>(); foreach(string NodeList in ParseParamValues("CleanNode")) { foreach(string NodeName in NodeList.Split('+', ';')) { CleanNodes.Add(NodeName); } } // Set up the standard properties which build scripts might need Dictionary<string, string> DefaultProperties = new Dictionary<string,string>(StringComparer.InvariantCultureIgnoreCase); DefaultProperties["Branch"] = P4Enabled ? P4Env.BuildRootP4 : "Unknown"; DefaultProperties["EscapedBranch"] = P4Enabled ? P4Env.BuildRootEscaped : "Unknown"; DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0"; DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0"; DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName; DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false"; DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString(); // Attempt to read existing Build Version information BuildVersion Version; if (BuildVersion.TryRead(FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Build", "Build.version").FullName, out Version)) { DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString(); DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString(); DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString(); } // Add any additional custom arguments from the command line (of the form -Set:X=Y) Dictionary<string, string> Arguments = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (string Param in Params) { const string Prefix = "set:"; if(Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase)) { int EqualsIdx = Param.IndexOf('='); if(EqualsIdx >= 0) { Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1); } else { LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length)); } } } // Find all the tasks from the loaded assemblies Dictionary<string, ScriptTask> NameToTask = new Dictionary<string,ScriptTask>(); if(!FindAvailableTasks(NameToTask, bPublicTasksOnly)) { return ExitCode.Error_Unknown; } // Generate documentation if(DocumentationFileName != null) { GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName)); return ExitCode.Success; } // Create a schema for the given tasks ScriptSchema Schema = new ScriptSchema(NameToTask); if(SchemaFileName != null) { FileReference FullSchemaFileName = new FileReference(SchemaFileName); Log("Writing schema to {0}...", FullSchemaFileName.FullName); Schema.Export(FullSchemaFileName); if(ScriptFileName == null) { return ExitCode.Success; } } // Check there was a script specified if(ScriptFileName == null) { LogError("Missing -Script= parameter for BuildGraph"); return ExitCode.Error_Unknown; } // Read the script from disk Graph Graph; if(!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph)) { return ExitCode.Error_Unknown; } // Create the temp storage handler DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage); if(bClearHistory) { Storage.CleanLocal(); } foreach(string CleanNode in CleanNodes) { Storage.CleanLocalNode(CleanNode); } // Convert the supplied target references into nodes HashSet<Node> TargetNodes = new HashSet<Node>(); if(TargetNames == null) { if(!bListOnly) { LogError("Missing -Target= parameter for BuildGraph"); return ExitCode.Error_Unknown; } TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); } else { foreach(string TargetName in TargetNames.Split(new char[]{ '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())) { Node[] Nodes; if(!Graph.TryResolveReference(TargetName, out Nodes)) { LogError("Target '{0}' is not in graph", TargetName); return ExitCode.Error_Unknown; } TargetNodes.UnionWith(Nodes); } } // Try to acquire tokens for all the target nodes we want to build if(TokenSignature != null) { // Find all the lock files HashSet<FileReference> RequiredTokens = new HashSet<FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens)); // List out all the required tokens if(SingleNodeName == null) { CommandUtils.Log("Required tokens:"); foreach(Node Node in TargetNodes) { foreach(FileReference RequiredToken in Node.RequiredTokens) { CommandUtils.Log(" '{0}' requires {1}", Node, RequiredToken); } } } // Try to create all the lock files List<FileReference> CreatedTokens = new List<FileReference>(); if(!bListOnly) { CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature))); } // Find all the tokens that we don't have Dictionary<FileReference, string> MissingTokens = new Dictionary<FileReference, string>(); foreach(FileReference RequiredToken in RequiredTokens) { string CurrentOwner = ReadTokenFile(RequiredToken); if(CurrentOwner != null && CurrentOwner != TokenSignature) { MissingTokens.Add(RequiredToken, CurrentOwner); } } // If we want to skip all the nodes with missing locks, adjust the target nodes to account for it if(MissingTokens.Count > 0) { if(bSkipTargetsWithoutTokens) { // Find all the nodes we're going to skip HashSet<Node> SkipNodes = new HashSet<Node>(); foreach(IGrouping<string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key)) { Log("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key); foreach(FileReference MissingToken in MissingTokensForBuild) { foreach(Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x))) { Log(" {0}", SkipNode); } } } // Write a list of everything left over if(SkipNodes.Count > 0) { TargetNodes.ExceptWith(SkipNodes); Log("Remaining target nodes:"); foreach(Node TargetNode in TargetNodes) { Log(" {0}", TargetNode); } if(TargetNodes.Count == 0) { Log(" None."); } } } else { foreach(KeyValuePair<FileReference, string> Pair in MissingTokens) { List<Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList(); LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value); } foreach(FileReference CreatedToken in CreatedTokens) { CreatedToken.Delete(); } return ExitCode.Error_Unknown; } } } // Cull the graph to include only those nodes Graph.Select(TargetNodes); // Collapse any triggers in the graph which are marked to be skipped HashSet<ManualTrigger> SkipTriggers = new HashSet<ManualTrigger>(); if(bSkipTriggers) { SkipTriggers.UnionWith(Graph.NameToTrigger.Values); } else { foreach(string SkipTriggerName in SkipTriggerNames) { ManualTrigger SkipTrigger; if(!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return ExitCode.Error_Unknown; } SkipTriggers.Add(SkipTrigger); } } Graph.SkipTriggers(SkipTriggers); // If a report for the whole build was requested, insert it into the graph if (ReportName != null) { Report NewReport = new Report(ReportName); NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); Graph.NameToReport.Add(ReportName, NewReport); } // Write out the preprocessed script if (PreprocessedFileName != null) { Graph.Write(new FileReference(PreprocessedFileName), (SchemaFileName != null)? new FileReference(SchemaFileName) : null); } // Find the triggers which we are explicitly running. ManualTrigger Trigger = null; if(TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return ExitCode.Error_Unknown; } // If we're just building a single node, find it Node SingleNode = null; if(SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode)) { LogError("Node '{0}' is not in the trimmed graph", SingleNodeName); return ExitCode.Error_Unknown; } // If we just want to show the contents of the graph, do so and exit. if(bListOnly) { HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); return ExitCode.Success; } // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file. if(SingleNode == null) { IEnumerable<GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger); foreach(GraphDiagnostic Diagnostic in Diagnostics) { if(Diagnostic.EventType == LogEventType.Warning) { CommandUtils.LogWarning(Diagnostic.Message); } else { CommandUtils.LogError(Diagnostic.Message); } } if(Diagnostics.Any(x => x.EventType == LogEventType.Error)) { return ExitCode.Error_Unknown; } } // Execute the command if(ExportFileName != null) { HashSet<Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes); } else if(SingleNode != null) { if(!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true)) { return ExitCode.Error_Unknown; } } else { if(!BuildAllNodes(new JobContext(this), Graph, Storage)) { return ExitCode.Error_Unknown; } } return ExitCode.Success; }
/// <summary> /// Build a node /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param> /// <param name="Node">The node to build</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> /// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param> /// <returns>True if the node built successfully, false otherwise.</returns> bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner) { DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); // Create the mapping of tag names to file sets Dictionary<string, HashSet<FileReference>> TagNameToFileSet = new Dictionary<string,HashSet<FileReference>>(); // Read all the input tags for this node, and build a list of referenced input storage blocks HashSet<TempStorageBlock> InputStorageBlocks = new HashSet<TempStorageBlock>(); foreach(NodeOutput Input in Node.Inputs) { TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName); TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir); InputStorageBlocks.UnionWith(FileList.Blocks); } // Read all the input storage blocks, keeping track of which block each file came from Dictionary<FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary<FileReference, TempStorageBlock>(); foreach(TempStorageBlock InputStorageBlock in InputStorageBlocks) { TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName); foreach(FileReference File in Manifest.Files.Select(x => x.ToFileReference(RootDir))) { TempStorageBlock CurrentStorageBlock; if(FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock)) { LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock); } FileToStorageBlock[File] = InputStorageBlock; } } // Add placeholder outputs for the current node foreach(NodeOutput Output in Node.Outputs) { TagNameToFileSet.Add(Output.TagName, new HashSet<FileReference>()); } // Execute the node if(bWithBanner) { Console.WriteLine(); CommandUtils.Log("========== Starting: {0} ==========", Node.Name); } if(!Node.Build(Job, TagNameToFileSet)) { return false; } if(bWithBanner) { CommandUtils.Log("========== Finished: {0} ==========", Node.Name); Console.WriteLine(); } // Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent) HashSet<FileReference> ReferencedOutputFiles = new HashSet<FileReference>(); foreach(Agent Agent in Graph.Agents) { bool bSameAgent = Agent.Nodes.Contains(Node); foreach(Node OtherNode in Agent.Nodes) { if(!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger) { foreach(NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node)) { ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]); } } } } // Find a block name for all new outputs Dictionary<FileReference, string> FileToOutputName = new Dictionary<FileReference, string>(); foreach(NodeOutput Output in Node.Outputs) { HashSet<FileReference> Files = TagNameToFileSet[Output.TagName]; foreach(FileReference File in Files) { if(!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir)) { if(Output == Node.DefaultOutput) { if(!FileToOutputName.ContainsKey(File)) { FileToOutputName[File] = ""; } } else { string OutputName; if(FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0) { FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1)); } else { FileToOutputName[File] = Output.TagName.Substring(1); } } } } } // Invert the dictionary to make a mapping of storage block to the files each contains Dictionary<string, HashSet<FileReference>> OutputStorageBlockToFiles = new Dictionary<string, HashSet<FileReference>>(); foreach(KeyValuePair<FileReference, string> Pair in FileToOutputName) { HashSet<FileReference> Files; if(!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files)) { Files = new HashSet<FileReference>(); OutputStorageBlockToFiles.Add(Pair.Value, Files); } Files.Add(Pair.Key); } // Write all the storage blocks, and update the mapping from file to storage block foreach(KeyValuePair<string, HashSet<FileReference>> Pair in OutputStorageBlockToFiles) { TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key); foreach(FileReference File in Pair.Value) { FileToStorageBlock.Add(File, OutputBlock); } Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x))); } // Publish all the output tags foreach(NodeOutput Output in Node.Outputs) { HashSet<FileReference> Files = TagNameToFileSet[Output.TagName]; HashSet<TempStorageBlock> StorageBlocks = new HashSet<TempStorageBlock>(); foreach(FileReference File in Files) { TempStorageBlock StorageBlock; if(FileToStorageBlock.TryGetValue(File, out StorageBlock)) { StorageBlocks.Add(StorageBlock); } } Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray()); } // Mark the node as succeeded Storage.MarkAsComplete(Node.Name); return true; }
/// <summary> /// Build a node /// </summary> /// <param name="Job">Information about the current job</param> /// <param name="Graph">The graph to which the node belongs. Used to determine which outputs need to be transferred to temp storage.</param> /// <param name="Node">The node to build</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> /// <param name="bWithBanner">Whether to write a banner before and after this node's log output</param> /// <returns>True if the node built successfully, false otherwise.</returns> bool BuildNode(JobContext Job, Graph Graph, Node Node, TempStorage Storage, bool bWithBanner) { DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); // Create the mapping of tag names to file sets Dictionary <string, HashSet <FileReference> > TagNameToFileSet = new Dictionary <string, HashSet <FileReference> >(); // Read all the input tags for this node, and build a list of referenced input storage blocks HashSet <TempStorageBlock> InputStorageBlocks = new HashSet <TempStorageBlock>(); foreach (NodeOutput Input in Node.Inputs) { TempStorageFileList FileList = Storage.ReadFileList(Input.ProducingNode.Name, Input.TagName); TagNameToFileSet[Input.TagName] = FileList.ToFileSet(RootDir); InputStorageBlocks.UnionWith(FileList.Blocks); } // Read the manifests for all the input storage blocks Dictionary <TempStorageBlock, TempStorageManifest> InputManifests = new Dictionary <TempStorageBlock, TempStorageManifest>(); foreach (TempStorageBlock InputStorageBlock in InputStorageBlocks) { TempStorageManifest Manifest = Storage.Retreive(InputStorageBlock.NodeName, InputStorageBlock.OutputName); InputManifests[InputStorageBlock] = Manifest; } // Read all the input storage blocks, keeping track of which block each file came from Dictionary <FileReference, TempStorageBlock> FileToStorageBlock = new Dictionary <FileReference, TempStorageBlock>(); foreach (KeyValuePair <TempStorageBlock, TempStorageManifest> Pair in InputManifests) { TempStorageBlock InputStorageBlock = Pair.Key; foreach (FileReference File in Pair.Value.Files.Select(x => x.ToFileReference(RootDir))) { TempStorageBlock CurrentStorageBlock; if (FileToStorageBlock.TryGetValue(File, out CurrentStorageBlock) && !TempStorage.IsDuplicateBuildProduct(File)) { LogError("File '{0}' was produced by {1} and {2}", File, InputStorageBlock, CurrentStorageBlock); } FileToStorageBlock[File] = InputStorageBlock; } } // Add placeholder outputs for the current node foreach (NodeOutput Output in Node.Outputs) { TagNameToFileSet.Add(Output.TagName, new HashSet <FileReference>()); } // Execute the node if (bWithBanner) { Console.WriteLine(); CommandUtils.LogInformation("========== Starting: {0} ==========", Node.Name); } if (!Node.Build(Job, TagNameToFileSet)) { return(false); } if (bWithBanner) { CommandUtils.LogInformation("========== Finished: {0} ==========", Node.Name); Console.WriteLine(); } // Check that none of the inputs have been clobbered Dictionary <string, string> ModifiedFiles = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (TempStorageFile File in InputManifests.Values.SelectMany(x => x.Files)) { string Message; if (!ModifiedFiles.ContainsKey(File.RelativePath) && !File.Compare(CommandUtils.RootDirectory, out Message)) { ModifiedFiles.Add(File.RelativePath, Message); } } if (ModifiedFiles.Count > 0) { throw new AutomationException("Build {0} from a previous step have been modified:\n{1}", (ModifiedFiles.Count == 1)? "product" : "products", String.Join("\n", ModifiedFiles.Select(x => x.Value))); } // Determine all the output files which are required to be copied to temp storage (because they're referenced by nodes in another agent) HashSet <FileReference> ReferencedOutputFiles = new HashSet <FileReference>(); foreach (Agent Agent in Graph.Agents) { bool bSameAgent = Agent.Nodes.Contains(Node); foreach (Node OtherNode in Agent.Nodes) { if (!bSameAgent || Node.ControllingTrigger != OtherNode.ControllingTrigger) { foreach (NodeOutput Input in OtherNode.Inputs.Where(x => x.ProducingNode == Node)) { ReferencedOutputFiles.UnionWith(TagNameToFileSet[Input.TagName]); } } } } // Find a block name for all new outputs Dictionary <FileReference, string> FileToOutputName = new Dictionary <FileReference, string>(); foreach (NodeOutput Output in Node.Outputs) { HashSet <FileReference> Files = TagNameToFileSet[Output.TagName]; foreach (FileReference File in Files) { if (!FileToStorageBlock.ContainsKey(File) && File.IsUnderDirectory(RootDir)) { if (Output == Node.DefaultOutput) { if (!FileToOutputName.ContainsKey(File)) { FileToOutputName[File] = ""; } } else { string OutputName; if (FileToOutputName.TryGetValue(File, out OutputName) && OutputName.Length > 0) { FileToOutputName[File] = String.Format("{0}+{1}", OutputName, Output.TagName.Substring(1)); } else { FileToOutputName[File] = Output.TagName.Substring(1); } } } } } // Invert the dictionary to make a mapping of storage block to the files each contains Dictionary <string, HashSet <FileReference> > OutputStorageBlockToFiles = new Dictionary <string, HashSet <FileReference> >(); foreach (KeyValuePair <FileReference, string> Pair in FileToOutputName) { HashSet <FileReference> Files; if (!OutputStorageBlockToFiles.TryGetValue(Pair.Value, out Files)) { Files = new HashSet <FileReference>(); OutputStorageBlockToFiles.Add(Pair.Value, Files); } Files.Add(Pair.Key); } // Write all the storage blocks, and update the mapping from file to storage block foreach (KeyValuePair <string, HashSet <FileReference> > Pair in OutputStorageBlockToFiles) { TempStorageBlock OutputBlock = new TempStorageBlock(Node.Name, Pair.Key); foreach (FileReference File in Pair.Value) { FileToStorageBlock.Add(File, OutputBlock); } Storage.Archive(Node.Name, Pair.Key, Pair.Value.ToArray(), Pair.Value.Any(x => ReferencedOutputFiles.Contains(x))); } // Publish all the output tags foreach (NodeOutput Output in Node.Outputs) { HashSet <FileReference> Files = TagNameToFileSet[Output.TagName]; HashSet <TempStorageBlock> StorageBlocks = new HashSet <TempStorageBlock>(); foreach (FileReference File in Files) { TempStorageBlock StorageBlock; if (FileToStorageBlock.TryGetValue(File, out StorageBlock)) { StorageBlocks.Add(StorageBlock); } } Storage.WriteFileList(Node.Name, Output.TagName, Files, StorageBlocks.ToArray()); } // Mark the node as succeeded Storage.MarkAsComplete(Node.Name); return(true); }
public virtual void ArchiveBuildProducts(TempStorageNodeInfo TempStorageNodeInfo, bool bLocalOnly) { TempStorage.StoreToTempStorage(TempStorageNodeInfo, BuildProducts, bLocalOnly, CommandUtils.CmdEnv.LocalRoot); }
/// <summary> /// Main entry point for the BuildGraph command /// </summary> public override ExitCode Execute() { // Parse the command line parameters string ScriptFileName = ParseParamValue("Script", null); string TargetNames = ParseParamValue("Target", null); string DocumentationFileName = ParseParamValue("Documentation", null); string SchemaFileName = ParseParamValue("Schema", null); string ExportFileName = ParseParamValue("Export", null); string PreprocessedFileName = ParseParamValue("Preprocess", null); string SharedStorageDir = ParseParamValue("SharedStorageDir", null); string SingleNodeName = ParseParamValue("SingleNode", null); string TriggerName = ParseParamValue("Trigger", null); string[] SkipTriggerNames = ParseParamValue("SkipTrigger", "").Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); bool bSkipTriggers = ParseParam("SkipTriggers"); string TokenSignature = ParseParamValue("TokenSignature", null); bool bSkipTargetsWithoutTokens = ParseParam("SkipTargetsWithoutTokens"); bool bResume = SingleNodeName != null || ParseParam("Resume"); bool bListOnly = ParseParam("ListOnly"); bool bShowDiagnostics = ParseParam("ShowDiagnostics"); bool bWriteToSharedStorage = ParseParam("WriteToSharedStorage") || CommandUtils.IsBuildMachine; bool bPublicTasksOnly = ParseParam("PublicTasksOnly"); string ReportName = ParseParamValue("ReportName", null); GraphPrintOptions PrintOptions = GraphPrintOptions.ShowCommandLineOptions; if (ParseParam("ShowDeps")) { PrintOptions |= GraphPrintOptions.ShowDependencies; } if (ParseParam("ShowNotifications")) { PrintOptions |= GraphPrintOptions.ShowNotifications; } // Parse any specific nodes to clean List <string> CleanNodes = new List <string>(); foreach (string NodeList in ParseParamValues("CleanNode")) { foreach (string NodeName in NodeList.Split('+', ';')) { CleanNodes.Add(NodeName); } } // Set up the standard properties which build scripts might need Dictionary <string, string> DefaultProperties = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); DefaultProperties["Branch"] = P4Enabled ? P4Env.Branch : "Unknown"; DefaultProperties["Depot"] = P4Enabled ? DefaultProperties["Branch"].Substring(2).Split('/').First() : "Unknown"; DefaultProperties["EscapedBranch"] = P4Enabled ? CommandUtils.EscapePath(P4Env.Branch) : "Unknown"; DefaultProperties["Change"] = P4Enabled ? P4Env.Changelist.ToString() : "0"; DefaultProperties["CodeChange"] = P4Enabled ? P4Env.CodeChangelist.ToString() : "0"; DefaultProperties["RootDir"] = CommandUtils.RootDirectory.FullName; DefaultProperties["IsBuildMachine"] = IsBuildMachine ? "true" : "false"; DefaultProperties["HostPlatform"] = HostPlatform.Current.HostEditorPlatform.ToString(); DefaultProperties["RestrictedFolderNames"] = String.Join(";", RestrictedFolders.Names); DefaultProperties["RestrictedFolderFilter"] = String.Join(";", RestrictedFolders.Names.Select(x => String.Format(".../{0}/...", x))); // Attempt to read existing Build Version information BuildVersion Version; if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) { DefaultProperties["EngineMajorVersion"] = Version.MajorVersion.ToString(); DefaultProperties["EngineMinorVersion"] = Version.MinorVersion.ToString(); DefaultProperties["EnginePatchVersion"] = Version.PatchVersion.ToString(); DefaultProperties["EngineCompatibleChange"] = Version.CompatibleChangelist.ToString(); } // Add any additional custom arguments from the command line (of the form -Set:X=Y) Dictionary <string, string> Arguments = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); foreach (string Param in Params) { const string Prefix = "set:"; if (Param.StartsWith(Prefix, StringComparison.InvariantCultureIgnoreCase)) { int EqualsIdx = Param.IndexOf('='); if (EqualsIdx >= 0) { Arguments[Param.Substring(Prefix.Length, EqualsIdx - Prefix.Length)] = Param.Substring(EqualsIdx + 1); } else { LogWarning("Missing value for '{0}'", Param.Substring(Prefix.Length)); } } } // Find all the tasks from the loaded assemblies Dictionary <string, ScriptTask> NameToTask = new Dictionary <string, ScriptTask>(); if (!FindAvailableTasks(NameToTask, bPublicTasksOnly)) { return(ExitCode.Error_Unknown); } // Generate documentation if (DocumentationFileName != null) { GenerateDocumentation(NameToTask, new FileReference(DocumentationFileName)); return(ExitCode.Success); } // Create a schema for the given tasks ScriptSchema Schema = new ScriptSchema(NameToTask); if (SchemaFileName != null) { FileReference FullSchemaFileName = new FileReference(SchemaFileName); LogInformation("Writing schema to {0}...", FullSchemaFileName.FullName); Schema.Export(FullSchemaFileName); if (ScriptFileName == null) { return(ExitCode.Success); } } // Check there was a script specified if (ScriptFileName == null) { LogError("Missing -Script= parameter for BuildGraph"); return(ExitCode.Error_Unknown); } // Read the script from disk Graph Graph; if (!ScriptReader.TryRead(new FileReference(ScriptFileName), Arguments, DefaultProperties, Schema, out Graph)) { return(ExitCode.Error_Unknown); } // Create the temp storage handler DirectoryReference RootDir = new DirectoryReference(CommandUtils.CmdEnv.LocalRoot); TempStorage Storage = new TempStorage(RootDir, DirectoryReference.Combine(RootDir, "Engine", "Saved", "BuildGraph"), (SharedStorageDir == null)? null : new DirectoryReference(SharedStorageDir), bWriteToSharedStorage); if (!bResume) { Storage.CleanLocal(); } foreach (string CleanNode in CleanNodes) { Storage.CleanLocalNode(CleanNode); } // Convert the supplied target references into nodes HashSet <Node> TargetNodes = new HashSet <Node>(); if (TargetNames == null) { if (!bListOnly) { LogError("Missing -Target= parameter for BuildGraph"); return(ExitCode.Error_Unknown); } TargetNodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); } else { foreach (string TargetName in TargetNames.Split(new char[] { '+', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())) { Node[] Nodes; if (!Graph.TryResolveReference(TargetName, out Nodes)) { LogError("Target '{0}' is not in graph", TargetName); return(ExitCode.Error_Unknown); } TargetNodes.UnionWith(Nodes); } } // Try to acquire tokens for all the target nodes we want to build if (TokenSignature != null) { // Find all the lock files HashSet <FileReference> RequiredTokens = new HashSet <FileReference>(TargetNodes.SelectMany(x => x.RequiredTokens)); // List out all the required tokens if (SingleNodeName == null) { CommandUtils.LogInformation("Required tokens:"); foreach (Node Node in TargetNodes) { foreach (FileReference RequiredToken in Node.RequiredTokens) { CommandUtils.LogInformation(" '{0}' requires {1}", Node, RequiredToken); } } } // Try to create all the lock files List <FileReference> CreatedTokens = new List <FileReference>(); if (!bListOnly) { CreatedTokens.AddRange(RequiredTokens.Where(x => WriteTokenFile(x, TokenSignature))); } // Find all the tokens that we don't have Dictionary <FileReference, string> MissingTokens = new Dictionary <FileReference, string>(); foreach (FileReference RequiredToken in RequiredTokens) { string CurrentOwner = ReadTokenFile(RequiredToken); if (CurrentOwner != null && CurrentOwner != TokenSignature) { MissingTokens.Add(RequiredToken, CurrentOwner); } } // If we want to skip all the nodes with missing locks, adjust the target nodes to account for it if (MissingTokens.Count > 0) { if (bSkipTargetsWithoutTokens) { // Find all the nodes we're going to skip HashSet <Node> SkipNodes = new HashSet <Node>(); foreach (IGrouping <string, FileReference> MissingTokensForBuild in MissingTokens.GroupBy(x => x.Value, x => x.Key)) { LogInformation("Skipping the following nodes due to {0}:", MissingTokensForBuild.Key); foreach (FileReference MissingToken in MissingTokensForBuild) { foreach (Node SkipNode in TargetNodes.Where(x => x.RequiredTokens.Contains(MissingToken) && SkipNodes.Add(x))) { LogInformation(" {0}", SkipNode); } } } // Write a list of everything left over if (SkipNodes.Count > 0) { TargetNodes.ExceptWith(SkipNodes); LogInformation("Remaining target nodes:"); foreach (Node TargetNode in TargetNodes) { LogInformation(" {0}", TargetNode); } if (TargetNodes.Count == 0) { LogInformation(" None."); } } } else { foreach (KeyValuePair <FileReference, string> Pair in MissingTokens) { List <Node> SkipNodes = TargetNodes.Where(x => x.RequiredTokens.Contains(Pair.Key)).ToList(); LogError("Cannot run {0} due to previous build: {1}", String.Join(", ", SkipNodes), Pair.Value); } foreach (FileReference CreatedToken in CreatedTokens) { FileReference.Delete(CreatedToken); } return(ExitCode.Error_Unknown); } } } // Cull the graph to include only those nodes Graph.Select(TargetNodes); // Collapse any triggers in the graph which are marked to be skipped HashSet <ManualTrigger> SkipTriggers = new HashSet <ManualTrigger>(); if (bSkipTriggers) { SkipTriggers.UnionWith(Graph.NameToTrigger.Values); } else { foreach (string SkipTriggerName in SkipTriggerNames) { ManualTrigger SkipTrigger; if (!Graph.NameToTrigger.TryGetValue(TriggerName, out SkipTrigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return(ExitCode.Error_Unknown); } SkipTriggers.Add(SkipTrigger); } } Graph.SkipTriggers(SkipTriggers); // If a report for the whole build was requested, insert it into the graph if (ReportName != null) { Report NewReport = new Report(ReportName); NewReport.Nodes.UnionWith(Graph.Agents.SelectMany(x => x.Nodes)); Graph.NameToReport.Add(ReportName, NewReport); } // Write out the preprocessed script if (PreprocessedFileName != null) { FileReference PreprocessedFileLocation = new FileReference(PreprocessedFileName); LogInformation("Writing {0}...", PreprocessedFileLocation); Graph.Write(PreprocessedFileLocation, (SchemaFileName != null)? new FileReference(SchemaFileName) : null); return(ExitCode.Success); } // Find the triggers which we are explicitly running. ManualTrigger Trigger = null; if (TriggerName != null && !Graph.NameToTrigger.TryGetValue(TriggerName, out Trigger)) { LogError("Couldn't find trigger '{0}'", TriggerName); return(ExitCode.Error_Unknown); } // If we're just building a single node, find it Node SingleNode = null; if (SingleNodeName != null && !Graph.NameToNode.TryGetValue(SingleNodeName, out SingleNode)) { LogError("Node '{0}' is not in the trimmed graph", SingleNodeName); return(ExitCode.Error_Unknown); } // If we just want to show the contents of the graph, do so and exit. if (bListOnly) { HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); } // Print out all the diagnostic messages which still apply, unless we're running a step as part of a build system or just listing the contents of the file. if (SingleNode == null && (!bListOnly || bShowDiagnostics)) { IEnumerable <GraphDiagnostic> Diagnostics = Graph.Diagnostics.Where(x => x.EnclosingTrigger == Trigger); foreach (GraphDiagnostic Diagnostic in Diagnostics) { if (Diagnostic.EventType == LogEventType.Console) { CommandUtils.LogInformation(Diagnostic.Message); } else if (Diagnostic.EventType == LogEventType.Warning) { CommandUtils.LogWarning(Diagnostic.Message); } else { CommandUtils.LogError(Diagnostic.Message); } } if (Diagnostics.Any(x => x.EventType == LogEventType.Error)) { return(ExitCode.Error_Unknown); } } // Export the graph to a file if (ExportFileName != null) { HashSet <Node> CompletedNodes = FindCompletedNodes(Graph, Storage); Graph.Print(CompletedNodes, PrintOptions); Graph.Export(new FileReference(ExportFileName), Trigger, CompletedNodes); return(ExitCode.Success); } // Execute the command if (!bListOnly) { if (SingleNode != null) { if (!BuildNode(new JobContext(this), Graph, SingleNode, Storage, bWithBanner: true)) { return(ExitCode.Error_Unknown); } } else { if (!BuildAllNodes(new JobContext(this), Graph, Storage)) { return(ExitCode.Error_Unknown); } } } return(ExitCode.Success); }
/// <summary> /// Find all the nodes in the graph which are already completed /// </summary> /// <param name="Graph">The graph instance</param> /// <param name="Storage">The temp storage backend which stores the shared state</param> HashSet<Node> FindCompletedNodes(Graph Graph, TempStorage Storage) { HashSet<Node> CompletedNodes = new HashSet<Node>(); foreach(Node Node in Graph.Agents.SelectMany(x => x.Nodes)) { if(Storage.IsComplete(Node.Name)) { CompletedNodes.Add(Node); } } return CompletedNodes; }