void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail) { Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>(); foreach (BuildNode NodeToDo in OrderedToDo) { if (NodeToDo.Node.BuildProducts != null) { throw new AutomationException("topological sort error"); } var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name); string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage; string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage; if (bFake) { StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder } if (string.IsNullOrEmpty(StorageRootIfAny)) { StorageRootIfAny = CmdEnv.LocalRoot; } // this is kinda complicated bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests (!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory()); if (NodeToDo.IsComplete) { if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine) { LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); NodeToDo.Node.BuildProducts = new List<string>(); } else { LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); bool WasLocal; try { NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny); } catch (Exception Ex) { if(GameNameIfAny != "") { NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny); } else { throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name); } } if (!WasLocal) { NodeToDo.Node.PostLoadFromSharedTempStorage(this); } } } else { if (SaveSuccessRecords) { // We save our status to a new temp storage location specifically named with a suffix so we can find it later. EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny); } double BuildDuration = 0.0; try { if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Failing node {0} by request.", NodeToDo.Name); } if (bFake) { LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); NodeToDo.DoFakeBuild(); } else { LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); DateTime StartTime = DateTime.UtcNow; using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name)) { NodeToDo.DoBuild(); } BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000; } TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny); } catch (Exception Ex) { //@todo: This is a dup of non-exception code. Consolidate this!! NodeHistory History = null; if (SaveSuccessRecords) { using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory")) { History = FindNodeHistory(NodeToDo, JobInfo); } using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus")) { // We save our status to a new temp storage location specifically named with a suffix so we can find it later. EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId")); } using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps")) { EC.UpdateECProps(NodeToDo); } if (IsBuildMachine) { using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails")) { GetFailureEmails(EC, NodeToDo, History, JobInfo); } } EC.UpdateECBuildTime(NodeToDo, BuildDuration); } LogConsole("{0}", Ex); if (History != null) { LogConsole("Changes since last green *********************************"); LogConsole(""); LogConsole(""); LogConsole(""); PrintDetailedChanges(History, P4Env.Changelist); LogConsole("End changes since last green"); } string FailInfo = ""; FailInfo += "********************************* Main log file"; FailInfo += Environment.NewLine + Environment.NewLine; FailInfo += LogUtils.GetLogTail(); FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine; string OtherLog = "See logfile for details: '"; if (FailInfo.Contains(OtherLog)) { string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length); if (LogFile.Contains("'")) { LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'"))); if (FileExists_NoExceptions(LogFile)) { FailInfo += "********************************* Sub log file " + LogFile; FailInfo += Environment.NewLine + Environment.NewLine; FailInfo += LogUtils.GetLogTail(LogFile); FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine; } } } string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log"); WriteAllText(Filename, FailInfo); throw(Ex); } if (SaveSuccessRecords) { NodeHistory History = null; using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory")) { History = FindNodeHistory(NodeToDo, JobInfo); } using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus")) { // We save our status to a new temp storage location specifically named with a suffix so we can find it later. EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny); } using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps")) { EC.UpdateECProps(NodeToDo); } if (IsBuildMachine) { using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails")) { GetFailureEmails(EC, NodeToDo, History, JobInfo); } } EC.UpdateECBuildTime(NodeToDo, BuildDuration); } } foreach (string Product in NodeToDo.Node.BuildProducts) { if (BuildProductToNodeMap.ContainsKey(Product)) { throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product); } BuildProductToNodeMap.Add(Product, NodeToDo); } } PrintRunTime(); }
static void FindCompletionState(IEnumerable<BuildNode> NodesToDo, JobInfo JobInfo, bool LocalOnly) { foreach(BuildNode NodeToDo in NodesToDo) { // Construct the full temp storage node info var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name); string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage; if (LocalOnly) { NodeToDo.IsComplete = TempStorage.LocalTempStorageManifestExists(TempStorageNodeInfo, bQuiet: true); } else { NodeToDo.IsComplete = TempStorage.TempStorageExists(TempStorageNodeInfo, GameNameIfAny, bLocalOnly: false, bQuiet: true); if(GameNameIfAny != "" && !NodeToDo.IsComplete) { NodeToDo.IsComplete = TempStorage.TempStorageExists(TempStorageNodeInfo, "UE4", bLocalOnly: false, bQuiet: true); } } LogVerbose("** {0}", NodeToDo.Name); if (!NodeToDo.IsComplete) { LogVerbose("***** GUBP Trigger Node was already triggered {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory()); } else { LogVerbose("***** GUBP Trigger Node was NOT yet triggered {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory()); } } }
/// <summary> /// Returns the full path to the temp storage directory for the given temp storage node and game (ie, P:\Builds\GameName\TmpStore\NodeInfoDirectory) /// </summary> /// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param> /// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param> /// <returns>The full path to the temp storage directory for the given storage block name for the given game.</returns> private static string SharedTempStorageDirectory(TempStorageNodeInfo TempStorageNodeInfo, string GameName) { return CommandUtils.CombinePaths(ResolveSharedTempStorageDirectory(GameName), TempStorageNodeInfo.GetRelativeDirectory()); }