/// <summary> /// Main entry point for GUBP /// </summary> public override void ExecuteBuild() { Log("************************* GUBP"); bool bPreflightBuild = false; int PreflightShelveCL = 0; string PreflightShelveCLString = GetEnvVar("uebp_PreflightShelveCL"); if ((!String.IsNullOrEmpty(PreflightShelveCLString) && IsBuildMachine) || ParseParam("PreflightTest")) { Log("**** Preflight shelve {0}", PreflightShelveCLString); if (!String.IsNullOrEmpty(PreflightShelveCLString)) { PreflightShelveCL = int.Parse(PreflightShelveCLString); if (PreflightShelveCL < 2000000) { throw new AutomationException(String.Format( "{0} does not look like a CL", PreflightShelveCL)); } } bPreflightBuild = true; } List<UnrealTargetPlatform> HostPlatforms = new List<UnrealTargetPlatform>(); if (!ParseParam("NoPC")) { HostPlatforms.Add(UnrealTargetPlatform.Win64); } if (!ParseParam("NoMac")) { HostPlatforms.Add(UnrealTargetPlatform.Mac); } if(!ParseParam("NoLinux") && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux) { HostPlatforms.Add(UnrealTargetPlatform.Linux); } string StoreName = ParseParamValue("Store"); string StoreSuffix = ParseParamValue("StoreSuffix", ""); string PreflightMangleSuffix = ""; if (bPreflightBuild) { int PreflightUID = ParseParamInt("PreflightUID", 0); PreflightMangleSuffix = String.Format("-PF-{0}-{1}", PreflightShelveCL, PreflightUID); StoreSuffix = StoreSuffix + PreflightMangleSuffix; } int CL = ParseParamInt("CL", 0); bool bCleanLocalTempStorage = ParseParam("CleanLocal"); bool bSkipTriggers = ParseParam("SkipTriggers"); bool bFake = ParseParam("fake"); bool bFakeEC = ParseParam("FakeEC"); bool bSaveSharedTempStorage = false; bool LocalOnly = true; string CLString = ""; if (String.IsNullOrEmpty(StoreName)) { if (P4Enabled) { if (CL == 0) { CL = P4Env.Changelist; } CLString = String.Format("{0}", CL); StoreName = P4Env.BuildRootEscaped + "-" + CLString; bSaveSharedTempStorage = CommandUtils.IsBuildMachine; LocalOnly = false; } else { StoreName = "TempLocal"; bSaveSharedTempStorage = false; } } StoreName = StoreName + StoreSuffix; if (bFakeEC) { LocalOnly = true; } if (bSaveSharedTempStorage) { if (!TempStorage.HaveSharedTempStorage(true)) { throw new AutomationException("Request to save to temp storage, but {0} is unavailable.", TempStorage.UE4TempStorageDirectory()); } } else if (!LocalOnly && !TempStorage.HaveSharedTempStorage(false)) { LogWarning("Looks like we want to use shared temp storage, but since we don't have it, we won't use it."); LocalOnly = true; } bool CommanderSetup = ParseParam("CommanderJobSetupOnly"); string ExplicitTriggerName = ""; if (CommanderSetup) { ExplicitTriggerName = ParseParamValue("TriggerNode", ""); } List<BuildNode> AllNodes; List<AggregateNode> AllAggregates; int TimeQuantum = 20; AddNodesForBranch(CL, HostPlatforms, bPreflightBuild, PreflightMangleSuffix, out AllNodes, out AllAggregates, ref TimeQuantum); LinkGraph(AllAggregates, AllNodes); FindControllingTriggers(AllNodes); FindCompletionState(AllNodes, StoreName, LocalOnly); ComputeDependentFrequencies(AllNodes); if (bCleanLocalTempStorage) // shared temp storage can never be wiped { TempStorage.DeleteLocalTempStorageManifests(CmdEnv); } int TimeIndex = ParseParamInt("TimeIndex", 0); if (TimeIndex == 0) { TimeIndex = ParseParamInt("UserTimeIndex", 0); } if (ParseParam("CIS") && ExplicitTriggerName == "" && CommanderSetup) // explicit triggers will already have a time index assigned { TimeIndex = UpdateCISCounter(TimeQuantum); Log("Setting TimeIndex to {0}", TimeIndex); } HashSet<BuildNode> NodesToDo = ParseNodesToDo(AllNodes, AllAggregates); CullNodesForTimeIndex(NodesToDo, TimeIndex); CullNodesForPreflight(NodesToDo, bPreflightBuild); TriggerNode ExplicitTrigger = null; if (CommanderSetup) { if (!String.IsNullOrEmpty(ExplicitTriggerName)) { foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>()) { if (Trigger.Name.Equals(ExplicitTriggerName, StringComparison.InvariantCultureIgnoreCase)) { Trigger.Activate(); ExplicitTrigger = Trigger; break; } } if (ExplicitTrigger == null) { throw new AutomationException("Could not find trigger node named {0}", ExplicitTriggerName); } } else { if (bSkipTriggers) { foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>()) { Trigger.Activate(); } } } } List<BuildNode> OrderedToDo = TopologicalSort(NodesToDo, ExplicitTrigger, false, false); List<TriggerNode> UnfinishedTriggers = FindUnfinishedTriggers(bSkipTriggers, ExplicitTrigger, OrderedToDo); PrintNodes(this, OrderedToDo, AllAggregates, UnfinishedTriggers, TimeQuantum); //check sorting CheckSortOrder(OrderedToDo); ElectricCommander EC = new ElectricCommander(this); string ShowHistoryParam = ParseParamValue("ShowHistory", null); if(ShowHistoryParam != null) { BuildNode Node = AllNodes.FirstOrDefault(x => x.Name.Equals(ShowHistoryParam, StringComparison.InvariantCultureIgnoreCase)); if(Node == null) { throw new AutomationException("Couldn't find node {0}", ShowHistoryParam); } NodeHistory History = FindNodeHistory(Node, CLString, StoreName); if(History == null) { throw new AutomationException("Couldn't get history for {0}", ShowHistoryParam); } PrintDetailedChanges(History, P4Env.Changelist); } else { string FakeFail = ParseParamValue("FakeFail"); if(CommanderSetup) { DoCommanderSetup(EC, AllNodes, AllAggregates, OrderedToDo, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, CLString, ExplicitTrigger, UnfinishedTriggers, FakeFail, bPreflightBuild); } else if(ParseParam("SaveGraph")) { SaveGraphVisualization(OrderedToDo); } else if(!ParseParam("ListOnly")) { ExecuteNodes(EC, OrderedToDo, bFake, bFakeEC, bSaveSharedTempStorage, CLString, StoreName, FakeFail); } } PrintRunTime(); }
void GetFailureEmails(ElectricCommander EC, BuildNode NodeToDo, NodeHistory History, string CLString, string StoreName, bool OnlyLateUpdates = false) { string FailCauserEMails = ""; string EMailNote = ""; bool SendSuccessForGreenAfterRed = false; int NumPeople = 0; if (History != null) { if (History.LastSucceeded > 0 && History.LastSucceeded < P4Env.Changelist) { int LastNonDuplicateFail = P4Env.Changelist; try { if (OnlyLateUpdates) { LastNonDuplicateFail = FindLastNonDuplicateFail(NodeToDo, History, CLString, StoreName); if (LastNonDuplicateFail < P4Env.Changelist) { Log("*** Red-after-red spam reduction, changed CL {0} to CL {1} because the errors didn't change.", P4Env.Changelist, LastNonDuplicateFail); } } } catch (Exception Ex) { LastNonDuplicateFail = P4Env.Changelist; Log(System.Diagnostics.TraceEventType.Warning, "Failed to FindLastNonDuplicateFail."); Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } if(LastNonDuplicateFail > History.LastSucceeded) { List<P4Connection.ChangeRecord> ChangeRecords = GetSourceChangeRecords(History.LastSucceeded + 1, LastNonDuplicateFail); foreach (P4Connection.ChangeRecord Record in ChangeRecords) { FailCauserEMails = GUBPNode.MergeSpaceStrings(FailCauserEMails, Record.UserEmail); } } if (!String.IsNullOrEmpty(FailCauserEMails)) { NumPeople++; foreach (char AChar in FailCauserEMails.ToCharArray()) { if (AChar == ' ') { NumPeople++; } } if (NumPeople > 50) { EMailNote = String.Format("This step has been broken for more than 50 changes. It last succeeded at CL {0}. ", History.LastSucceeded); } } } else if (History.LastSucceeded <= 0) { EMailNote = String.Format("This step has been broken for more than a few days, so there is no record of it ever succeeding. "); } if (EMailNote != "" && !String.IsNullOrEmpty(History.FailedString)) { EMailNote += String.Format("It has failed at CLs {0}. ", History.FailedString); } if (EMailNote != "" && !String.IsNullOrEmpty(History.InProgressString)) { EMailNote += String.Format("These CLs are being built right now {0}. ", History.InProgressString); } if (History.LastSucceeded > 0 && History.LastSucceeded < P4Env.Changelist && History.LastFailed > History.LastSucceeded && History.LastFailed < P4Env.Changelist) { SendSuccessForGreenAfterRed = ParseParam("CIS"); } } if (History == null) { EC.UpdateEmailProperties(NodeToDo, 0, "", FailCauserEMails, EMailNote, SendSuccessForGreenAfterRed); } else { EC.UpdateEmailProperties(NodeToDo, History.LastSucceeded, History.FailedString, FailCauserEMails, EMailNote, SendSuccessForGreenAfterRed); } }
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrdereredToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, string CLString, string StoreName, string FakeFail) { Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>(); foreach (BuildNode NodeToDo in OrdereredToDo) { if (NodeToDo.Node.BuildProducts != null || NodeToDo.Node.AllDependencyBuildProducts != null) { throw new AutomationException("topological sort error"); } NodeToDo.Node.AllDependencyBuildProducts = new List<string>(); NodeToDo.Node.AllDependencies = new List<string>(); foreach (BuildNode Dep in NodeToDo.Dependencies) { NodeToDo.Node.AddAllDependent(Dep.Name); if (Dep.Node.AllDependencies == null) { throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name); } foreach (string DepDep in Dep.Node.AllDependencies) { NodeToDo.Node.AddAllDependent(DepDep); } if (Dep.Node.BuildProducts == null) { throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name); } foreach (string Prod in Dep.Node.BuildProducts) { NodeToDo.Node.AddDependentBuildProduct(Prod); } if (Dep.Node.AllDependencyBuildProducts == null) { throw new AutomationException("Node {0} was not processed yet2? Processing {1}", Dep.Name, NodeToDo.Name); } foreach (string Prod in Dep.Node.AllDependencyBuildProducts) { NodeToDo.Node.AddDependentBuildProduct(Prod); } } string NodeStoreName = StoreName + "-" + NodeToDo.Name; string GameNameIfAny = NodeToDo.Node.GameNameIfAnyForTempStorage(); string StorageRootIfAny = NodeToDo.Node.RootIfAnyForTempStorage(); if (bFake) { StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder } // this is kinda complicated bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests (!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter Log("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, NodeStoreName); if (NodeToDo.IsComplete) { if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine) { Log("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, NodeStoreName); NodeToDo.Node.BuildProducts = new List<string>(); } else { Log("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, NodeStoreName); bool WasLocal; try { NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny); } catch { if(GameNameIfAny != "") { NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, "", StorageRootIfAny); } else { throw new AutomationException("Build Products cannot be found for node {0}", NodeToDo.Name); } } if (!WasLocal) { NodeToDo.Node.PostLoadFromSharedTempStorage(this); } } } else { if (SaveSuccessRecords) { EC.SaveStatus(NodeToDo, StartedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny); } double BuildDuration = 0.0; try { if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Failing node {0} by request.", NodeToDo.Name); } if (bFake) { Log("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName); NodeToDo.Node.DoFakeBuild(this); } else { Log("***** Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName); DateTime StartTime = DateTime.UtcNow; using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name)) { NodeToDo.Node.DoBuild(this); } BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000; } using(TelemetryStopwatch StoreBuildProductsStopwatch = new TelemetryStopwatch("StoreBuildProducts")) { double StoreDuration = 0.0; DateTime StartTime = DateTime.UtcNow; TempStorage.StoreToTempStorage(CmdEnv, NodeStoreName, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny); StoreDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000; Log("Took {0} seconds to store build products", StoreDuration); if (IsBuildMachine) { EC.RunECTool(String.Format("setProperty \"/myJobStep/StoreDuration\" \"{0}\"", StoreDuration.ToString())); } } if (ParseParam("StompCheck")) { foreach (string Dep in NodeToDo.Node.AllDependencies) { try { bool WasLocal; using(TelemetryStopwatch RetrieveBuildProductsStopwatch = new TelemetryStopwatch("RetrieveBuildProducts")) { TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny); } if (!WasLocal) { throw new AutomationException("Retrieve was not local?"); } } catch(Exception Ex) { throw new AutomationException("Node {0} stomped Node {1} Ex: {2}", NodeToDo.Name, Dep, LogUtils.FormatException(Ex)); } } } } catch (Exception Ex) { NodeHistory History = null; if (SaveSuccessRecords) { using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory")) { History = FindNodeHistory(NodeToDo, CLString, StoreName); } using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus")) { EC.SaveStatus(NodeToDo, FailedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId")); } using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps")) { EC.UpdateECProps(NodeToDo); } if (IsBuildMachine) { using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails")) { GetFailureEmails(EC, NodeToDo, History, CLString, StoreName); } } EC.UpdateECBuildTime(NodeToDo, BuildDuration); } Log("{0}", ExceptionToString(Ex)); if (History != null) { Log("Changes since last green *********************************"); Log(""); Log(""); Log(""); PrintDetailedChanges(History, P4Env.Changelist); Log("End changes since last green"); } string FailInfo = ""; FailInfo += "********************************* Main log file"; FailInfo += Environment.NewLine + Environment.NewLine; FailInfo += LogUtils.GetLogTail(); FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine; string OtherLog = "See logfile for details: '"; if (FailInfo.Contains(OtherLog)) { string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length); if (LogFile.Contains("'")) { LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'"))); if (FileExists_NoExceptions(LogFile)) { FailInfo += "********************************* Sub log file " + LogFile; FailInfo += Environment.NewLine + Environment.NewLine; FailInfo += LogUtils.GetLogTail(LogFile); FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine; } } } string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log"); WriteAllText(Filename, FailInfo); throw(Ex); } if (SaveSuccessRecords) { NodeHistory History = null; using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory")) { History = FindNodeHistory(NodeToDo, CLString, StoreName); } using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus")) { EC.SaveStatus(NodeToDo, SucceededTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny); } using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps")) { EC.UpdateECProps(NodeToDo); } if (IsBuildMachine) { using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails")) { GetFailureEmails(EC, NodeToDo, History, CLString, StoreName); } } EC.UpdateECBuildTime(NodeToDo, BuildDuration); } } foreach (string Product in NodeToDo.Node.BuildProducts) { if (BuildProductToNodeMap.ContainsKey(Product)) { throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product], NodeToDo.Name, Product); } BuildProductToNodeMap.Add(Product, NodeToDo); } } PrintRunTime(); }
private void DoCommanderSetup(ElectricCommander EC, IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail, bool bPreflightBuild) { List<BuildNode> SortedNodes = TopologicalSort(new HashSet<BuildNode>(AllNodes), null, SubSort: false, DoNotConsiderCompletion: true); Log("******* {0} GUBP Nodes", SortedNodes.Count); List<BuildNode> FilteredOrdereredToDo = new List<BuildNode>(); using(TelemetryStopwatch StartFilterTimer = new TelemetryStopwatch("FilterNodes")) { // remove nodes that have unfinished triggers foreach (BuildNode NodeToDo in OrdereredToDo) { if (NodeToDo.ControllingTriggers.Length == 0 || !UnfinishedTriggers.Contains(NodeToDo.ControllingTriggers.Last())) { // if we are triggering, then remove nodes that are not controlled by the trigger or are dependencies of this trigger if (ExplicitTrigger != null && ExplicitTrigger != NodeToDo && !ExplicitTrigger.DependsOn(NodeToDo) && !NodeToDo.DependsOn(ExplicitTrigger)) { continue; // this wasn't on the chain related to the trigger we are triggering, so it is not relevant } // in preflight builds, we are either skipping triggers (and running things downstream) or we just stop at triggers and don't make them available for triggering. if (bPreflightBuild && !bSkipTriggers && (NodeToDo is TriggerNode)) { continue; } FilteredOrdereredToDo.Add(NodeToDo); } } } using(TelemetryStopwatch PrintNodesTimer = new TelemetryStopwatch("SetupCommanderPrint")) { Log("*********** EC Nodes, in order."); PrintNodes(this, FilteredOrdereredToDo, AllAggregates, UnfinishedTriggers, TimeQuantum); } EC.DoCommanderSetup(AllNodes, AllAggregates, FilteredOrdereredToDo, SortedNodes, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, CLString, ExplicitTrigger, UnfinishedTriggers, FakeFail); }
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail) { Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>(); foreach (BuildNode NodeToDo in OrderedToDo) { if (NodeToDo.Node.BuildProducts != null) { throw new AutomationException("topological sort error"); } var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name); string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage; string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage; if (bFake) { StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder } if (string.IsNullOrEmpty(StorageRootIfAny)) { StorageRootIfAny = CmdEnv.LocalRoot; } // this is kinda complicated bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests (!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory()); if (NodeToDo.IsComplete) { if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine) { LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); NodeToDo.Node.BuildProducts = new List<string>(); } else { LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); bool WasLocal; try { NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny); } catch (Exception Ex) { if(GameNameIfAny != "") { NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny); } else { throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name); } } if (!WasLocal) { NodeToDo.Node.PostLoadFromSharedTempStorage(this); } } } else { if (SaveSuccessRecords) { // We save our status to a new temp storage location specifically named with a suffix so we can find it later. EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny); } double BuildDuration = 0.0; try { if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Failing node {0} by request.", NodeToDo.Name); } if (bFake) { LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); NodeToDo.DoFakeBuild(); } else { LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory()); DateTime StartTime = DateTime.UtcNow; using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name)) { NodeToDo.DoBuild(); } BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000; } TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny); } catch (Exception Ex) { //@todo: This is a dup of non-exception code. Consolidate this!! NodeHistory History = null; if (SaveSuccessRecords) { using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory")) { History = FindNodeHistory(NodeToDo, JobInfo); } using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus")) { // We save our status to a new temp storage location specifically named with a suffix so we can find it later. EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId")); } using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps")) { EC.UpdateECProps(NodeToDo); } if (IsBuildMachine) { using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails")) { GetFailureEmails(EC, NodeToDo, History, JobInfo); } } EC.UpdateECBuildTime(NodeToDo, BuildDuration); } LogConsole("{0}", Ex); if (History != null) { LogConsole("Changes since last green *********************************"); LogConsole(""); LogConsole(""); LogConsole(""); PrintDetailedChanges(History, P4Env.Changelist); LogConsole("End changes since last green"); } string FailInfo = ""; FailInfo += "********************************* Main log file"; FailInfo += Environment.NewLine + Environment.NewLine; FailInfo += LogUtils.GetLogTail(); FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine; string OtherLog = "See logfile for details: '"; if (FailInfo.Contains(OtherLog)) { string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length); if (LogFile.Contains("'")) { LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'"))); if (FileExists_NoExceptions(LogFile)) { FailInfo += "********************************* Sub log file " + LogFile; FailInfo += Environment.NewLine + Environment.NewLine; FailInfo += LogUtils.GetLogTail(LogFile); FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine; } } } string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log"); WriteAllText(Filename, FailInfo); throw(Ex); } if (SaveSuccessRecords) { NodeHistory History = null; using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory")) { History = FindNodeHistory(NodeToDo, JobInfo); } using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus")) { // We save our status to a new temp storage location specifically named with a suffix so we can find it later. EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny); } using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps")) { EC.UpdateECProps(NodeToDo); } if (IsBuildMachine) { using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails")) { GetFailureEmails(EC, NodeToDo, History, JobInfo); } } EC.UpdateECBuildTime(NodeToDo, BuildDuration); } } foreach (string Product in NodeToDo.Node.BuildProducts) { if (BuildProductToNodeMap.ContainsKey(Product)) { throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product); } BuildProductToNodeMap.Add(Product, NodeToDo); } } PrintRunTime(); }
/// <summary> /// Main entry point for GUBP /// </summary> public override void ExecuteBuild() { LogConsole("************************* GUBP"); List<UnrealTargetPlatform> HostPlatforms = new List<UnrealTargetPlatform>(); if (!ParseParam("NoPC")) { HostPlatforms.Add(UnrealTargetPlatform.Win64); } if (!ParseParam("NoMac")) { HostPlatforms.Add(UnrealTargetPlatform.Mac); } if(!ParseParam("NoLinux") && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux) { HostPlatforms.Add(UnrealTargetPlatform.Linux); } TempStorage.SetZipTempStorage(!ParseParam("NoZipTempStorage")); bool bCleanLocalTempStorage = ParseParam("CleanLocal"); bool bSkipTriggers = ParseParam("SkipTriggers"); bool bFake = ParseParam("fake"); bool bFakeEC = ParseParam("FakeEC"); bool bSaveSharedTempStorage = P4Enabled && (IsBuildMachine || GlobalCommandLine.UseLocalBuildStorage); // encapsulate the logic and temp vars to determine the job info. // Ensures the temp vars don't escape the scope to be used later. Func<JobInfo> GetJobInfo = () => { int PreflightShelveCL = 0; int PreflightUID = 0; string PreflightShelveCLString = GetEnvVar("uebp_PreflightShelveCL"); // We must be passed a valid shelve and be a build machine (or be running a preflight test) to be a valid preflight. if (!string.IsNullOrEmpty(PreflightShelveCLString) && (IsBuildMachine || ParseParam("PreflightTest"))) { LogConsole("**** Preflight shelve {0}", PreflightShelveCLString); PreflightShelveCL = int.Parse(PreflightShelveCLString); if (PreflightShelveCL < 2000000) { throw new AutomationException("{0} does not look like a CL", PreflightShelveCL); } PreflightUID = ParseParamInt("PreflightUID", 0); } return new JobInfo( P4Enabled ? P4Env.BuildRootEscaped : "NoP4", P4Enabled ? ParseParamInt("CL", P4Env.Changelist) : 0, PreflightShelveCL, PreflightUID); }; var JobInfo = GetJobInfo(); bool LocalOnly = !P4Enabled || bFakeEC; if (bSaveSharedTempStorage) { if (!TempStorage.IsSharedTempStorageAvailable(true)) { throw new AutomationException("Request to save to shared temp storage, but shared temp storage is unavailable or does not exist."); } } else if (!LocalOnly && !TempStorage.IsSharedTempStorageAvailable(false)) { LogWarning("Looks like we want to use shared temp storage, but since we don't have it, we won't use it."); LocalOnly = true; } bool CommanderSetup = ParseParam("CommanderJobSetupOnly"); string ExplicitTriggerName = ""; if (CommanderSetup) { ExplicitTriggerName = ParseParamValue("TriggerNode", ""); } List<BuildNode> AllNodes; List<AggregateNode> AllAggregates; int TimeQuantum = 20; AddNodesForBranch(HostPlatforms, JobInfo, out AllNodes, out AllAggregates, ref TimeQuantum); LinkGraph(AllAggregates, AllNodes); FindControllingTriggers(AllNodes); FindCompletionState(AllNodes, JobInfo, LocalOnly); ComputeDependentFrequencies(AllNodes); if (bCleanLocalTempStorage) // shared temp storage can never be wiped { TempStorage.DeleteLocalTempStorageManifests(); } int TimeIndex = ParseParamInt("TimeIndex", 0); if (TimeIndex == 0) { TimeIndex = ParseParamInt("UserTimeIndex", 0); } if (ParseParam("CIS") && ExplicitTriggerName == "" && CommanderSetup) // explicit triggers will already have a time index assigned { TimeIndex = UpdateCISCounter(TimeQuantum); Log("Setting TimeIndex to {0}", TimeIndex); } HashSet<BuildNode> NodesToDo = ParseNodesToDo(AllNodes, AllAggregates); CullNodesForTimeIndex(NodesToDo, TimeIndex); if (JobInfo.IsPreflight) { CullNodesForPreflight(NodesToDo); } TriggerNode ExplicitTrigger = null; if (CommanderSetup) { if (!String.IsNullOrEmpty(ExplicitTriggerName)) { foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>()) { if (Trigger.Name.Equals(ExplicitTriggerName, StringComparison.InvariantCultureIgnoreCase)) { Trigger.Activate(); ExplicitTrigger = Trigger; break; } } if (ExplicitTrigger == null) { throw new AutomationException("Could not find trigger node named {0}", ExplicitTriggerName); } } else { if (bSkipTriggers) { foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>()) { Trigger.Activate(); } } } } List<BuildNode> OrderedToDo = TopologicalSort(NodesToDo, ExplicitTrigger, false, false); List<TriggerNode> UnfinishedTriggers = FindUnfinishedTriggers(bSkipTriggers, ExplicitTrigger, OrderedToDo); PrintNodes(this, OrderedToDo, AllAggregates, UnfinishedTriggers, TimeQuantum); //check sorting CheckSortOrder(OrderedToDo); ElectricCommander EC = new ElectricCommander(this); string ShowHistoryParam = ParseParamValue("ShowHistory", null); if(ShowHistoryParam != null) { BuildNode Node = AllNodes.FirstOrDefault(x => x.Name.Equals(ShowHistoryParam, StringComparison.InvariantCultureIgnoreCase)); if(Node == null) { throw new AutomationException("Couldn't find node {0}", ShowHistoryParam); } NodeHistory History = FindNodeHistory(Node, JobInfo); if(History == null) { throw new AutomationException("Couldn't get history for {0}", ShowHistoryParam); } PrintDetailedChanges(History, P4Env.Changelist); } else { string FakeFail = ParseParamValue("FakeFail"); if(CommanderSetup) { DoCommanderSetup(EC, AllNodes, AllAggregates, OrderedToDo, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, ExplicitTrigger, UnfinishedTriggers, FakeFail, JobInfo.IsPreflight); } else if(!ParseParam("ListOnly")) { ExecuteNodes(EC, OrderedToDo, bFake, bFakeEC, bSaveSharedTempStorage, JobInfo, FakeFail); } } PrintRunTime(); }