/// <summary>
/// Main entry point for GUBP
/// </summary>
public override void ExecuteBuild()
{
Log("************************* GUBP");
bool bPreflightBuild = false;
int PreflightShelveCL = 0;
string PreflightShelveCLString = GetEnvVar("uebp_PreflightShelveCL");
if ((!String.IsNullOrEmpty(PreflightShelveCLString) && IsBuildMachine) || ParseParam("PreflightTest"))
{
Log("**** Preflight shelve {0}", PreflightShelveCLString);
if (!String.IsNullOrEmpty(PreflightShelveCLString))
{
PreflightShelveCL = int.Parse(PreflightShelveCLString);
if (PreflightShelveCL < 2000000)
{
throw new AutomationException(String.Format( "{0} does not look like a CL", PreflightShelveCL));
}
}
bPreflightBuild = true;
}
List<UnrealTargetPlatform> HostPlatforms = new List<UnrealTargetPlatform>();
if (!ParseParam("NoPC"))
{
HostPlatforms.Add(UnrealTargetPlatform.Win64);
}
if (!ParseParam("NoMac"))
{
HostPlatforms.Add(UnrealTargetPlatform.Mac);
}
if(!ParseParam("NoLinux") && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux)
{
HostPlatforms.Add(UnrealTargetPlatform.Linux);
}
string StoreName = ParseParamValue("Store");
string StoreSuffix = ParseParamValue("StoreSuffix", "");
string PreflightMangleSuffix = "";
if (bPreflightBuild)
{
int PreflightUID = ParseParamInt("PreflightUID", 0);
PreflightMangleSuffix = String.Format("-PF-{0}-{1}", PreflightShelveCL, PreflightUID);
StoreSuffix = StoreSuffix + PreflightMangleSuffix;
}
int CL = ParseParamInt("CL", 0);
bool bCleanLocalTempStorage = ParseParam("CleanLocal");
bool bSkipTriggers = ParseParam("SkipTriggers");
bool bFake = ParseParam("fake");
bool bFakeEC = ParseParam("FakeEC");
bool bSaveSharedTempStorage = false;
bool LocalOnly = true;
string CLString = "";
if (String.IsNullOrEmpty(StoreName))
{
if (P4Enabled)
{
if (CL == 0)
{
CL = P4Env.Changelist;
}
CLString = String.Format("{0}", CL);
StoreName = P4Env.BuildRootEscaped + "-" + CLString;
bSaveSharedTempStorage = CommandUtils.IsBuildMachine;
LocalOnly = false;
}
else
{
StoreName = "TempLocal";
bSaveSharedTempStorage = false;
}
}
StoreName = StoreName + StoreSuffix;
if (bFakeEC)
{
LocalOnly = true;
}
if (bSaveSharedTempStorage)
{
if (!TempStorage.HaveSharedTempStorage(true))
{
throw new AutomationException("Request to save to temp storage, but {0} is unavailable.", TempStorage.UE4TempStorageDirectory());
}
}
else if (!LocalOnly && !TempStorage.HaveSharedTempStorage(false))
{
LogWarning("Looks like we want to use shared temp storage, but since we don't have it, we won't use it.");
LocalOnly = true;
}
bool CommanderSetup = ParseParam("CommanderJobSetupOnly");
string ExplicitTriggerName = "";
if (CommanderSetup)
{
ExplicitTriggerName = ParseParamValue("TriggerNode", "");
}
List<BuildNode> AllNodes;
List<AggregateNode> AllAggregates;
int TimeQuantum = 20;
AddNodesForBranch(CL, HostPlatforms, bPreflightBuild, PreflightMangleSuffix, out AllNodes, out AllAggregates, ref TimeQuantum);
LinkGraph(AllAggregates, AllNodes);
FindControllingTriggers(AllNodes);
FindCompletionState(AllNodes, StoreName, LocalOnly);
ComputeDependentFrequencies(AllNodes);
if (bCleanLocalTempStorage) // shared temp storage can never be wiped
{
TempStorage.DeleteLocalTempStorageManifests(CmdEnv);
}
int TimeIndex = ParseParamInt("TimeIndex", 0);
if (TimeIndex == 0)
{
TimeIndex = ParseParamInt("UserTimeIndex", 0);
}
if (ParseParam("CIS") && ExplicitTriggerName == "" && CommanderSetup) // explicit triggers will already have a time index assigned
{
TimeIndex = UpdateCISCounter(TimeQuantum);
Log("Setting TimeIndex to {0}", TimeIndex);
}
HashSet<BuildNode> NodesToDo = ParseNodesToDo(AllNodes, AllAggregates);
CullNodesForTimeIndex(NodesToDo, TimeIndex);
CullNodesForPreflight(NodesToDo, bPreflightBuild);
TriggerNode ExplicitTrigger = null;
if (CommanderSetup)
{
if (!String.IsNullOrEmpty(ExplicitTriggerName))
{
foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>())
{
if (Trigger.Name.Equals(ExplicitTriggerName, StringComparison.InvariantCultureIgnoreCase))
{
Trigger.Activate();
ExplicitTrigger = Trigger;
break;
}
}
if (ExplicitTrigger == null)
{
throw new AutomationException("Could not find trigger node named {0}", ExplicitTriggerName);
}
}
else
{
if (bSkipTriggers)
{
foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>())
{
Trigger.Activate();
}
}
}
}
List<BuildNode> OrderedToDo = TopologicalSort(NodesToDo, ExplicitTrigger, false, false);
List<TriggerNode> UnfinishedTriggers = FindUnfinishedTriggers(bSkipTriggers, ExplicitTrigger, OrderedToDo);
PrintNodes(this, OrderedToDo, AllAggregates, UnfinishedTriggers, TimeQuantum);
//check sorting
CheckSortOrder(OrderedToDo);
ElectricCommander EC = new ElectricCommander(this);
string ShowHistoryParam = ParseParamValue("ShowHistory", null);
if(ShowHistoryParam != null)
{
BuildNode Node = AllNodes.FirstOrDefault(x => x.Name.Equals(ShowHistoryParam, StringComparison.InvariantCultureIgnoreCase));
if(Node == null)
{
throw new AutomationException("Couldn't find node {0}", ShowHistoryParam);
}
NodeHistory History = FindNodeHistory(Node, CLString, StoreName);
if(History == null)
{
throw new AutomationException("Couldn't get history for {0}", ShowHistoryParam);
}
PrintDetailedChanges(History, P4Env.Changelist);
}
else
{
string FakeFail = ParseParamValue("FakeFail");
if(CommanderSetup)
{
DoCommanderSetup(EC, AllNodes, AllAggregates, OrderedToDo, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, CLString, ExplicitTrigger, UnfinishedTriggers, FakeFail, bPreflightBuild);
}
else if(ParseParam("SaveGraph"))
{
SaveGraphVisualization(OrderedToDo);
}
else if(!ParseParam("ListOnly"))
{
ExecuteNodes(EC, OrderedToDo, bFake, bFakeEC, bSaveSharedTempStorage, CLString, StoreName, FakeFail);
}
}
PrintRunTime();
}
void GetFailureEmails(ElectricCommander EC, BuildNode NodeToDo, NodeHistory History, string CLString, string StoreName, bool OnlyLateUpdates = false)
{
string FailCauserEMails = "";
string EMailNote = "";
bool SendSuccessForGreenAfterRed = false;
int NumPeople = 0;
if (History != null)
{
if (History.LastSucceeded > 0 && History.LastSucceeded < P4Env.Changelist)
{
int LastNonDuplicateFail = P4Env.Changelist;
try
{
if (OnlyLateUpdates)
{
LastNonDuplicateFail = FindLastNonDuplicateFail(NodeToDo, History, CLString, StoreName);
if (LastNonDuplicateFail < P4Env.Changelist)
{
Log("*** Red-after-red spam reduction, changed CL {0} to CL {1} because the errors didn't change.", P4Env.Changelist, LastNonDuplicateFail);
}
}
}
catch (Exception Ex)
{
LastNonDuplicateFail = P4Env.Changelist;
Log(System.Diagnostics.TraceEventType.Warning, "Failed to FindLastNonDuplicateFail.");
Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex));
}
if(LastNonDuplicateFail > History.LastSucceeded)
{
List<P4Connection.ChangeRecord> ChangeRecords = GetSourceChangeRecords(History.LastSucceeded + 1, LastNonDuplicateFail);
foreach (P4Connection.ChangeRecord Record in ChangeRecords)
{
FailCauserEMails = GUBPNode.MergeSpaceStrings(FailCauserEMails, Record.UserEmail);
}
}
if (!String.IsNullOrEmpty(FailCauserEMails))
{
NumPeople++;
foreach (char AChar in FailCauserEMails.ToCharArray())
{
if (AChar == ' ')
{
NumPeople++;
}
}
if (NumPeople > 50)
{
EMailNote = String.Format("This step has been broken for more than 50 changes. It last succeeded at CL {0}. ", History.LastSucceeded);
}
}
}
else if (History.LastSucceeded <= 0)
{
EMailNote = String.Format("This step has been broken for more than a few days, so there is no record of it ever succeeding. ");
}
if (EMailNote != "" && !String.IsNullOrEmpty(History.FailedString))
{
EMailNote += String.Format("It has failed at CLs {0}. ", History.FailedString);
}
if (EMailNote != "" && !String.IsNullOrEmpty(History.InProgressString))
{
EMailNote += String.Format("These CLs are being built right now {0}. ", History.InProgressString);
}
if (History.LastSucceeded > 0 && History.LastSucceeded < P4Env.Changelist && History.LastFailed > History.LastSucceeded && History.LastFailed < P4Env.Changelist)
{
SendSuccessForGreenAfterRed = ParseParam("CIS");
}
}
if (History == null)
{
EC.UpdateEmailProperties(NodeToDo, 0, "", FailCauserEMails, EMailNote, SendSuccessForGreenAfterRed);
}
else
{
EC.UpdateEmailProperties(NodeToDo, History.LastSucceeded, History.FailedString, FailCauserEMails, EMailNote, SendSuccessForGreenAfterRed);
}
}
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrdereredToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, string CLString, string StoreName, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (NodeToDo.Node.BuildProducts != null || NodeToDo.Node.AllDependencyBuildProducts != null)
{
throw new AutomationException("topological sort error");
}
NodeToDo.Node.AllDependencyBuildProducts = new List<string>();
NodeToDo.Node.AllDependencies = new List<string>();
foreach (BuildNode Dep in NodeToDo.Dependencies)
{
NodeToDo.Node.AddAllDependent(Dep.Name);
if (Dep.Node.AllDependencies == null)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name);
}
foreach (string DepDep in Dep.Node.AllDependencies)
{
NodeToDo.Node.AddAllDependent(DepDep);
}
if (Dep.Node.BuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name);
}
foreach (string Prod in Dep.Node.BuildProducts)
{
NodeToDo.Node.AddDependentBuildProduct(Prod);
}
if (Dep.Node.AllDependencyBuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet2? Processing {1}", Dep.Name, NodeToDo.Name);
}
foreach (string Prod in Dep.Node.AllDependencyBuildProducts)
{
NodeToDo.Node.AddDependentBuildProduct(Prod);
}
}
string NodeStoreName = StoreName + "-" + NodeToDo.Name;
string GameNameIfAny = NodeToDo.Node.GameNameIfAnyForTempStorage();
string StorageRootIfAny = NodeToDo.Node.RootIfAnyForTempStorage();
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
Log("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, NodeStoreName);
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
Log("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, NodeStoreName);
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
Log("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, NodeStoreName);
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException("Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
EC.SaveStatus(NodeToDo, StartedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
Log("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName);
NodeToDo.Node.DoFakeBuild(this);
}
else
{
Log("***** Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName);
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.Node.DoBuild(this);
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
using(TelemetryStopwatch StoreBuildProductsStopwatch = new TelemetryStopwatch("StoreBuildProducts"))
{
double StoreDuration = 0.0;
DateTime StartTime = DateTime.UtcNow;
TempStorage.StoreToTempStorage(CmdEnv, NodeStoreName, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
StoreDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
Log("Took {0} seconds to store build products", StoreDuration);
if (IsBuildMachine)
{
EC.RunECTool(String.Format("setProperty \"/myJobStep/StoreDuration\" \"{0}\"", StoreDuration.ToString()));
}
}
if (ParseParam("StompCheck"))
{
foreach (string Dep in NodeToDo.Node.AllDependencies)
{
try
{
bool WasLocal;
using(TelemetryStopwatch RetrieveBuildProductsStopwatch = new TelemetryStopwatch("RetrieveBuildProducts"))
{
TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
if (!WasLocal)
{
throw new AutomationException("Retrieve was not local?");
}
}
catch(Exception Ex)
{
throw new AutomationException("Node {0} stomped Node {1} Ex: {2}", NodeToDo.Name, Dep, LogUtils.FormatException(Ex));
}
}
}
}
catch (Exception Ex)
{
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, CLString, StoreName);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
EC.SaveStatus(NodeToDo, FailedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, CLString, StoreName);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
Log("{0}", ExceptionToString(Ex));
if (History != null)
{
Log("Changes since last green *********************************");
Log("");
Log("");
Log("");
PrintDetailedChanges(History, P4Env.Changelist);
Log("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, CLString, StoreName);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
EC.SaveStatus(NodeToDo, SucceededTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, CLString, StoreName);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product], NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
private void DoCommanderSetup(ElectricCommander EC, IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail, bool bPreflightBuild)
{
List<BuildNode> SortedNodes = TopologicalSort(new HashSet<BuildNode>(AllNodes), null, SubSort: false, DoNotConsiderCompletion: true);
Log("******* {0} GUBP Nodes", SortedNodes.Count);
List<BuildNode> FilteredOrdereredToDo = new List<BuildNode>();
using(TelemetryStopwatch StartFilterTimer = new TelemetryStopwatch("FilterNodes"))
{
// remove nodes that have unfinished triggers
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (NodeToDo.ControllingTriggers.Length == 0 || !UnfinishedTriggers.Contains(NodeToDo.ControllingTriggers.Last()))
{
// if we are triggering, then remove nodes that are not controlled by the trigger or are dependencies of this trigger
if (ExplicitTrigger != null && ExplicitTrigger != NodeToDo && !ExplicitTrigger.DependsOn(NodeToDo) && !NodeToDo.DependsOn(ExplicitTrigger))
{
continue; // this wasn't on the chain related to the trigger we are triggering, so it is not relevant
}
// in preflight builds, we are either skipping triggers (and running things downstream) or we just stop at triggers and don't make them available for triggering.
if (bPreflightBuild && !bSkipTriggers && (NodeToDo is TriggerNode))
{
continue;
}
FilteredOrdereredToDo.Add(NodeToDo);
}
}
}
using(TelemetryStopwatch PrintNodesTimer = new TelemetryStopwatch("SetupCommanderPrint"))
{
Log("*********** EC Nodes, in order.");
PrintNodes(this, FilteredOrdereredToDo, AllAggregates, UnfinishedTriggers, TimeQuantum);
}
EC.DoCommanderSetup(AllNodes, AllAggregates, FilteredOrdereredToDo, SortedNodes, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, CLString, ExplicitTrigger, UnfinishedTriggers, FakeFail);
}
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (NodeToDo.Node.BuildProducts != null)
{
throw new AutomationException("topological sort error");
}
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage;
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
if (string.IsNullOrEmpty(StorageRootIfAny))
{
StorageRootIfAny = CmdEnv.LocalRoot;
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.DoFakeBuild();
}
else
{
LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.DoBuild();
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
//@todo: This is a dup of non-exception code. Consolidate this!!
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
LogConsole("{0}", Ex);
if (History != null)
{
LogConsole("Changes since last green *********************************");
LogConsole("");
LogConsole("");
LogConsole("");
PrintDetailedChanges(History, P4Env.Changelist);
LogConsole("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
/// <summary>
/// Main entry point for GUBP
/// </summary>
public override void ExecuteBuild()
{
LogConsole("************************* GUBP");
List<UnrealTargetPlatform> HostPlatforms = new List<UnrealTargetPlatform>();
if (!ParseParam("NoPC"))
{
HostPlatforms.Add(UnrealTargetPlatform.Win64);
}
if (!ParseParam("NoMac"))
{
HostPlatforms.Add(UnrealTargetPlatform.Mac);
}
if(!ParseParam("NoLinux") && UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux)
{
HostPlatforms.Add(UnrealTargetPlatform.Linux);
}
TempStorage.SetZipTempStorage(!ParseParam("NoZipTempStorage"));
bool bCleanLocalTempStorage = ParseParam("CleanLocal");
bool bSkipTriggers = ParseParam("SkipTriggers");
bool bFake = ParseParam("fake");
bool bFakeEC = ParseParam("FakeEC");
bool bSaveSharedTempStorage = P4Enabled && (IsBuildMachine || GlobalCommandLine.UseLocalBuildStorage);
// encapsulate the logic and temp vars to determine the job info.
// Ensures the temp vars don't escape the scope to be used later.
Func<JobInfo> GetJobInfo = () =>
{
int PreflightShelveCL = 0;
int PreflightUID = 0;
string PreflightShelveCLString = GetEnvVar("uebp_PreflightShelveCL");
// We must be passed a valid shelve and be a build machine (or be running a preflight test) to be a valid preflight.
if (!string.IsNullOrEmpty(PreflightShelveCLString) && (IsBuildMachine || ParseParam("PreflightTest")))
{
LogConsole("**** Preflight shelve {0}", PreflightShelveCLString);
PreflightShelveCL = int.Parse(PreflightShelveCLString);
if (PreflightShelveCL < 2000000)
{
throw new AutomationException("{0} does not look like a CL", PreflightShelveCL);
}
PreflightUID = ParseParamInt("PreflightUID", 0);
}
return new JobInfo(
P4Enabled ? P4Env.BuildRootEscaped : "NoP4",
P4Enabled ? ParseParamInt("CL", P4Env.Changelist) : 0,
PreflightShelveCL,
PreflightUID);
};
var JobInfo = GetJobInfo();
bool LocalOnly = !P4Enabled || bFakeEC;
if (bSaveSharedTempStorage)
{
if (!TempStorage.IsSharedTempStorageAvailable(true))
{
throw new AutomationException("Request to save to shared temp storage, but shared temp storage is unavailable or does not exist.");
}
}
else if (!LocalOnly && !TempStorage.IsSharedTempStorageAvailable(false))
{
LogWarning("Looks like we want to use shared temp storage, but since we don't have it, we won't use it.");
LocalOnly = true;
}
bool CommanderSetup = ParseParam("CommanderJobSetupOnly");
string ExplicitTriggerName = "";
if (CommanderSetup)
{
ExplicitTriggerName = ParseParamValue("TriggerNode", "");
}
List<BuildNode> AllNodes;
List<AggregateNode> AllAggregates;
int TimeQuantum = 20;
AddNodesForBranch(HostPlatforms, JobInfo, out AllNodes, out AllAggregates, ref TimeQuantum);
LinkGraph(AllAggregates, AllNodes);
FindControllingTriggers(AllNodes);
FindCompletionState(AllNodes, JobInfo, LocalOnly);
ComputeDependentFrequencies(AllNodes);
if (bCleanLocalTempStorage) // shared temp storage can never be wiped
{
TempStorage.DeleteLocalTempStorageManifests();
}
int TimeIndex = ParseParamInt("TimeIndex", 0);
if (TimeIndex == 0)
{
TimeIndex = ParseParamInt("UserTimeIndex", 0);
}
if (ParseParam("CIS") && ExplicitTriggerName == "" && CommanderSetup) // explicit triggers will already have a time index assigned
{
TimeIndex = UpdateCISCounter(TimeQuantum);
Log("Setting TimeIndex to {0}", TimeIndex);
}
HashSet<BuildNode> NodesToDo = ParseNodesToDo(AllNodes, AllAggregates);
CullNodesForTimeIndex(NodesToDo, TimeIndex);
if (JobInfo.IsPreflight)
{
CullNodesForPreflight(NodesToDo);
}
TriggerNode ExplicitTrigger = null;
if (CommanderSetup)
{
if (!String.IsNullOrEmpty(ExplicitTriggerName))
{
foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>())
{
if (Trigger.Name.Equals(ExplicitTriggerName, StringComparison.InvariantCultureIgnoreCase))
{
Trigger.Activate();
ExplicitTrigger = Trigger;
break;
}
}
if (ExplicitTrigger == null)
{
throw new AutomationException("Could not find trigger node named {0}", ExplicitTriggerName);
}
}
else
{
if (bSkipTriggers)
{
foreach (TriggerNode Trigger in AllNodes.OfType<TriggerNode>())
{
Trigger.Activate();
}
}
}
}
List<BuildNode> OrderedToDo = TopologicalSort(NodesToDo, ExplicitTrigger, false, false);
List<TriggerNode> UnfinishedTriggers = FindUnfinishedTriggers(bSkipTriggers, ExplicitTrigger, OrderedToDo);
PrintNodes(this, OrderedToDo, AllAggregates, UnfinishedTriggers, TimeQuantum);
//check sorting
CheckSortOrder(OrderedToDo);
ElectricCommander EC = new ElectricCommander(this);
string ShowHistoryParam = ParseParamValue("ShowHistory", null);
if(ShowHistoryParam != null)
{
BuildNode Node = AllNodes.FirstOrDefault(x => x.Name.Equals(ShowHistoryParam, StringComparison.InvariantCultureIgnoreCase));
if(Node == null)
{
throw new AutomationException("Couldn't find node {0}", ShowHistoryParam);
}
NodeHistory History = FindNodeHistory(Node, JobInfo);
if(History == null)
{
throw new AutomationException("Couldn't get history for {0}", ShowHistoryParam);
}
PrintDetailedChanges(History, P4Env.Changelist);
}
else
{
string FakeFail = ParseParamValue("FakeFail");
if(CommanderSetup)
{
DoCommanderSetup(EC, AllNodes, AllAggregates, OrderedToDo, TimeIndex, TimeQuantum, bSkipTriggers, bFake, bFakeEC, ExplicitTrigger, UnfinishedTriggers, FakeFail, JobInfo.IsPreflight);
}
else if(!ParseParam("ListOnly"))
{
ExecuteNodes(EC, OrderedToDo, bFake, bFakeEC, bSaveSharedTempStorage, JobInfo, FakeFail);
}
}
PrintRunTime();
}