void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (NodeToDo.Node.BuildProducts != null)
{
throw new AutomationException("topological sort error");
}
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage;
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
if (string.IsNullOrEmpty(StorageRootIfAny))
{
StorageRootIfAny = CmdEnv.LocalRoot;
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.DoFakeBuild();
}
else
{
LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.DoBuild();
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
//@todo: This is a dup of non-exception code. Consolidate this!!
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
LogConsole("{0}", Ex);
if (History != null)
{
LogConsole("Changes since last green *********************************");
LogConsole("");
LogConsole("");
LogConsole("");
PrintDetailedChanges(History, P4Env.Changelist);
LogConsole("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
static void FindCompletionState(IEnumerable<BuildNode> NodesToDo, JobInfo JobInfo, bool LocalOnly)
{
foreach(BuildNode NodeToDo in NodesToDo)
{
// Construct the full temp storage node info
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
if (LocalOnly)
{
NodeToDo.IsComplete = TempStorage.LocalTempStorageManifestExists(TempStorageNodeInfo, bQuiet: true);
}
else
{
NodeToDo.IsComplete = TempStorage.TempStorageExists(TempStorageNodeInfo, GameNameIfAny, bLocalOnly: false, bQuiet: true);
if(GameNameIfAny != "" && !NodeToDo.IsComplete)
{
NodeToDo.IsComplete = TempStorage.TempStorageExists(TempStorageNodeInfo, "UE4", bLocalOnly: false, bQuiet: true);
}
}
LogVerbose("** {0}", NodeToDo.Name);
if (!NodeToDo.IsComplete)
{
LogVerbose("***** GUBP Trigger Node was already triggered {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
}
else
{
LogVerbose("***** GUBP Trigger Node was NOT yet triggered {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
}
}
}
/// <summary>
/// Returns the full path to the temp storage directory for the given temp storage node and game (ie, P:\Builds\GameName\TmpStore\NodeInfoDirectory)
/// </summary>
/// <param name="TempStorageNodeInfo">Node info descibing the block of temp storage (essentially used to identify a subdirectory insides the game's temp storage folder).</param>
/// <param name="GameName">game name to determine the temp storage folder for. Empty is equivalent to "UE4".</param>
/// <returns>The full path to the temp storage directory for the given storage block name for the given game.</returns>
private static string SharedTempStorageDirectory(TempStorageNodeInfo TempStorageNodeInfo, string GameName)
{
return CommandUtils.CombinePaths(ResolveSharedTempStorageDirectory(GameName), TempStorageNodeInfo.GetRelativeDirectory());
}
