void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrdereredToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, string CLString, string StoreName, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (NodeToDo.Node.BuildProducts != null || NodeToDo.Node.AllDependencyBuildProducts != null)
{
throw new AutomationException("topological sort error");
}
NodeToDo.Node.AllDependencyBuildProducts = new List<string>();
NodeToDo.Node.AllDependencies = new List<string>();
foreach (BuildNode Dep in NodeToDo.Dependencies)
{
NodeToDo.Node.AddAllDependent(Dep.Name);
if (Dep.Node.AllDependencies == null)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name);
}
foreach (string DepDep in Dep.Node.AllDependencies)
{
NodeToDo.Node.AddAllDependent(DepDep);
}
if (Dep.Node.BuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo.Name);
}
foreach (string Prod in Dep.Node.BuildProducts)
{
NodeToDo.Node.AddDependentBuildProduct(Prod);
}
if (Dep.Node.AllDependencyBuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet2? Processing {1}", Dep.Name, NodeToDo.Name);
}
foreach (string Prod in Dep.Node.AllDependencyBuildProducts)
{
NodeToDo.Node.AddDependentBuildProduct(Prod);
}
}
string NodeStoreName = StoreName + "-" + NodeToDo.Name;
string GameNameIfAny = NodeToDo.Node.GameNameIfAnyForTempStorage();
string StorageRootIfAny = NodeToDo.Node.RootIfAnyForTempStorage();
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
Log("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, NodeStoreName);
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
Log("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, NodeStoreName);
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
Log("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, NodeStoreName);
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException("Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
EC.SaveStatus(NodeToDo, StartedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
Log("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName);
NodeToDo.Node.DoFakeBuild(this);
}
else
{
Log("***** Building GUBP Node {0} for {1}", NodeToDo.Name, NodeStoreName);
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.Node.DoBuild(this);
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
using(TelemetryStopwatch StoreBuildProductsStopwatch = new TelemetryStopwatch("StoreBuildProducts"))
{
double StoreDuration = 0.0;
DateTime StartTime = DateTime.UtcNow;
TempStorage.StoreToTempStorage(CmdEnv, NodeStoreName, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
StoreDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
Log("Took {0} seconds to store build products", StoreDuration);
if (IsBuildMachine)
{
EC.RunECTool(String.Format("setProperty \"/myJobStep/StoreDuration\" \"{0}\"", StoreDuration.ToString()));
}
}
if (ParseParam("StompCheck"))
{
foreach (string Dep in NodeToDo.Node.AllDependencies)
{
try
{
bool WasLocal;
using(TelemetryStopwatch RetrieveBuildProductsStopwatch = new TelemetryStopwatch("RetrieveBuildProducts"))
{
TempStorage.RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
if (!WasLocal)
{
throw new AutomationException("Retrieve was not local?");
}
}
catch(Exception Ex)
{
throw new AutomationException("Node {0} stomped Node {1} Ex: {2}", NodeToDo.Name, Dep, LogUtils.FormatException(Ex));
}
}
}
}
catch (Exception Ex)
{
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, CLString, StoreName);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
EC.SaveStatus(NodeToDo, FailedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, CLString, StoreName);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
Log("{0}", ExceptionToString(Ex));
if (History != null)
{
Log("Changes since last green *********************************");
Log("");
Log("");
Log("");
PrintDetailedChanges(History, P4Env.Changelist);
Log("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, CLString, StoreName);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
EC.SaveStatus(NodeToDo, SucceededTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, CLString, StoreName);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product], NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
void ExecuteNodes(ElectricCommander EC, List<BuildNode> OrderedToDo, bool bFake, bool bFakeEC, bool bSaveSharedTempStorage, JobInfo JobInfo, string FakeFail)
{
Dictionary<string, BuildNode> BuildProductToNodeMap = new Dictionary<string, BuildNode>();
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (NodeToDo.Node.BuildProducts != null)
{
throw new AutomationException("topological sort error");
}
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name);
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
string StorageRootIfAny = NodeToDo.RootIfAnyForTempStorage;
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
if (string.IsNullOrEmpty(StorageRootIfAny))
{
StorageRootIfAny = CmdEnv.LocalRoot;
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!(NodeToDo is TriggerNode) || NodeToDo.IsSticky); // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
LogConsole("***** Running GUBP Node {0} -> {1} : {2}", NodeToDo.Name, GameNameIfAny, TempStorageNodeInfo.GetRelativeDirectory());
if (NodeToDo.IsComplete)
{
if (NodeToDo.Name == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
LogConsole("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.Node.BuildProducts = new List<string>();
}
else
{
LogConsole("***** Retrieving GUBP Node {0} from {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
bool WasLocal;
try
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
if(GameNameIfAny != "")
{
NodeToDo.Node.BuildProducts = TempStorage.RetrieveFromTempStorage(TempStorageNodeInfo, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException(Ex, "Build Products cannot be found for node {0}", NodeToDo.Name);
}
}
if (!WasLocal)
{
NodeToDo.Node.PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
double BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo.Name, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo.Name);
}
if (bFake)
{
LogConsole("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
NodeToDo.DoFakeBuild();
}
else
{
LogConsole("***** Building GUBP Node {0} for {1}", NodeToDo.Name, TempStorageNodeInfo.GetRelativeDirectory());
DateTime StartTime = DateTime.UtcNow;
using(TelemetryStopwatch DoBuildStopwatch = new TelemetryStopwatch("DoBuild.{0}", NodeToDo.Name))
{
NodeToDo.DoBuild();
}
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
TempStorage.StoreToTempStorage(TempStorageNodeInfo, NodeToDo.Node.BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
}
catch (Exception Ex)
{
//@todo: This is a dup of non-exception code. Consolidate this!!
NodeHistory History = null;
if (SaveSuccessRecords)
{
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
LogConsole("{0}", Ex);
if (History != null)
{
LogConsole("Changes since last green *********************************");
LogConsole("");
LogConsole("");
LogConsole("");
PrintDetailedChanges(History, P4Env.Changelist);
LogConsole("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
NodeHistory History = null;
using(TelemetryStopwatch UpdateNodeHistoryStopwatch = new TelemetryStopwatch("UpdateNodeHistory"))
{
History = FindNodeHistory(NodeToDo, JobInfo);
}
using(TelemetryStopwatch SaveNodeStatusStopwatch = new TelemetryStopwatch("SaveNodeStatus"))
{
// We save our status to a new temp storage location specifically named with a suffix so we can find it later.
EC.SaveStatus(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), bSaveSharedTempStorage, GameNameIfAny);
}
using(TelemetryStopwatch UpdateECPropsStopwatch = new TelemetryStopwatch("UpdateECProps"))
{
EC.UpdateECProps(NodeToDo);
}
if (IsBuildMachine)
{
using(TelemetryStopwatch GetFailEmailsStopwatch = new TelemetryStopwatch("GetFailEmails"))
{
GetFailureEmails(EC, NodeToDo, History, JobInfo);
}
}
EC.UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (string Product in NodeToDo.Node.BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product].ToString(), NodeToDo.Name, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}