/// <summary>
/// Initializes a new instance of the <see cref="SpiceInstance"/> class.
/// </summary>
/// <param name="parameters">The method description.</param>
/// <param name="state">The biasing simulation state.</param>
/// <param name="maxOrder">The maximum order.</param>
protected SpiceInstance(SpiceMethod parameters, IBiasingSimulationState state, int maxOrder)
{
Parameters = parameters.ThrowIfNull(nameof(parameters));
State = state.ThrowIfNull(nameof(state));
MaxOrder = maxOrder;
States = new NodeHistory <SpiceIntegrationState>(maxOrder + 2);
}
public void WhenNodeHasPlacementWithName_ThenZoneAndProjectIdAreNotNull()
{
var instance = new InstanceHistory(
1,
new InstanceLocator("project-1", "zone-1", "instance-1"),
InstanceHistoryState.MissingImage,
null,
null);
var node = new NodeHistory(
"server-1",
null,
DateTime.UtcNow.AddDays(-1),
DateTime.UtcNow,
0,
new[]
{
new NodePlacement(
DateTime.UtcNow.AddDays(-1),
DateTime.UtcNow,
instance)
});
Assert.AreEqual("project-1", node.ProjectId);
Assert.AreEqual("zone-1", node.Zone);
}
public void WhenNodeHasPlacementWithoutName_ThenZoneAndProjectIdAreNull()
{
var instance = new InstanceHistory(
1,
null,
InstanceHistoryState.MissingName,
null,
null);
var node = new NodeHistory(
"server-1",
null,
DateTime.UtcNow.AddDays(-1),
DateTime.UtcNow,
0,
new[]
{
new NodePlacement(
DateTime.UtcNow.AddDays(-1),
DateTime.UtcNow,
instance)
});
Assert.IsNull(node.ProjectId);
Assert.IsNull(node.Zone);
}
public void WhenNodeHasPlacements_ThenMaxInstancePlacementsByDayUsesFirstAndLastDate()
{
var baselineDate = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var instance = new InstanceHistory(
1,
new InstanceLocator("project-1", "zone-1", "instance-1"),
InstanceHistoryState.MissingImage,
null,
null);
var node = new NodeHistory(
"server-1",
null,
baselineDate,
baselineDate.AddDays(2),
0,
new[]
{
new NodePlacement(
baselineDate.AddDays(1),
baselineDate.AddDays(2),
instance),
new NodePlacement(
baselineDate,
baselineDate.AddDays(1),
instance)
});
var histogram = node.MaxInstancePlacementsByDay;
Assert.AreEqual(baselineDate, histogram.First().Timestamp);
Assert.AreEqual(baselineDate.AddDays(2), histogram.Last().Timestamp);
}
public void WhenNodeHasNoPlacements_ThenMaxInstancePlacementsByDayIsEmpty()
{
var node = new NodeHistory(
"server-1",
DateTime.UtcNow.AddDays(-1),
DateTime.UtcNow,
0,
Enumerable.Empty <NodePlacement>());
Assert.IsFalse(node.MaxInstancePlacementsByDay.Any());
}
public void WhenNodeHasNoPlacements_ThenZoneAndProjectIdAreNull()
{
var node = new NodeHistory(
"server-1",
DateTime.UtcNow.AddDays(-1),
DateTime.UtcNow,
0,
Enumerable.Empty <NodePlacement>());
Assert.IsNull(node.ProjectId);
Assert.IsNull(node.Zone);
}
internal override void Repopulate()
{
// Get instances, filtered by whatever filter applies,
// then derive sole tenant nodes (ignoring fleet).
this.currentNodeSet = NodeSetHistory.FromInstancyHistory(
this.parent.GetInstancesMatchingCurrentFilters(),
Tenancies.SoleTenant);
// Create histogram, disregarding the date selection.
this.Histogram = this.currentNodeSet.MaxNodesByDay;
Debug.Assert(this.currentNodeSet.Nodes.All(n => n.ServerId != null));
// For the list of nodes, apply the date selection.
this.Nodes.Clear();
this.Nodes.AddRange(this.currentNodeSet.Nodes
.Where(n => n.FirstUse <= this.Selection.EndDate && n.LastUse >= this.Selection.StartDate));
this.selectedNode = null;
RepopulateNodePlacements();
}
private void AddToHistory(Node previousNode, Node currentCandidateNode)
{
if (previousNode == currentCandidateNode)
{
return;
}
// Add node to dictionary first
NodeDictionary.Add(previousNode.Id, previousNode);
if (NodeHistory.ContainsKey(previousNode.Id))
{
NodeHistory[previousNode.Id].Add(currentCandidateNode.Id);
}
else
{
NodeHistory[previousNode.Id] = new List <string>();
}
NodeHistory[previousNode.Id].Add(currentCandidateNode.Id);
}
public override void ExecuteBuild()
{
Log("************************* GUBP");
var HostPlatforms = new List<UnrealTargetPlatform>();
if (!ParseParam("OnlyMac"))
{
HostPlatforms.Add(UnrealTargetPlatform.Win64);
}
if (ParseParam("WithMac") || HostPlatforms.Count == 0)
{
HostPlatforms.Add(UnrealTargetPlatform.Mac);
}
StoreName = ParseParamValue("Store");
string StoreSuffix = ParseParamValue("StoreSuffix", "");
CL = ParseParamInt("CL", 0);
bool bCleanLocalTempStorage = ParseParam("CleanLocal");
bool bChanges = ParseParam("Changes") || ParseParam("AllChanges");
bool bHistory = ParseParam("History") || bChanges;
bool bListOnly = ParseParam("ListOnly") || bHistory;
bool bSkipTriggers = ParseParam("SkipTriggers");
bool bFake = ParseParam("fake");
bool bFakeEC = ParseParam("FakeEC");
bWide = ParseParam("Wide");
bool bSaveSharedTempStorage = false;
if (bHistory && !P4Enabled)
{
throw new AutomationException("-Changes and -History require -P4.");
}
bool LocalOnly = true;
string CLString = "";
if (String.IsNullOrEmpty(StoreName))
{
if (P4Enabled)
{
if (CL == 0)
{
CL = P4Env.Changelist;
}
CLString = String.Format("{0}", CL);
StoreName = "GUBP-" + P4Env.BuildRootEscaped + "-CL-" + CLString + "-" + StoreSuffix;
bSaveSharedTempStorage = CommandUtils.IsBuildMachine;
LocalOnly = false;
}
else
{
StoreName = "TempLocal" + "-" + StoreSuffix;
bSaveSharedTempStorage = false;
}
}
if (bFakeEC)
{
LocalOnly = true;
}
if (bSaveSharedTempStorage)
{
if (!HaveSharedTempStorage())
{
throw new AutomationException("Request to save to temp storage, but {0} is unavailable.", UE4TempStorageDirectory());
}
bSignBuildProducts = true;
}
else if (!LocalOnly && !HaveSharedTempStorage())
{
Log("Looks like we want to use shared temp storage, but since we don't have it, we won't use it.");
LocalOnly = false;
}
Log("************************* CL: {0}", CL);
Log("************************* P4Enabled: {0}", P4Enabled);
Log("************************* HostPlatform: {0}", HostPlatforms.ToString());
Log("************************* StoreName: {0}", StoreName.ToString());
Log("************************* bCleanLocalTempStorage: {0}", bCleanLocalTempStorage);
Log("************************* bSkipTriggers: {0}", bSkipTriggers);
Log("************************* bSaveSharedTempStorage: {0}", bSaveSharedTempStorage);
Log("************************* bSignBuildProducts: {0}", bSignBuildProducts);
Log("************************* bFake: {0}", bFake);
Log("************************* bFakeEC: {0}", bFakeEC);
Log("************************* bHistory: {0}", bHistory);
GUBPNodes = new Dictionary<string, GUBPNode>();
Branch = new BranchInfo(HostPlatforms);
if (IsBuildMachine || ParseParam("AllPlatforms"))
{
ActivePlatforms = new List<UnrealTargetPlatform>();
foreach (var GameProj in Branch.CodeProjects)
{
foreach (var Kind in BranchInfo.MonolithicKinds)
{
if (GameProj.Properties.Targets.ContainsKey(Kind))
{
var Target = GameProj.Properties.Targets[Kind];
foreach (var HostPlatform in HostPlatforms)
{
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
foreach (var Plat in Platforms)
{
if (Target.Rules.SupportsPlatform(Plat) && !ActivePlatforms.Contains(Plat))
{
ActivePlatforms.Add(Plat);
}
}
}
}
}
}
}
else
{
ActivePlatforms = new List<UnrealTargetPlatform>(CommandUtils.KnownTargetPlatforms);
}
foreach (var Plat in ActivePlatforms)
{
Log("Active Platform: {0}", Plat.ToString());
}
AddNode(new VersionFilesNode());
foreach (var HostPlatform in HostPlatforms)
{
AddNode(new RootEditorNode(HostPlatform));
AddNode(new ToolsNode(HostPlatform));
AddNode(new EditorAndToolsNode(HostPlatform));
AddNode(new NonUnityTestNode(HostPlatform));
foreach (var Plat in ActivePlatforms)
{
if (Plat != HostPlatform && Plat != GetAltHostPlatform(HostPlatform))
{
AddNode(new EditorPlatformNode(HostPlatform, Plat));
}
}
int NumShared = 0;
foreach (var CodeProj in Branch.CodeProjects)
{
var Options = CodeProj.Options(HostPlatform);
if (Options.bIsPromotable && !Options.bSeparateGamePromotion)
{
NumShared++;
}
if (!Options.bIsPromotable && !Options.bTestWithShared)
{
continue; // we skip things that aren't promotable and aren't tested
}
AddNode(new EditorGameNode(this, HostPlatform, CodeProj));
{
var EditorTests = CodeProj.Properties.Targets[TargetRules.TargetType.Editor].Rules.GUBP_GetEditorTests_EditorTypeOnly();
var EditorTestNodes = new List<string>();
foreach (var Test in EditorTests)
{
EditorTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, CodeProj, Test.Key, Test.Value)));
}
if (EditorTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, CodeProj, "AllEditorTests", EditorTestNodes, true, 0.0f));
}
}
var ServerPlatforms = new List<UnrealTargetPlatform>();
var GamePlatforms = new List<UnrealTargetPlatform>();
var GameTestNodes = new List<string>();
foreach (var Kind in BranchInfo.MonolithicKinds)
{
if (CodeProj.Properties.Targets.ContainsKey(Kind))
{
var Target = CodeProj.Properties.Targets[Kind];
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
foreach (var Plat in Platforms)
{
if (ActivePlatforms.Contains(Plat))
{
if (Kind == TargetRules.TargetType.Server && !ServerPlatforms.Contains(Plat))
{
ServerPlatforms.Add(Plat);
}
if (Kind == TargetRules.TargetType.Game && !GamePlatforms.Contains(Plat))
{
GamePlatforms.Add(Plat);
}
if (!GUBPNodes.ContainsKey(GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, CodeProj, Plat)))
{
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, CodeProj, Plat));
}
string CookedPlatform = Platform.Platforms[Plat].GetCookPlatform(Kind == TargetRules.TargetType.Server, Kind == TargetRules.TargetType.Client, "");
if (!GUBPNodes.ContainsKey(CookNode.StaticGetFullName(HostPlatform, CodeProj, CookedPlatform)))
{
AddNode(new CookNode(this, HostPlatform, CodeProj, Plat, CookedPlatform));
}
if (!GUBPNodes.ContainsKey(GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, CodeProj, Plat)))
{
AddNode(new GamePlatformCookedAndCompiledNode(this, HostPlatform, CodeProj, Plat, true));
}
var GameTests = Target.Rules.GUBP_GetGameTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), Plat);
var RequiredPlatforms = new List<UnrealTargetPlatform> { Plat };
var ThisMonoGameTestNodes = new List<string>();
foreach (var Test in GameTests)
{
ThisMonoGameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, CodeProj, Test.Key + "_" + Plat.ToString(), Test.Value, false, RequiredPlatforms)));
}
if (ThisMonoGameTestNodes.Count > 0)
{
GameTestNodes.Add(AddNode(new GameAggregateNode(this, HostPlatform, CodeProj, "CookedTests_" + Plat.ToString() + "_" + Kind.ToString(), ThisMonoGameTestNodes, true, 0.0f)));
}
}
}
}
}
foreach (var ServerPlatform in ServerPlatforms)
{
foreach (var GamePlatform in GamePlatforms)
{
var Target = CodeProj.Properties.Targets[TargetRules.TargetType.Game];
var ClientServerTests = Target.Rules.GUBP_GetClientServerTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), ServerPlatform, GamePlatform);
var RequiredPlatforms = new List<UnrealTargetPlatform> { ServerPlatform };
if (ServerPlatform != GamePlatform)
{
RequiredPlatforms.Add(GamePlatform);
}
foreach (var Test in ClientServerTests)
{
GameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, CodeProj, Test.Key + "_" + GamePlatform.ToString() + "_" + ServerPlatform.ToString(), Test.Value, false, RequiredPlatforms, true)));
}
}
}
if (GameTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, CodeProj, "AllCookedTests", GameTestNodes, false));
}
}
bool DoASharedPromotable = false;
{
var Options = Branch.BaseEngineProject.Options(HostPlatform);
if (!Options.bIsPromotable || Options.bSeparateGamePromotion)
{
if (NumShared > 0)
{
throw new AutomationException("We assume that if we have shared promotable, the base engine is in it. Some games are marked as shared and promotable, but the base engine is not.");
}
}
else
{
DoASharedPromotable = true;
}
}
if (DoASharedPromotable)
{
Dictionary<string, List<UnrealTargetPlatform>> NonCodeProjectNames;
{
var Target = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Editor];
NonCodeProjectNames = Target.Rules.GUBP_NonCodeProjects_BaseEditorTypeOnly(HostPlatform);
var Options = Branch.BaseEngineProject.Options(HostPlatform);
if (!Options.bIsPromotable || Options.bSeparateGamePromotion)
{
throw new AutomationException("We assume that if we have shared promotable, the base engine is in it.");
}
}
{
var EditorTests = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Editor].Rules.GUBP_GetEditorTests_EditorTypeOnly();
var EditorTestNodes = new List<string>();
foreach (var Test in EditorTests)
{
EditorTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, Branch.BaseEngineProject, Test.Key, Test.Value)));
foreach (var NonCodeProject in Branch.NonCodeProjects)
{
if (!NonCodeProjectNames.ContainsKey(NonCodeProject.GameName))
{
continue;
}
EditorTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, NonCodeProject, Test.Key, Test.Value)));
}
}
if (EditorTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, Branch.BaseEngineProject, "AllEditorTests", EditorTestNodes, true, 0.0f));
}
}
var ServerPlatforms = new List<UnrealTargetPlatform>();
var GamePlatforms = new List<UnrealTargetPlatform>();
foreach (var Kind in BranchInfo.MonolithicKinds)
{
if (Branch.BaseEngineProject.Properties.Targets.ContainsKey(Kind))
{
var Target = Branch.BaseEngineProject.Properties.Targets[Kind];
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
foreach (var Plat in Platforms)
{
if (ActivePlatforms.Contains(Plat))
{
if (Kind == TargetRules.TargetType.Server && !ServerPlatforms.Contains(Plat))
{
ServerPlatforms.Add(Plat);
}
if (Kind == TargetRules.TargetType.Game && !GamePlatforms.Contains(Plat))
{
GamePlatforms.Add(Plat);
}
if (!GUBPNodes.ContainsKey(GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, Branch.BaseEngineProject, Plat)))
{
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, Branch.BaseEngineProject, Plat));
}
}
}
}
}
var GameTestNodes = new List<string>();
var GameCookNodes = new List<string>();
//foreach (var Kind in BranchInfo.MonolithicKinds)//for now, non-code projects don't do client or server.
{
var Kind = TargetRules.TargetType.Game;
if (Branch.BaseEngineProject.Properties.Targets.ContainsKey(Kind))
{
var Target = Branch.BaseEngineProject.Properties.Targets[Kind];
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
foreach (var Plat in Platforms)
{
if (ActivePlatforms.Contains(Plat))
{
foreach (var NonCodeProject in Branch.NonCodeProjects)
{
if (!NonCodeProjectNames.ContainsKey(NonCodeProject.GameName) || !NonCodeProjectNames[NonCodeProject.GameName].Contains(Plat))
{
continue;
}
string CookedPlatform = Platform.Platforms[Plat].GetCookPlatform(Kind == TargetRules.TargetType.Server, Kind == TargetRules.TargetType.Client, "");
if (!GUBPNodes.ContainsKey(CookNode.StaticGetFullName(HostPlatform, NonCodeProject, CookedPlatform)))
{
GameCookNodes.Add(AddNode(new CookNode(this, HostPlatform, NonCodeProject, Plat, CookedPlatform)));
}
if (!GUBPNodes.ContainsKey(GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, NonCodeProject, Plat)))
{
AddNode(new GamePlatformCookedAndCompiledNode(this, HostPlatform, NonCodeProject, Plat, false));
}
var GameTests = Target.Rules.GUBP_GetGameTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), Plat);
var RequiredPlatforms = new List<UnrealTargetPlatform> { Plat };
var ThisMonoGameTestNodes = new List<string>();
foreach (var Test in GameTests)
{
ThisMonoGameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, NonCodeProject, Test.Key + "_" + Plat.ToString(), Test.Value, false, RequiredPlatforms)));
}
if (ThisMonoGameTestNodes.Count > 0)
{
GameTestNodes.Add(AddNode(new GameAggregateNode(this, HostPlatform, NonCodeProject, "CookedTests_" + Plat.ToString() + "_" + Kind.ToString(), ThisMonoGameTestNodes, true, 0.0f)));
}
}
}
}
}
}
#if false
//for now, non-code projects don't do client or server.
foreach (var ServerPlatform in ServerPlatforms)
{
var ServerTarget = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Server];
foreach (var GamePlatform in GamePlatforms)
{
var Target = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Game];
foreach (var NonCodeProject in Branch.NonCodeProjects)
{
if (!NonCodeProjectNames.ContainsKey(NonCodeProject.GameName) || !NonCodeProjectNames.ContainsKey(NonCodeProject.GameName) ||
!NonCodeProjectNames[NonCodeProject.GameName].Contains(ServerPlatform) || !NonCodeProjectNames[NonCodeProject.GameName].Contains(GamePlatform) )
{
continue;
}
var ClientServerTests = Target.Rules.GUBP_GetClientServerTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), ServerPlatform, GamePlatform);
var RequiredPlatforms = new List<UnrealTargetPlatform> { ServerPlatform };
if (ServerPlatform != GamePlatform)
{
RequiredPlatforms.Add(GamePlatform);
}
foreach (var Test in ClientServerTests)
{
GameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, NonCodeProject, Test.Key + "_" + GamePlatform.ToString() + "_" + ServerPlatform.ToString(), Test.Value, false, RequiredPlatforms, true)));
}
}
}
}
#endif
if (GameTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, Branch.BaseEngineProject, "AllCookedTests", GameTestNodes, false));
}
}
AddCustomNodes(HostPlatform);
}
int NumSharedAllHosts = 0;
foreach (var CodeProj in Branch.CodeProjects)
{
if (CodeProj.Properties.Targets.ContainsKey(TargetRules.TargetType.Editor))
{
var Target = CodeProj.Properties.Targets[TargetRules.TargetType.Editor];
bool AnySeparate = false;
var PromotedHosts = new List<UnrealTargetPlatform>();
foreach (var HostPlatform in HostPlatforms)
{
var Options = CodeProj.Options(HostPlatform);
AnySeparate = AnySeparate || Options.bSeparateGamePromotion;
if (Options.bIsPromotable)
{
if (!Options.bSeparateGamePromotion)
{
NumSharedAllHosts++;
}
PromotedHosts.Add(HostPlatform);
}
}
if (PromotedHosts.Count > 0)
{
AddNode(new GameAggregatePromotableNode(this, PromotedHosts, CodeProj));
if (AnySeparate)
{
AddNode(new WaitForGamePromotionUserInput(this, CodeProj, false));
AddNode(new GameLabelPromotableNode(this, CodeProj, false));
AddNode(new WaitForGamePromotionUserInput(this, CodeProj, true));
AddNode(new GameLabelPromotableNode(this, CodeProj, true));
}
}
}
}
if (NumSharedAllHosts > 0)
{
AddNode(new GameAggregatePromotableNode(this, HostPlatforms, Branch.BaseEngineProject));
AddNode(new SharedAggregatePromotableNode(this, HostPlatforms));
AddNode(new WaitForSharedPromotionUserInput(this, false));
AddNode(new SharedLabelPromotableNode(this, false));
AddNode(new WaitForTestShared(this));
AddNode(new WaitForSharedPromotionUserInput(this, true));
AddNode(new SharedLabelPromotableNode(this, true));
}
var NodesToDo = new HashSet<string>();
if (bCleanLocalTempStorage) // shared temp storage can never be wiped
{
DeleteLocalTempStorageManifests(CmdEnv);
}
Log("******* Caching completion");
GUBPNodesCompleted = new Dictionary<string, bool>();
GUBPNodesControllingTrigger = new Dictionary<string, string>();
GUBPNodesControllingTriggerDotName = new Dictionary<string, string>();
GUBPNodesHistory = new Dictionary<string, NodeHistory>();
foreach (var Node in GUBPNodes)
{
Log("** {0}", Node.Key);
NodeIsAlreadyComplete(Node.Key, LocalOnly); // cache these now to avoid spam later
GetControllingTriggerDotName(Node.Key);
if (CLString != "" && StoreName.Contains(CLString))
{
if (GUBPNodes[Node.Key].RunInEC() && !GUBPNodes[Node.Key].TriggerNode())
{
string GameNameIfAny = GUBPNodes[Node.Key].GameNameIfAnyForTempStorage();
string NodeStoreWildCard = StoreName.Replace(CLString, "*") + "-" + GUBPNodes[Node.Key].GetFullName();
var History = new NodeHistory();
History.AllStarted = ConvertCLToIntList(FindTempStorageManifests(CmdEnv, NodeStoreWildCard + StartedTempStorageSuffix, false, true, GameNameIfAny));
History.AllSucceeded = ConvertCLToIntList(FindTempStorageManifests(CmdEnv, NodeStoreWildCard + SucceededTempStorageSuffix, false, true, GameNameIfAny));
History.AllFailed = ConvertCLToIntList(FindTempStorageManifests(CmdEnv, NodeStoreWildCard + FailedTempStorageSuffix, false, true, GameNameIfAny));
if (History.AllSucceeded.Count > 0)
{
History.LastSucceeded = History.AllSucceeded[History.AllSucceeded.Count - 1];
foreach (var Failed in History.AllFailed)
{
if (Failed > History.LastSucceeded)
{
History.Failed.Add(Failed);
History.FailedString = GUBPNode.MergeSpaceStrings(History.FailedString, String.Format("{0}", Failed));
}
}
foreach (var Started in History.AllStarted)
{
if (Started > History.LastSucceeded && !History.Failed.Contains(Started))
{
History.InProgress.Add(Started);
History.InProgressString = GUBPNode.MergeSpaceStrings(History.InProgressString, String.Format("{0}", Started));
}
}
GUBPNodesHistory.Add(Node.Key, History);
}
}
}
}
Log("******* {0} GUBP Nodes", GUBPNodes.Count);
foreach (var Node in GUBPNodes)
{
Log(" {0}", Node.Key);
}
bool bOnlyNode = false;
string PreconditionSuffix = "_PreconditionOnly";
{
string NodeSpec = ParseParamValue("Node");
if (String.IsNullOrEmpty(NodeSpec))
{
NodeSpec = ParseParamValue("OnlyNode");
bOnlyNode = true;
}
if (!String.IsNullOrEmpty(NodeSpec))
{
if (NodeSpec.Equals("Noop", StringComparison.InvariantCultureIgnoreCase) || NodeSpec.EndsWith(PreconditionSuffix, StringComparison.InvariantCultureIgnoreCase))
{
Log("Request for Noop node, done.");
PrintRunTime();
return;
}
List<string> Nodes = new List<string>(NodeSpec.Split('+'));
foreach (var NodeArg in Nodes)
{
var NodeName = NodeArg.Trim();
if (!String.IsNullOrEmpty(NodeName))
{
foreach (var Node in GUBPNodes)
{
if (Node.Value.GetFullName().Equals(NodeArg, StringComparison.InvariantCultureIgnoreCase))
{
NodesToDo.Add(Node.Key);
NodeName = null;
break;
}
}
if (NodeName != null)
{
throw new AutomationException("Could not find node named {0}", NodeName);
}
}
}
}
}
string GameSpec = ParseParamValue("Game");
if (!String.IsNullOrEmpty(GameSpec))
{
List<string> Games = new List<string>(GameSpec.Split('+'));
foreach (var GameArg in Games)
{
var GameName = GameArg.Trim();
if (!String.IsNullOrEmpty(GameName))
{
foreach (var GameProj in Branch.CodeProjects)
{
if (GameProj.GameName.Equals(GameName, StringComparison.InvariantCultureIgnoreCase))
{
NodesToDo.Add(GameAggregatePromotableNode.StaticGetFullName(GameProj));
foreach (var Node in GUBPNodes)
{
if (Node.Value.GameNameIfAnyForTempStorage() == GameProj.GameName)
{
NodesToDo.Add(Node.Key);
}
}
GameName = null;
}
}
if (GameName != null)
{
throw new AutomationException("Could not find game named {0}", GameName);
}
}
}
}
if (NodesToDo.Count == 0)
{
Log("No nodes specified, adding all nodes");
foreach (var Node in GUBPNodes)
{
NodesToDo.Add(Node.Key);
}
}
Log("Desired Nodes");
foreach (var NodeToDo in NodesToDo)
{
Log(" {0}", NodeToDo);
}
if (!bOnlyNode)
{
bool bDoneWithDependencies = false;
while (!bDoneWithDependencies)
{
bDoneWithDependencies = true;
var Fringe = new HashSet<string>();
foreach (var NodeToDo in NodesToDo)
{
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a dependency of {1}.", Dep, NodeToDo);
}
if (!NodesToDo.Contains(Dep))
{
Fringe.Add(Dep);
bDoneWithDependencies = false;
}
}
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a pseudodependency of {1}.", Dep, NodeToDo);
}
}
}
NodesToDo.UnionWith(Fringe);
}
}
string ExplicitTrigger = "";
bool CommanderSetup = ParseParam("CommanderJobSetupOnly");
string FakeFail = ParseParamValue("FakeFail");
if (CommanderSetup)
{
ExplicitTrigger = ParseParamValue("TriggerNode");
if (!String.IsNullOrEmpty(ExplicitTrigger))
{
bool bFoundIt = false;
foreach (var Node in GUBPNodes)
{
if (Node.Value.GetFullName().Equals(ExplicitTrigger, StringComparison.InvariantCultureIgnoreCase))
{
if (Node.Value.TriggerNode() && Node.Value.RunInEC())
{
Node.Value.SetAsExplicitTrigger();
bFoundIt = true;
break;
}
}
}
if (!bFoundIt)
{
throw new AutomationException("Could not find trigger node named {0}", ExplicitTrigger);
}
}
else
{
ExplicitTrigger = "";
if (bSkipTriggers)
{
foreach (var Node in GUBPNodes)
{
if (Node.Value.TriggerNode() && Node.Value.RunInEC())
{
Node.Value.SetAsExplicitTrigger();
}
}
}
}
}
var OrdereredToDo = new List<string>();
// here we do a topological sort of the nodes, subject to a lexographical and priority sort
while (NodesToDo.Count > 0)
{
bool bProgressMade = false;
float BestPriority = -1E20f;
string BestNode = "";
bool BestPseudoReady = false;
foreach (var NodeToDo in NodesToDo)
{
bool bReady = true;
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Dependency {0} node found.", Dep);
}
if (NodesToDo.Contains(Dep))
{
bReady = false;
break;
}
}
bool bPseudoReady = true;
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Pseudodependency {0} node found.", Dep);
}
if (NodesToDo.Contains(Dep))
{
bPseudoReady = false;
break;
}
}
var Priority = GUBPNodes[NodeToDo].Priority();
if (bReady && BestNode != "")
{
if (String.Compare(GetControllingTriggerDotName(BestNode), GetControllingTriggerDotName(NodeToDo)) < 0) //sorted by controlling trigger
{
bReady = false;
}
else if (String.Compare(GetControllingTriggerDotName(BestNode), GetControllingTriggerDotName(NodeToDo)) == 0) //sorted by controlling trigger
{
if (GUBPNodes[BestNode].IsSticky() && !GUBPNodes[NodeToDo].IsSticky()) //sticky nodes first
{
bReady = false;
}
else if (GUBPNodes[BestNode].IsSticky() == GUBPNodes[NodeToDo].IsSticky())
{
if (BestPseudoReady && !bPseudoReady)
{
bReady = false;
}
else if (BestPseudoReady == bPseudoReady)
{
bool IamLateTrigger = GUBPNodes[NodeToDo].TriggerNode() && NodeToDo != ExplicitTrigger && !NodeIsAlreadyComplete(NodeToDo, LocalOnly);
bool BestIsLateTrigger = GUBPNodes[BestNode].TriggerNode() && BestNode != ExplicitTrigger && !NodeIsAlreadyComplete(BestNode, LocalOnly);
if (BestIsLateTrigger && !IamLateTrigger)
{
bReady = false;
}
else if (BestIsLateTrigger == IamLateTrigger)
{
if (Priority < BestPriority)
{
bReady = false;
}
else if (Priority == BestPriority)
{
if (BestNode.CompareTo(NodeToDo) < 0)
{
bReady = false;
}
}
}
}
}
}
}
if (bReady)
{
BestPriority = Priority;
BestNode = NodeToDo;
BestPseudoReady = bPseudoReady;
bProgressMade = true;
}
}
if (bProgressMade)
{
OrdereredToDo.Add(BestNode);
NodesToDo.Remove(BestNode);
}
if (!bProgressMade && NodesToDo.Count > 0)
{
Log("Cycle in GUBP, could not resolve:");
foreach (var NodeToDo in NodesToDo)
{
Log(" {0}", NodeToDo);
}
throw new AutomationException("Cycle in GUBP");
}
}
// find all unfinished triggers, excepting the one we are triggering right now
var UnfinishedTriggers = new List<string>();
if (!bSkipTriggers)
{
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].TriggerNode() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly))
{
if (String.IsNullOrEmpty(ExplicitTrigger) || ExplicitTrigger != NodeToDo)
{
UnfinishedTriggers.Add(NodeToDo);
}
}
}
}
Log("*********** Desired And Dependent Nodes, in order.");
PrintNodes(this, OrdereredToDo, LocalOnly, UnfinishedTriggers);
if (CommanderSetup)
{
if (OrdereredToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
// we don't really need this yet, but it is left here for future use
var ECProps = new List<string>();
var ECJobProps = new List<string>();
if (ExplicitTrigger != "")
{
ECJobProps.Add("IsRoot=0");
}
else
{
ECJobProps.Add("IsRoot=1");
}
var FilteredOrdereredToDo = new List<string>();
// remove nodes that have unfinished triggers
foreach (var NodeToDo in OrdereredToDo)
{
string ControllingTrigger = GetControllingTrigger(NodeToDo);
bool bNoUnfinishedTriggers = !UnfinishedTriggers.Contains(ControllingTrigger);
if (bNoUnfinishedTriggers)
{
// if we are triggering, then remove nodes that are not controlled by the trigger or are dependencies of this trigger
if (!String.IsNullOrEmpty(ExplicitTrigger))
{
if (ExplicitTrigger != NodeToDo && !NodeDependsOn(NodeToDo, ExplicitTrigger) && !NodeDependsOn(ExplicitTrigger, NodeToDo))
{
continue; // this wasn't on the chain related to the trigger we are triggering, so it is not relevant
}
}
FilteredOrdereredToDo.Add(NodeToDo);
}
}
OrdereredToDo = FilteredOrdereredToDo;
Log("*********** EC Nodes, in order.");
PrintNodes(this, OrdereredToDo, LocalOnly, UnfinishedTriggers);
// here we are just making sure everything before the explicit trigger is completed.
if (!String.IsNullOrEmpty(ExplicitTrigger))
{
foreach (var NodeToDo in FilteredOrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly) && NodeToDo != ExplicitTrigger && !NodeDependsOn(ExplicitTrigger, NodeToDo)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo, ExplicitTrigger);
}
}
}
string LastSticky = "";
bool HitNonSticky = false;
bool bHaveECNodes = false;
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly)) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (GUBPNodes[NodeToDo].IsSticky())
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
string ParentPath = ParseParamValue("ParentPath");
string BaseArgs = String.Format("createJobStep --parentPath {0}", ParentPath);
if (LastSticky == "" && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
string Args = String.Format("{0} --subprocedure GUBP_UAT_Node --parallel 0 --jobStepName Noop --actualParameter NodeName=Noop --actualParameter Sticky=1 --releaseMode release", BaseArgs);
RunECTool(Args);
}
var FakeECArgs = new List<string>();
bool AddEmailProps = !ParseParam("NoAddEmailProps");
var BranchForEmail = "";
if (P4Enabled)
{
BranchForEmail = P4Env.BuildRootP4;
}
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly)) // if something is already finished, we don't put it into EC
{
string EMails = "";
if (AddEmailProps)
{
if (bFake)
{
EMails = "kellan.carr[epic] gil.gribb[epic]";
}
else
{
EMails = GetEMailListForNode(this, NodeToDo);
}
EMails = EMails.Trim().Replace("[epic]", "@epicgames.com");
ECProps.Add("FailEmails/" + NodeToDo + "=" + EMails);
}
if (GUBPNodes[NodeToDo].SendSuccessEmail())
{
ECProps.Add("SendSuccessEmail/" + NodeToDo + "=1");
}
else
{
ECProps.Add("SendSuccessEmail/" + NodeToDo + "=0");
}
ECProps.Add(string.Format("AgentRequirementString/{0}={1}", NodeToDo, GUBPNodes[NodeToDo].ECAgentString()));
ECProps.Add(string.Format("RequiredMemory/{0}={1}", NodeToDo, GUBPNodes[NodeToDo].AgentMemoryRequirement()));
ECProps.Add(string.Format("Timeouts/{0}={1}", NodeToDo, GUBPNodes[NodeToDo].TimeoutInMinutes()));
if (GUBPNodesHistory.ContainsKey(NodeToDo))
{
ECProps.Add(string.Format("LastGreen/{0}={1}", NodeToDo, GUBPNodesHistory[NodeToDo].LastSucceeded));
ECProps.Add(string.Format("RedsSince/{0}={1}", NodeToDo, GUBPNodesHistory[NodeToDo].FailedString));
ECProps.Add(string.Format("InProgress/{0}={1}", NodeToDo, GUBPNodesHistory[NodeToDo].InProgressString));
}
else
{
ECProps.Add(string.Format("LastGreen/{0}=0", NodeToDo));
ECProps.Add(string.Format("RedsSince/{0}=", NodeToDo));
ECProps.Add(string.Format("InProgress/{0}=", NodeToDo));
}
bool Sticky = GUBPNodes[NodeToDo].IsSticky();
bool DoParallel = !Sticky;
if (Sticky && GUBPNodes[NodeToDo].ECAgentString() != "")
{
throw new AutomationException("Node {1} is sticky but has agent requirements.", NodeToDo);
}
string Procedure = GUBPNodes[NodeToDo].ECProcedure();
string Args = String.Format("{0} --subprocedure {1} --parallel {2} --jobStepName {3} --actualParameter NodeName={4}",
BaseArgs, Procedure, DoParallel ? 1 : 0, NodeToDo, NodeToDo);
string ProcedureParams = GUBPNodes[NodeToDo].ECProcedureParams();
if (!String.IsNullOrEmpty(ProcedureParams))
{
Args = Args + " " + ProcedureParams;
}
string PreCondition = "";
string RunCondition = "";
var EcDeps = GetECDependencies(NodeToDo);
var UncompletedEcDeps = new List<string>();
foreach (var Dep in EcDeps)
{
if (GUBPNodes[Dep].RunInEC() && !NodeIsAlreadyComplete(Dep, LocalOnly) && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
UncompletedEcDeps.Add(Dep);
}
}
if (UncompletedEcDeps.Count > 0)
{
//$[/javascript if(getProperty("$[/myWorkflow/ParentWorkflow]/GUBP_Game_$[/myWorkflow/GameName]_MonoPlatform1_Exists") != "1" || getProperty("$[/myWorkflow/ParentWorkflow]/$[$[/myWorkflow/ParentWorkflow]/GUBP_Game_$[/myWorkflow/GameName]_MonoPlatform1_Name]_EditorPlatform_Success") != "1") true; else false;]
{
PreCondition = "\"$[/javascript if(";
int Index = 0;
foreach (var Dep in UncompletedEcDeps)
{
PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + Dep + "]/status') == 'completed'";
Index++;
if (Index != UncompletedEcDeps.Count)
{
PreCondition = PreCondition + " && ";
}
}
PreCondition = PreCondition + ") true;]\"";
}
{
RunCondition = "\"$[/javascript if(";
int Index = 0;
foreach (var Dep in UncompletedEcDeps)
{
RunCondition = RunCondition + "('$[" + ParentPath + "/jobSteps[" + Dep + "]/outcome]' == 'success' || ";
RunCondition = RunCondition + "'$[" + ParentPath + "/jobSteps[" + Dep + "]/outcome]' == 'warning')";
Index++;
if (Index != UncompletedEcDeps.Count)
{
RunCondition = RunCondition + " && ";
}
}
RunCondition = RunCondition + ") true; else false;]\"";
}
}
if (!String.IsNullOrEmpty(PreCondition))
{
Args = Args + " --precondition " + PreCondition;
}
if (!String.IsNullOrEmpty(RunCondition))
{
Args = Args + " --condition " + RunCondition;
}
if (Sticky && NodeToDo == LastSticky)
{
Args = Args + " --releaseMode release";
}
RunECTool(Args);
if (bFakeEC &&
!UnfinishedTriggers.Contains(NodeToDo) &&
GUBPNodes[NodeToDo].ECProcedure().StartsWith("GUBP_UAT_Node") // other things we really can't test
) // unfinished triggers are never run directly by EC, rather it does another job setup
{
string Arg = String.Format("gubp -Node={0} -FakeEC {1} {2} {3} {4}",
NodeToDo,
bFake ? "-Fake" : "" ,
ParseParam("AllPlatforms") ? "-AllPlatforms" : "",
ParseParam("UnfinishedTriggersFirst") ? "-UnfinishedTriggersFirst" : "",
ParseParam("UnfinishedTriggersParallel") ? "-UnfinishedTriggersParallel" : ""
);
string Node = ParseParamValue("-Node");
if (!String.IsNullOrEmpty(Node))
{
Arg = Arg + " -Node=" + Node;
}
if (!String.IsNullOrEmpty(FakeFail))
{
Arg = Arg + " -FakeFail=" + FakeFail;
}
FakeECArgs.Add(Arg);
}
}
}
{
ECProps.Add("GUBP_LoadedProps=1");
string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
}
{
ECProps.Add("GUBP_LoadedJobProps=1");
string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
}
if (bFakeEC)
{
foreach (var Args in FakeECArgs)
{
RunUAT(CmdEnv, Args);
}
}
Log("Commander setup only, done.");
PrintRunTime();
return;
}
if (ParseParam("SaveGraph"))
{
SaveGraphVisualization(OrdereredToDo);
}
if (bListOnly)
{
Log("List only, done.");
return;
}
var BuildProductToNodeMap = new Dictionary<string, string>();
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].BuildProducts != null || GUBPNodes[NodeToDo].AllDependencyBuildProducts != null)
{
throw new AutomationException("topological sort error");
}
GUBPNodes[NodeToDo].AllDependencyBuildProducts = new List<string>();
if (GUBPNodes[NodeToDo].FullNamesOfDependencies == null)
{
throw new AutomationException("Node {0} was not processed yet?", NodeToDo);
}
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
if (GUBPNodes[Dep].BuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet2? Processing {1}", Dep, NodeToDo);
}
foreach (var Prod in GUBPNodes[Dep].BuildProducts)
{
GUBPNodes[NodeToDo].AddDependentBuildProduct(Prod);
}
if (GUBPNodes[Dep].AllDependencyBuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet3? Processing {1}", Dep, NodeToDo);
}
foreach (var Prod in GUBPNodes[Dep].AllDependencyBuildProducts)
{
GUBPNodes[NodeToDo].AddDependentBuildProduct(Prod);
}
}
string NodeStoreName = StoreName + "-" + GUBPNodes[NodeToDo].GetFullName();
string GameNameIfAny = GUBPNodes[NodeToDo].GameNameIfAnyForTempStorage();
string StorageRootIfAny = GUBPNodes[NodeToDo].RootIfAnyForTempStorage();
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!GUBPNodes[NodeToDo].TriggerNode() || GUBPNodes[NodeToDo].IsSticky()) // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
&& (GUBPNodes[NodeToDo].RunInEC() || !GUBPNodes[NodeToDo].IsAggregate()); //aggregates not in EC can be "run" multiple times, so we can't track those
Log("***** Running GUBP Node {0} -> {1} : {2}", GUBPNodes[NodeToDo].GetFullName(), GameNameIfAny, NodeStoreName);
if (NodeIsAlreadyComplete(NodeToDo, LocalOnly))
{
Log("***** Retrieving GUBP Node {0} from {1}", GUBPNodes[NodeToDo].GetFullName(), NodeStoreName);
GUBPNodes[NodeToDo].BuildProducts = RetrieveFromTempStorage(CmdEnv, NodeStoreName, GameNameIfAny, StorageRootIfAny);
}
else
{
if (SaveSuccessRecords)
{
SaveStatus(NodeToDo, StartedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo);
}
if (bFake)
{
Log("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo, NodeStoreName);
GUBPNodes[NodeToDo].DoFakeBuild(this);
}
else
{
Log("***** Building GUBP Node {0} for {1}", NodeToDo, NodeStoreName);
GUBPNodes[NodeToDo].DoBuild(this);
}
if (!GUBPNodes[NodeToDo].IsAggregate())
{
StoreToTempStorage(CmdEnv, NodeStoreName, GUBPNodes[NodeToDo].BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
}
}
catch (Exception Ex)
{
if (SaveSuccessRecords)
{
SaveStatus(NodeToDo, FailedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
Log("{0}", ExceptionToString(Ex));
if (GUBPNodesHistory.ContainsKey(NodeToDo))
{
var History = GUBPNodesHistory[NodeToDo];
Log("Changes since last green *********************************");
Log("");
Log("");
Log("");
PrintDetailedChanges(History);
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
SaveStatus(NodeToDo, SucceededTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
}
foreach (var Product in GUBPNodes[NodeToDo].BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product], NodeToDo, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}
void PrintDetailedChanges(NodeHistory History, bool bShowAllChanges = false)
{
string Me = String.Format("{0} <<<< local sync", P4Env.Changelist);
int LastOutputForChanges = 0;
int LastGreen = History.LastSucceeded;
if (bShowAllChanges)
{
LastGreen = 0;
}
foreach (var cl in History.AllStarted)
{
if (cl < LastGreen)
{
continue;
}
if (P4Env.Changelist < cl && Me != "")
{
LastOutputForChanges = PrintChanges(LastOutputForChanges, P4Env.Changelist, LastGreen);
Log(" {0}", Me);
Me = "";
}
string Status = "In Process";
if (History.AllSucceeded.Contains(cl))
{
Status = "ok";
}
if (History.AllFailed.Contains(cl))
{
Status = "FAIL";
}
LastOutputForChanges = PrintChanges(LastOutputForChanges, cl, LastGreen);
Log(" {0} {1}", cl, Status);
}
if (Me != "")
{
LastOutputForChanges = PrintChanges(LastOutputForChanges, P4Env.Changelist, LastGreen);
Log(" {0}", Me);
}
}
internal IEnumerable <NodePlacement> GetNodePlacementsMatchingCurrentFilters(NodeHistory node)
{
return(node.Placements
.Where(p => this.Model.IsInstanceAnnotatedAs(
p.Instance, this.osTypes, this.licenseTypes)));
}
static NodeHistory FindNodeHistory(BuildNode NodeToDo, string CLString, string StoreName)
{
NodeHistory History = null;
if (!(NodeToDo is TriggerNode) && CLString != "")
{
string GameNameIfAny = NodeToDo.Node.GameNameIfAnyForTempStorage();
string NodeStoreWildCard = StoreName.Replace(CLString, "*") + "-" + NodeToDo.Name;
History = new NodeHistory();
History.AllStarted = ConvertCLToIntList(TempStorage.FindTempStorageManifests(CmdEnv, NodeStoreWildCard + StartedTempStorageSuffix, false, true, GameNameIfAny));
History.AllSucceeded = ConvertCLToIntList(TempStorage.FindTempStorageManifests(CmdEnv, NodeStoreWildCard + SucceededTempStorageSuffix, false, true, GameNameIfAny));
History.AllFailed = ConvertCLToIntList(TempStorage.FindTempStorageManifests(CmdEnv, NodeStoreWildCard + FailedTempStorageSuffix, false, true, GameNameIfAny));
int CL;
int.TryParse(CLString, out CL);
if (History.AllFailed.Count > 0)
{
History.LastFailed = History.AllFailed.LastOrDefault(x => x < CL);
}
if (History.AllSucceeded.Count > 0)
{
History.LastSucceeded = History.AllSucceeded.LastOrDefault(x => x < CL);
foreach (int Failed in History.AllFailed)
{
if (Failed > History.LastSucceeded)
{
History.Failed.Add(Failed);
History.FailedString = GUBPNode.MergeSpaceStrings(History.FailedString, String.Format("{0}", Failed));
}
}
foreach (int Started in History.AllStarted)
{
if (Started > History.LastSucceeded && !History.Failed.Contains(Started))
{
History.InProgress.Add(Started);
History.InProgressString = GUBPNode.MergeSpaceStrings(History.InProgressString, String.Format("{0}", Started));
}
}
}
}
return History;
}
int FindLastNonDuplicateFail(BuildNode NodeToDo, NodeHistory History, string CLString, string StoreName)
{
int Result = P4Env.Changelist;
string GameNameIfAny = NodeToDo.Node.GameNameIfAnyForTempStorage();
string NodeStore = StoreName + "-" + NodeToDo.Name + FailedTempStorageSuffix;
List<int> BackwardsFails = new List<int>(History.AllFailed);
BackwardsFails.Add(P4Env.Changelist);
BackwardsFails.Sort();
BackwardsFails.Reverse();
HashSet<string> CurrentErrors = null;
foreach (int CL in BackwardsFails)
{
if (CL > P4Env.Changelist)
{
continue;
}
if (CL <= History.LastSucceeded)
{
break;
}
string ThisNodeStore = NodeStore.Replace(CLString, String.Format("{0}", CL));
TempStorage.DeleteLocalTempStorage(CmdEnv, ThisNodeStore, true); // these all clash locally, which is fine we just retrieve them from shared
List<string> Files = null;
try
{
bool WasLocal;
Files = TempStorage.RetrieveFromTempStorage(CmdEnv, ThisNodeStore, out WasLocal, GameNameIfAny); // this will fail on our CL if we didn't fail or we are just setting up the branch
}
catch (Exception)
{
}
if (Files == null)
{
continue;
}
if (Files.Count != 1)
{
throw new AutomationException("Unexpected number of files for fail record {0}", Files.Count);
}
string ErrorFile = Files[0];
HashSet<string> ThisErrors = ECJobPropsUtils.ErrorsFromProps(ErrorFile);
if (CurrentErrors == null)
{
CurrentErrors = ThisErrors;
}
else
{
if (CurrentErrors.Count == 0 || !HashSetEqual(CurrentErrors, ThisErrors))
{
break;
}
Result = CL;
}
}
return Result;
}
static NodeHistory FindNodeHistory(BuildNode NodeToDo, JobInfo JobInfo)
{
NodeHistory History = null;
// Don't get node history on nodes that don't have a valid CL.
if (!(NodeToDo is TriggerNode) && JobInfo.Changelist > 0)
{
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
History = new NodeHistory
{
AllStarted = TempStorage.FindMatchingSharedTempStorageNodeCLs(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + StartedTempStorageSuffix), GameNameIfAny),
AllSucceeded = TempStorage.FindMatchingSharedTempStorageNodeCLs(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + SucceededTempStorageSuffix), GameNameIfAny),
AllFailed = TempStorage.FindMatchingSharedTempStorageNodeCLs(new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix), GameNameIfAny)
};
if (History.AllFailed.Count > 0)
{
History.LastFailed = History.AllFailed.LastOrDefault(x => x < JobInfo.Changelist);
}
if (History.AllSucceeded.Count > 0)
{
History.LastSucceeded = History.AllSucceeded.LastOrDefault(x => x < JobInfo.Changelist);
foreach (int Failed in History.AllFailed)
{
if (Failed > History.LastSucceeded)
{
History.Failed.Add(Failed);
History.FailedString = GUBPNode.MergeSpaceStrings(History.FailedString, String.Format("{0}", Failed));
}
}
foreach (int Started in History.AllStarted)
{
if (Started > History.LastSucceeded && !History.Failed.Contains(Started))
{
History.InProgress.Add(Started);
History.InProgressString = GUBPNode.MergeSpaceStrings(History.InProgressString, String.Format("{0}", Started));
}
}
}
}
return History;
}
int FindLastNonDuplicateFail(BuildNode NodeToDo, NodeHistory History, JobInfo JobInfo)
{
int Result = P4Env.Changelist;
string GameNameIfAny = NodeToDo.GameNameIfAnyForTempStorage;
var TempStorageNodeInfo = new TempStorageNodeInfo(JobInfo, NodeToDo.Name + FailedTempStorageSuffix);
List<int> BackwardsFails = new List<int>(History.AllFailed);
BackwardsFails.Add(P4Env.Changelist);
BackwardsFails.Sort();
BackwardsFails.Reverse();
HashSet<string> CurrentErrors = null;
foreach (int CL in BackwardsFails)
{
if (CL > P4Env.Changelist)
{
continue;
}
if (CL <= History.LastSucceeded)
{
break;
}
// Find any local temp storage manifest for this changelist and delete it.
var ThisTempStorageNodeInfo = new TempStorageNodeInfo(JobInfo.CreateWithNewChangelist(CL), TempStorageNodeInfo.NodeStorageName);
TempStorage.DeleteLocalTempStorageManifest(ThisTempStorageNodeInfo, true); // these all clash locally, which is fine we just retrieve them from shared
List<string> Files = null;
try
{
bool WasLocal;
Files = TempStorage.RetrieveFromTempStorage(ThisTempStorageNodeInfo, out WasLocal, GameNameIfAny, CmdEnv.LocalRoot); // this will fail on our CL if we didn't fail or we are just setting up the branch
}
catch (Exception)
{
}
if (Files == null)
{
continue;
}
if (Files.Count != 1)
{
throw new AutomationException("Unexpected number of files for fail record {0}", Files.Count);
}
string ErrorFile = Files[0];
HashSet<string> ThisErrors = ECJobPropsUtils.ErrorsFromProps(ErrorFile);
if (CurrentErrors == null)
{
CurrentErrors = ThisErrors;
}
else
{
if (CurrentErrors.Count == 0 || !HashSetEqual(CurrentErrors, ThisErrors))
{
break;
}
Result = CL;
}
}
return Result;
}
protected virtual void WriteHistory(INode node, String action, Boolean success, String remark)
{
NodeHistory.Create(node, action, success, remark, Environment.MachineName, UserHost);
}
protected virtual NodeHistory WriteHistory(String action, Boolean success, INode node, String remark = null)
{
var ip = UserHost;
return(NodeHistory.Create(node, action, success, remark, Environment.MachineName, ip));
}
void PrintDetailedChanges(NodeHistory History, bool bShowAllChanges = false)
{
var StartTime = DateTime.UtcNow;
string Me = String.Format("{0} <<<< local sync", P4Env.Changelist);
int LastOutputForChanges = 0;
int LastGreen = History.LastSucceeded;
if (bShowAllChanges)
{
if (History.AllStarted.Count > 0)
{
LastGreen = History.AllStarted[0];
}
}
foreach (var cl in History.AllStarted)
{
if (cl < LastGreen)
{
continue;
}
if (P4Env.Changelist < cl && Me != "")
{
LastOutputForChanges = PrintChanges(LastOutputForChanges, P4Env.Changelist, LastGreen);
Log(" {0}", Me);
Me = "";
}
string Status = "In Process";
if (History.AllSucceeded.Contains(cl))
{
Status = "ok";
}
if (History.AllFailed.Contains(cl))
{
Status = "FAIL";
}
LastOutputForChanges = PrintChanges(LastOutputForChanges, cl, LastGreen);
Log(" {0} {1}", cl, Status);
}
if (Me != "")
{
LastOutputForChanges = PrintChanges(LastOutputForChanges, P4Env.Changelist, LastGreen);
Log(" {0}", Me);
}
var BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
Log("Took {0}s to get P4 history", BuildDuration / 1000);
}
void GetFailureEmails(ElectricCommander EC, BuildNode NodeToDo, NodeHistory History, string CLString, string StoreName, bool OnlyLateUpdates = false)
{
string FailCauserEMails = "";
string EMailNote = "";
bool SendSuccessForGreenAfterRed = false;
int NumPeople = 0;
if (History != null)
{
if (History.LastSucceeded > 0 && History.LastSucceeded < P4Env.Changelist)
{
int LastNonDuplicateFail = P4Env.Changelist;
try
{
if (OnlyLateUpdates)
{
LastNonDuplicateFail = FindLastNonDuplicateFail(NodeToDo, History, CLString, StoreName);
if (LastNonDuplicateFail < P4Env.Changelist)
{
Log("*** Red-after-red spam reduction, changed CL {0} to CL {1} because the errors didn't change.", P4Env.Changelist, LastNonDuplicateFail);
}
}
}
catch (Exception Ex)
{
LastNonDuplicateFail = P4Env.Changelist;
Log(System.Diagnostics.TraceEventType.Warning, "Failed to FindLastNonDuplicateFail.");
Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex));
}
if(LastNonDuplicateFail > History.LastSucceeded)
{
List<P4Connection.ChangeRecord> ChangeRecords = GetSourceChangeRecords(History.LastSucceeded + 1, LastNonDuplicateFail);
foreach (P4Connection.ChangeRecord Record in ChangeRecords)
{
FailCauserEMails = GUBPNode.MergeSpaceStrings(FailCauserEMails, Record.UserEmail);
}
}
if (!String.IsNullOrEmpty(FailCauserEMails))
{
NumPeople++;
foreach (char AChar in FailCauserEMails.ToCharArray())
{
if (AChar == ' ')
{
NumPeople++;
}
}
if (NumPeople > 50)
{
EMailNote = String.Format("This step has been broken for more than 50 changes. It last succeeded at CL {0}. ", History.LastSucceeded);
}
}
}
else if (History.LastSucceeded <= 0)
{
EMailNote = String.Format("This step has been broken for more than a few days, so there is no record of it ever succeeding. ");
}
if (EMailNote != "" && !String.IsNullOrEmpty(History.FailedString))
{
EMailNote += String.Format("It has failed at CLs {0}. ", History.FailedString);
}
if (EMailNote != "" && !String.IsNullOrEmpty(History.InProgressString))
{
EMailNote += String.Format("These CLs are being built right now {0}. ", History.InProgressString);
}
if (History.LastSucceeded > 0 && History.LastSucceeded < P4Env.Changelist && History.LastFailed > History.LastSucceeded && History.LastFailed < P4Env.Changelist)
{
SendSuccessForGreenAfterRed = ParseParam("CIS");
}
}
if (History == null)
{
EC.UpdateEmailProperties(NodeToDo, 0, "", FailCauserEMails, EMailNote, SendSuccessForGreenAfterRed);
}
else
{
EC.UpdateEmailProperties(NodeToDo, History.LastSucceeded, History.FailedString, FailCauserEMails, EMailNote, SendSuccessForGreenAfterRed);
}
}
void UpdateNodeHistory(string Node, string CLString)
{
if (GUBPNodes[Node].RunInEC() && !GUBPNodes[Node].TriggerNode() && CLString != "")
{
string GameNameIfAny = GUBPNodes[Node].GameNameIfAnyForTempStorage();
string NodeStoreWildCard = StoreName.Replace(CLString, "*") + "-" + GUBPNodes[Node].GetFullName();
var History = new NodeHistory();
History.AllStarted = ConvertCLToIntList(FindTempStorageManifests(CmdEnv, NodeStoreWildCard + StartedTempStorageSuffix, false, true, GameNameIfAny));
History.AllSucceeded = ConvertCLToIntList(FindTempStorageManifests(CmdEnv, NodeStoreWildCard + SucceededTempStorageSuffix, false, true, GameNameIfAny));
History.AllFailed = ConvertCLToIntList(FindTempStorageManifests(CmdEnv, NodeStoreWildCard + FailedTempStorageSuffix, false, true, GameNameIfAny));
if (History.AllFailed.Count > 0)
{
History.LastFailed = History.AllFailed[History.AllFailed.Count - 1];
}
if (History.AllSucceeded.Count > 0)
{
History.LastSucceeded = History.AllSucceeded[History.AllSucceeded.Count - 1];
foreach (var Failed in History.AllFailed)
{
if (Failed > History.LastSucceeded)
{
History.Failed.Add(Failed);
History.FailedString = GUBPNode.MergeSpaceStrings(History.FailedString, String.Format("{0}", Failed));
}
}
foreach (var Started in History.AllStarted)
{
if (Started > History.LastSucceeded && !History.Failed.Contains(Started))
{
History.InProgress.Add(Started);
History.InProgressString = GUBPNode.MergeSpaceStrings(History.InProgressString, String.Format("{0}", Started));
}
}
}
if (GUBPNodesHistory.ContainsKey(Node))
{
GUBPNodesHistory.Remove(Node);
}
GUBPNodesHistory.Add(Node, History);
}
}
/// <summary>
/// Print a list of source changes, along with the success state for other builds of this node.
/// </summary>
/// <param name="History">History for this node</param>
static void PrintDetailedChanges(NodeHistory History, int CurrentCL)
{
DateTime StartTime = DateTime.UtcNow;
// Find all the changelists that we're interested in
SortedSet<int> BuildChanges = new SortedSet<int>();
BuildChanges.UnionWith(History.AllStarted.Where(x => x >= History.LastSucceeded));
BuildChanges.Add(CurrentCL);
// Find all the changelists that we're interested in
List<P4Connection.ChangeRecord> ChangeRecords = GetSourceChangeRecords(BuildChanges.First(), BuildChanges.Last());
// Print all the changes in the set
int LastCL = BuildChanges.First();
foreach(int BuildCL in BuildChanges)
{
// Show all the changes in this range
foreach(P4Connection.ChangeRecord Record in ChangeRecords.Where(x => x.CL > LastCL && x.CL <= BuildCL))
{
string[] Lines = Record.Summary.Split(new char[]{ '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
Log(" {0} {1} {2}", Record.CL, Record.UserEmail, (Lines.Length > 0)? Lines[0] : "");
}
// Show the status of this build
string BuildStatus;
if(BuildCL == CurrentCL)
{
BuildStatus = "this sync";
}
else if (History.AllSucceeded.Contains(BuildCL))
{
BuildStatus = "built";
}
else if (History.AllFailed.Contains(BuildCL))
{
BuildStatus = "FAILED";
}
else
{
BuildStatus = "running";
}
Log(" {0} {1} {2}", (BuildCL == CurrentCL)? ">>>>" : " ", BuildCL, BuildStatus.ToString());
// Update the last CL now that we've output this one
LastCL = BuildCL;
}
Log("Took {0:0.0}s to get P4 history", (DateTime.UtcNow - StartTime).TotalSeconds);
}
private void WriteHistory(Node node, String action, Boolean success, String remark) => NodeHistory.Create(node, action, success, remark, Environment.MachineName, UserHost);
