/// <summary>
/// Writes our current result to a CSV file. It's expected that this function is called multiple times so results are
/// updated as we go
/// </summary>
void WriteCSVResults(string InFileName, List <BenchmarkTaskBase> InTasks, Dictionary <BenchmarkTaskBase, List <TimeSpan> > InResults)
{
Log.TraceInformation("Writing results to {0}", InFileName);
try
{
List <string> Lines = new List <string>();
// first line is machine name,CPU count,Iteration 1, Iteration 2 etc
string FirstLine = string.Format("{0},{1}", Environment.MachineName, Environment.ProcessorCount);
if (InTasks.Count() > 0)
{
int Iterations = InResults[InTasks.First()].Count();
if (Iterations > 0)
{
for (int i = 0; i < Iterations; i++)
{
FirstLine += ",";
FirstLine += string.Format("Iteration {0}", i + 1);
}
}
}
Lines.Add(FirstLine);
foreach (var Task in InTasks)
{
// start with Name, StartTime
string Line = string.Format("{0},{1}", Task.GetFullTaskName(), Task.StartTime.ToString("yyyy-dd-MM HH:mm:ss"));
// now append all iteration times
foreach (TimeSpan TaskTime in InResults[Task])
{
Line += ",";
if (TaskTime == TimeSpan.Zero)
{
Line += "FAILED";
}
else
{
Line += TaskTime.ToString(@"hh\:mm\:ss");
}
}
Lines.Add(Line);
}
File.WriteAllLines(InFileName, Lines.ToArray());
}
catch (Exception Ex)
{
Log.TraceError("Failed to write CSV to {0}. {1}", InFileName, Ex);
}
}
/// <summary>
/// Report how long the task took
/// </summary>
public void Report()
{
if (!Failed)
{
Log.TraceInformation("Task {0}:\t\t\t\t{1}", GetFullTaskName(), TaskTime.ToString(@"hh\:mm\:ss"));
}
else
{
Log.TraceInformation("Task {0}::\t\t\t\tFailed. {1}", GetFullTaskName(), FailureString);
}
}
public override ExitCode Execute()
{
BenchmarkOptions Options = new BenchmarkOptions();
Options.ParseParams(this.Params);
List <BenchmarkTaskBase> Tasks = new List <BenchmarkTaskBase>();
Dictionary <BenchmarkTaskBase, List <TimeSpan> > Results = new Dictionary <BenchmarkTaskBase, List <TimeSpan> >();
for (int ProjectIndex = 0; ProjectIndex < Options.ProjectsToTest.Count(); ProjectIndex++)
{
string Project = Options.ProjectsToTest.ElementAt(ProjectIndex);
FileReference ProjectFile = ProjectUtils.FindProjectFileFromName(Project);
if (ProjectFile == null && !Project.Equals("UE4", StringComparison.OrdinalIgnoreCase))
{
throw new AutomationException("Could not find project file for {0}", Project);
}
if (Options.DoBuildEditorTests)
{
Tasks.AddRange(AddBuildTests(ProjectFile, BuildHostPlatform.Current.Platform, "Editor", Options));
}
// do startup tests
if (Options.DoPIETests)
{
Tasks.AddRange(AddPIETests(ProjectFile, Options));
}
foreach (var ClientPlatform in Options.PlatformsToTest)
{
// build a client if the project supports it
string TargetName = ProjectSupportsClientBuild(ProjectFile) ? "Client" : "Game";
if (Options.DoBuildClientTests)
{
// do build tests
Tasks.AddRange(AddBuildTests(ProjectFile, ClientPlatform, TargetName, Options));
}
// do cook tests
if (Options.DoCookTests)
{
Tasks.AddRange(AddCookTests(ProjectFile, ClientPlatform, Options));
}
}
}
Log.TraceInformation("Will execute tests:");
foreach (var Task in Tasks)
{
Log.TraceInformation("{0}", Task.GetFullTaskName());
}
if (!Options.Preview)
{
// create results lists
foreach (var Task in Tasks)
{
Results.Add(Task, new List <TimeSpan>());
}
DateTime StartTime = DateTime.Now;
for (int i = 0; i < Options.Iterations; i++)
{
foreach (var Task in Tasks)
{
Log.TraceInformation("Starting task {0} (Pass {1})", Task.GetFullTaskName(), i + 1);
Task.Run();
Log.TraceInformation("Task {0} took {1}", Task.GetFullTaskName(), Task.TaskTime.ToString(@"hh\:mm\:ss"));
Results[Task].Add(Task.TaskTime);
// write results so far
WriteCSVResults(Options.FileName, Tasks, Results);
Log.TraceInformation("Waiting {0} secs until next task", Options.TimeBetweenTasks);
Thread.Sleep(Options.TimeBetweenTasks * 1000);
}
}
Log.TraceInformation("**********************************************************************");
Log.TraceInformation("Test Results:");
foreach (var Task in Tasks)
{
string TimeString = "";
IEnumerable <TimeSpan> TaskTimes = Results[Task];
foreach (var TaskTime in TaskTimes)
{
if (TimeString.Length > 0)
{
TimeString += ", ";
}
if (TaskTime == TimeSpan.Zero)
{
TimeString += "Failed";
}
else
{
TimeString += TaskTime.ToString(@"hh\:mm\:ss");
}
}
var AvgTimeString = "";
if (TaskTimes.Count() > 1)
{
var AvgTime = new TimeSpan(TaskTimes.Sum(T => T.Ticks) / TaskTimes.Count());
AvgTimeString = string.Format(" (Avg: {0})", AvgTime.ToString(@"hh\:mm\:ss"));
}
Log.TraceInformation("Task {0}:\t{1}{2}", Task.GetFullTaskName(), TimeString, AvgTimeString);
}
Log.TraceInformation("**********************************************************************");
TimeSpan Elapsed = DateTime.Now - StartTime;
Log.TraceInformation("Total benchmark time: {0}", Elapsed.ToString(@"hh\:mm\:ss"));
WriteCSVResults(Options.FileName, Tasks, Results);
}
return(ExitCode.Success);
}