/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
DirectoryReference FromDir = ResolveDirectory(Parameters.FromDir);
// Find all the input files
IEnumerable <FileReference> Files;
if (Parameters.Files == null)
{
Files = FromDir.EnumerateFileReferences("*", System.IO.SearchOption.AllDirectories);
}
else
{
Files = ResolveFilespec(FromDir, Parameters.Files, TagNameToFileSet);
}
// Create the zip file
FileReference ArchiveFile = ResolveFile(Parameters.ZipFile);
CommandUtils.ZipFiles(ArchiveFile, FromDir, Files);
// Apply the optional tag to the produced archive
if (!String.IsNullOrEmpty(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, Parameters.Tag).Add(ArchiveFile);
}
// Add the archive to the set of build products
BuildProducts.Add(ArchiveFile);
return(true);
}
/// <summary>
/// Execute the task and run the cooker.
/// </summary>
/// <param name="BuildProducts">List of build products for the current node. Cooking build products will be appended to this list.</param>
/// <returns>True if the node succeeded</returns>
public override bool Execute(List <string> BuildProducts)
{
// Figure out the project that this target belongs to
FileReference ProjectFile;
if (UProjectInfo.TryGetProjectForTarget(Target, out ProjectFile))
{
ProjectFile = null;
}
// Execute the cooker
using (TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", ProjectFile.GetFileNameWithoutExtension(), CookPlatform))
{
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, CookPlatform, Arguments);
}
// Find all the cooked files
DirectoryReference CookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", CookPlatform);
List <FileReference> CookedFiles = CookedDirectory.EnumerateFileReferences().ToList();
if (CookedFiles.Count == 0)
{
throw new AutomationException("Cooking did not produce any files in {0}", CookedDirectory.FullName);
}
BuildProducts.AddRange(CookedFiles.Select(x => x.FullName));
return(true);
}
static void FindFilesInternal(DirectoryReference Directory, string[] ExcludedDirectorySuffixes, List<FileReference> FoundFiles)
{
foreach (FileReference File in Directory.EnumerateFileReferences())
{
if (ShouldInclude(File, DefaultExcludedFileSuffixes))
{
FoundFiles.Add(File);
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Figure out the project that this target belongs to
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = new FileReference(Parameters.Project);
if (!ProjectFile.Exists())
{
CommandUtils.LogError("Missing project file - {0}", ProjectFile.FullName);
return(false);
}
}
// Execute the cooker
using (TelemetryStopwatch CookStopwatch = new TelemetryStopwatch("Cook.{0}.{1}", (ProjectFile == null)? "UE4" : ProjectFile.GetFileNameWithoutExtension(), Parameters.Platform))
{
string[] Maps = (Parameters.Maps == null)? null : Parameters.Maps.Split(new char[] { '+' });
string Arguments = (Parameters.Versioned ? "" : "-Unversioned ") + "-LogCmds=\"LogSavePackage Warning\" " + Parameters.Arguments;
CommandUtils.CookCommandlet(ProjectFile, "UE4Editor-Cmd.exe", Maps, null, null, null, Parameters.Platform, Arguments);
}
// Find all the cooked files
List <FileReference> CookedFiles = new List <FileReference>();
foreach (string Platform in Parameters.Platform.Split('+'))
{
DirectoryReference PlatformCookedDirectory = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "Cooked", Platform);
if (!PlatformCookedDirectory.Exists())
{
CommandUtils.LogError("Cook output directory not found ({0})", PlatformCookedDirectory.FullName);
return(false);
}
List <FileReference> PlatformCookedFiles = PlatformCookedDirectory.EnumerateFileReferences("*", System.IO.SearchOption.AllDirectories).ToList();
if (PlatformCookedFiles.Count == 0)
{
CommandUtils.LogError("Cooking did not produce any files in {0}", PlatformCookedDirectory.FullName);
return(false);
}
CookedFiles.AddRange(PlatformCookedFiles);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(CookedFiles);
}
// Add them to the set of build products
BuildProducts.UnionWith(CookedFiles);
return(true);
}
/// <summary>
/// Gather compile time telemetry for the given files
/// </summary>
/// <param name="FileToCompileEnvironment">Mapping of source file to the environment used to compile it</param>
/// <param name="WorkingDir">The working directory for output files</param>
/// <param name="NumSamples">Number of samples to take</param>
/// <param name="MaxParallel">Maximum number of tasks to run in parallel.</param>
/// <param name="Log">Log writer</param>
public static void Generate(DirectoryReference InputDir, DirectoryReference WorkingDir, Dictionary <SourceFile, CompileEnvironment> FileToCompileEnvironment, int NumSamples, int Shard, int NumShards, int MaxParallel, LineBasedTextWriter Log)
{
Stopwatch Timer = Stopwatch.StartNew();
// Create an intermediate directory
DirectoryReference IntermediateDir = DirectoryReference.Combine(WorkingDir, "Timing");
IntermediateDir.CreateDirectory();
// Map of unique fragment to timing data
Dictionary <SourceFragment, FragmentTimingData> FragmentToTimingData = new Dictionary <SourceFragment, FragmentTimingData>();
// Map of unique fragment key to timing data
Dictionary <string, FragmentTimingData> DigestToTimingData = new Dictionary <string, FragmentTimingData>();
// List of all the sequences to time
HashSet <string> UniqueNames = new HashSet <string>();
foreach (KeyValuePair <SourceFile, CompileEnvironment> Pair in FileToCompileEnvironment)
{
// Find all the fragments in this file
List <SourceFragment> Fragments = new List <SourceFragment>();
List <Tuple <int, SourceFile> > IncludeHistory = new List <Tuple <int, SourceFile> >();
Pair.Key.FindIncludedFragments(Fragments, IncludeHistory, new HashSet <SourceFile>());
// Create a sequence for each unique fragment
FragmentTimingData PrevTimingData = null;
for (int Idx = 0; Idx < Fragments.Count; Idx++)
{
FragmentTimingData TimingData = null;
if (!FragmentToTimingData.ContainsKey(Fragments[Idx]) || (Idx + 1 < Fragments.Count && !FragmentToTimingData.ContainsKey(Fragments[Idx + 1])))
{
// Create a sequence for this fragment
SourceFragment LastFragment = Fragments[Idx];
// Create a unique key for this sequence by concatenating all the fragment names
string Digest = Utility.ComputeDigest(String.Join("\n", Fragments.Take(Idx + 1).Select(x => x.Location.FullName)));
// Try to get an existing sequence for this key, otherwise create a new one;
if (!DigestToTimingData.TryGetValue(Digest, out TimingData))
{
// Find a unique name for this sequence
string UniqueName = LastFragment.Location.GetFileName();
for (int NameIdx = 2; !UniqueNames.Add(UniqueName); NameIdx++)
{
UniqueName = String.Format("{0}_{1}{2}", LastFragment.Location.GetFileNameWithoutExtension(), NameIdx, LastFragment.Location.GetExtension());
}
// Add the object for this sequence
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, UniqueName);
TimingData = new FragmentTimingData(UniqueName, Digest, PrevTimingData, Fragments.Take(Idx + 1).ToArray(), IncludeHistory, IntermediateFile, Pair.Value);
DigestToTimingData.Add(Digest, TimingData);
}
// Add it to the unique mapping of fragments
if (!FragmentToTimingData.ContainsKey(LastFragment))
{
FragmentToTimingData[LastFragment] = TimingData;
}
}
PrevTimingData = TimingData;
}
}
// Read any existing shard timing data in the output folder
foreach (FileReference IntermediateFile in IntermediateDir.EnumerateFileReferences("*.csv"))
{
string[] Lines = File.ReadAllLines(IntermediateFile.FullName);
foreach (string Line in Lines.Skip(1))
{
string[] Tokens = Line.Split(',');
if (Tokens.Length == 5)
{
FragmentTimingData TimingData;
if (DigestToTimingData.TryGetValue(Tokens[1], out TimingData) && TimingData.Samples.Count < NumSamples)
{
FragmentTimingSample Sample = new FragmentTimingSample();
Sample.TotalTime = Double.Parse(Tokens[2]);
Sample.FrontendTime = Double.Parse(Tokens[3]);
Sample.BackendTime = Double.Parse(Tokens[4]);
TimingData.Samples.Add(Sample);
}
}
}
}
// Find all the remaining fragments, and repeat each one by the number of times it has to be executed
List <FragmentTimingData> FilteredFragments = DigestToTimingData.Values.ToList();
FilteredFragments.RemoveAll(x => (int)(Math.Abs((long)x.Digest.GetHashCode()) % NumShards) != (Shard - 1));
// Get the initial number of compile times for each fragment. We avoid saving before this number.
List <int> InitialCompileCount = FilteredFragments.Select(x => x.Samples.Count).ToList();
// Create all the actions to execute
List <Action> Actions = new List <Action>();
foreach (FragmentTimingData Fragment in FilteredFragments)
{
FragmentTimingData FragmentCopy = Fragment;
for (int SampleIdx = Fragment.Samples.Count; SampleIdx < NumSamples; SampleIdx++)
{
int SampleIdxCopy = SampleIdx;
Actions.Add(() => FragmentCopy.Compile(IntermediateDir, SampleIdxCopy));
}
}
// Randomize the order to ensure that compile times are not consistently affected by other files being compiled simultaneously.
Random Random = new Random();
Actions = Actions.OrderBy(x => Random.Next()).ToList();
// Compile them all
if (Actions.Count > 0)
{
Utility.ParallelForWithStatus("Compiling fragments...", 0, Actions.Count, new ParallelOptions {
MaxDegreeOfParallelism = MaxParallel
}, Idx => Actions[Idx](), Log);
}
// Write out the results
if (NumShards > 1)
{
// If we're running a sharded build, write out intermediate files containing the results
FileReference OutputFile = FileReference.Combine(IntermediateDir, String.Format("Shard{0}.csv", Shard));
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
Writer.WriteLine("Name,Digest,TotalTime,FrontendTime,BackendTime");
for (int Idx = 0; Idx < FilteredFragments.Count; Idx++)
{
FragmentTimingData FilteredFragment = FilteredFragments[Idx];
for (int SampleIdx = InitialCompileCount[Idx]; SampleIdx < FilteredFragment.Samples.Count; SampleIdx++)
{
FragmentTimingSample Sample = FilteredFragment.Samples[SampleIdx];
Writer.WriteLine("{0},{1},{2},{3},{4}", FilteredFragment.UniqueName, FilteredFragment.Digest, Sample.TotalTime, Sample.FrontendTime, Sample.BackendTime);
}
}
}
}
else
{
// Write out the fragment report
FileReference FragmentReport = FileReference.Combine(WorkingDir, "Timing.csv");
Log.WriteLine("Writing {0}...", FragmentReport);
using (StreamWriter Writer = new StreamWriter(FragmentReport.FullName))
{
// Write the header
Writer.Write("Fragment,MinLine,MaxLine");
// Write the labels for each sample type
string[] Types = new string[] { "Total", "Frontend", "Backend" };
for (int Idx = 0; Idx < Types.Length; Idx++)
{
for (int SampleIdx = 0; SampleIdx < NumSamples; SampleIdx++)
{
Writer.Write(",{0}{1}", Types[Idx], SampleIdx + 1);
}
Writer.Write(",{0}Min,{0}Max,{0}Avg,{0}Exc", Types[Idx]);
}
Writer.WriteLine();
// Write all the results
Func <FragmentTimingSample, double>[] TimeFieldDelegates = new Func <FragmentTimingSample, double>[] { x => x.TotalTime, x => x.FrontendTime, x => x.BackendTime };
foreach (FragmentTimingData TimingData in FragmentToTimingData.Values)
{
Writer.Write("{0},{1},{2}", TimingData.Fragment.Location.GetFileName(), TimingData.Fragment.MarkupMin + 1, TimingData.Fragment.MarkupMax + 1);
foreach (Func <FragmentTimingSample, double> TimeFieldDelegate in TimeFieldDelegates)
{
foreach (FragmentTimingSample Sample in TimingData.Samples)
{
Writer.Write(",{0:0.000}", TimeFieldDelegate(Sample));
}
Writer.Write(",{0:0.000}", TimingData.Samples.Min(x => TimeFieldDelegate(x)));
Writer.Write(",{0:0.000}", TimingData.Samples.Max(x => TimeFieldDelegate(x)));
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)));
if (TimingData.PrevFragmentData == null)
{
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)));
}
else
{
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)) - TimingData.PrevFragmentData.Samples.Average(x => TimeFieldDelegate(x)));
}
}
Writer.WriteLine();
}
}
}
}
private static void FindOutputFilesHelper(HashSet<FileReference> OutputFiles, DirectoryReference BaseDir, string SearchPrefix, PhysXTargetLib TargetLib)
{
if(!BaseDir.Exists())
{
return;
}
foreach (FileReference FoundFile in BaseDir.EnumerateFileReferences(SearchPrefix))
{
string FileNameUpper = FoundFile.GetFileName().ToString().ToUpper();
bool bIncludeFile = false;
if(TargetLib == PhysXTargetLib.APEX)
{
bIncludeFile = FileGeneratedByAPEX(FileNameUpper);
}
else
{
bIncludeFile = !FileGeneratedByAPEX(FileNameUpper);
}
if(bIncludeFile)
{
OutputFiles.Add(FoundFile);
}
}
}