/// <summary>
/// Attempts to write an owner to a token file transactionally
/// </summary>
/// <returns>True if the lock was acquired, false otherwise</returns>
public bool WriteTokenFile(FileReference Location, string Signature)
{
// Check it doesn't already exist
if (FileReference.Exists(Location))
{
return(false);
}
// Make sure the directory exists
try
{
DirectoryReference.CreateDirectory(Location.Directory);
}
catch (Exception Ex)
{
throw new AutomationException(Ex, "Unable to create '{0}'", Location.Directory);
}
// Create a temp file containing the owner name
string TempFileName;
for (int Idx = 0;; Idx++)
{
TempFileName = String.Format("{0}.{1}.tmp", Location.FullName, Idx);
try
{
byte[] Bytes = Encoding.UTF8.GetBytes(Signature);
using (FileStream Stream = File.Open(TempFileName, FileMode.CreateNew, FileAccess.Write, FileShare.None))
{
Stream.Write(Bytes, 0, Bytes.Length);
}
break;
}
catch (IOException)
{
if (!File.Exists(TempFileName))
{
throw;
}
}
}
// Try to move the temporary file into place.
try
{
File.Move(TempFileName, Location.FullName);
return(true);
}
catch
{
if (!File.Exists(TempFileName))
{
throw;
}
return(false);
}
}
private static void GenerateTempTarget(FileReference RawProjectPath)
{
DirectoryReference TempDir = DirectoryReference.Combine(RawProjectPath.Directory, "Intermediate", "Source");
DirectoryReference.CreateDirectory(TempDir);
// Get the project name for use in temporary files
string ProjectName = RawProjectPath.GetFileNameWithoutExtension();
// Create a target.cs file
FileReference TargetLocation = FileReference.Combine(TempDir, ProjectName + ".Target.cs");
using (StreamWriter Writer = new StreamWriter(TargetLocation.FullName))
{
Writer.WriteLine("using UnrealBuildTool;");
Writer.WriteLine();
Writer.WriteLine("public class {0}Target : TargetRules", ProjectName);
Writer.WriteLine("{");
Writer.WriteLine("\tpublic {0}Target(TargetInfo Target) : base(Target)", ProjectName);
Writer.WriteLine("\t{");
Writer.WriteLine("\t\tType = TargetType.Game;");
Writer.WriteLine("\t\tExtraModuleNames.Add(\"{0}\");", ProjectName);
Writer.WriteLine("\t}");
Writer.WriteLine("}");
}
// Create a build.cs file
FileReference ModuleLocation = FileReference.Combine(TempDir, ProjectName + ".Build.cs");
using (StreamWriter Writer = new StreamWriter(ModuleLocation.FullName))
{
Writer.WriteLine("using UnrealBuildTool;");
Writer.WriteLine();
Writer.WriteLine("public class {0} : ModuleRules", ProjectName);
Writer.WriteLine("{");
Writer.WriteLine("\tpublic {0}(ReadOnlyTargetRules Target) : base(Target)", ProjectName);
Writer.WriteLine("\t{");
Writer.WriteLine("\t\tPCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;");
Writer.WriteLine();
Writer.WriteLine("\t\tPrivateDependencyModuleNames.Add(\"Core\");");
Writer.WriteLine("\t\tPrivateDependencyModuleNames.Add(\"Core\");");
Writer.WriteLine("\t}");
Writer.WriteLine("}");
}
// Create a main module cpp file
FileReference SourceFileLocation = FileReference.Combine(TempDir, ProjectName + ".cpp");
using (StreamWriter Writer = new StreamWriter(SourceFileLocation.FullName))
{
Writer.WriteLine("#include \"CoreTypes.h\"");
Writer.WriteLine("#include \"Modules/ModuleManager.h\"");
Writer.WriteLine();
Writer.WriteLine("IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, {0}, \"{0}\");", ProjectName);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// If we're merging telemetry from the child process, get a temp filename for it
FileReference TelemetryFile = null;
if (Parameters.MergeTelemetryWithPrefix != null)
{
TelemetryFile = FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Intermediate", "UAT", "Telemetry.json");
DirectoryReference.CreateDirectory(TelemetryFile.Directory);
}
// Run the command
StringBuilder CommandLine = new StringBuilder();
if (Parameters.Arguments == null || (!Parameters.Arguments.CaseInsensitiveContains("-p4") && !Parameters.Arguments.CaseInsensitiveContains("-nop4")))
{
CommandLine.AppendFormat("{0} ", CommandUtils.P4Enabled ? "-p4" : "-nop4");
}
if (Parameters.Arguments == null || (!Parameters.Arguments.CaseInsensitiveContains("-submit") && !Parameters.Arguments.CaseInsensitiveContains("-nosubmit")))
{
if (GlobalCommandLine.Submit.IsSet)
{
CommandLine.Append("-submit ");
}
if (GlobalCommandLine.NoSubmit.IsSet)
{
CommandLine.Append("-nosubmit ");
}
}
CommandLine.Append(Parameters.Name);
if (!String.IsNullOrEmpty(Parameters.Arguments))
{
CommandLine.AppendFormat(" {0}", Parameters.Arguments);
}
if (TelemetryFile != null)
{
CommandLine.AppendFormat(" -Telemetry={0}", CommandUtils.MakePathSafeToUseWithCommandLine(TelemetryFile.FullName));
}
CommandUtils.RunUAT(CommandUtils.CmdEnv, CommandLine.ToString(), Identifier: Parameters.Name);
// Merge in any new telemetry data that was produced
if (TelemetryFile != null && FileReference.Exists(TelemetryFile))
{
Log.TraceLog("Merging telemetry from {0}", TelemetryFile);
TelemetryData NewTelemetry;
if (TelemetryData.TryRead(TelemetryFile, out NewTelemetry))
{
CommandUtils.Telemetry.Merge(Parameters.MergeTelemetryWithPrefix, NewTelemetry);
}
else
{
Log.TraceWarning("Unable to read UAT telemetry file from {0}", TelemetryFile);
}
}
}
private static void MakeFreshDirectoryIfRequired(DirectoryReference Directory)
{
if (!DirectoryReference.Exists(Directory))
{
DirectoryReference.CreateDirectory(Directory);
}
else
{
InternalUtils.SafeDeleteDirectory(Directory.FullName);
DirectoryReference.CreateDirectory(Directory);
}
}
public override void ExecuteBuild()
{
// Get the output directory
string TargetDirParam = ParseParamValue("TargetDir");
if (TargetDirParam == null)
{
throw new AutomationException("Missing -TargetDir=... argument to CopyUAT");
}
// Construct a dummy UE4Build object to get a list of the UAT and UBT build products
UE4Build Build = new UE4Build(this);
Build.AddUATFilesToBuildProducts();
if (ParseParam("WithLauncher"))
{
Build.AddUATLauncherFilesToBuildProducts();
}
Build.AddUBTFilesToBuildProducts();
// Get a list of all the input files
List <FileReference> SourceFiles = new List <FileReference>();
foreach (string BuildProductFile in Build.BuildProductFiles)
{
FileReference SourceFile = new FileReference(BuildProductFile);
SourceFiles.Add(SourceFile);
FileReference SourceSymbolFile = SourceFile.ChangeExtension(".pdb");
if (FileReference.Exists(SourceSymbolFile))
{
SourceFiles.Add(SourceSymbolFile);
}
FileReference DocumentationFile = SourceFile.ChangeExtension(".xml");
if (FileReference.Exists(DocumentationFile))
{
SourceFiles.Add(DocumentationFile);
}
}
// Copy all the files over
DirectoryReference TargetDir = new DirectoryReference(TargetDirParam);
foreach (FileReference SourceFile in SourceFiles)
{
FileReference TargetFile = FileReference.Combine(TargetDir, SourceFile.MakeRelativeTo(CommandUtils.RootDirectory));
DirectoryReference.CreateDirectory(TargetFile.Directory);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
}
Log("Copied {0} files to {1}", SourceFiles.Count, TargetDir);
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// If we're merging telemetry from the child process, get a temp filename for it
FileReference TelemetryFile = null;
if (Parameters.MergeTelemetryWithPrefix != null)
{
TelemetryFile = FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Intermediate", "UAT", "Telemetry.json");
DirectoryReference.CreateDirectory(TelemetryFile.Directory);
}
// Run the command
StringBuilder CommandLine = new StringBuilder();
if (Parameters.Arguments == null || (!Parameters.Arguments.CaseInsensitiveContains("-p4") && !Parameters.Arguments.CaseInsensitiveContains("-nop4")))
{
CommandLine.AppendFormat("{0} ", CommandUtils.P4Enabled ? "-p4" : "-nop4");
}
if (Parameters.Arguments == null || (!Parameters.Arguments.CaseInsensitiveContains("-submit") && !Parameters.Arguments.CaseInsensitiveContains("-nosubmit")))
{
CommandLine.AppendFormat("{0} ", CommandUtils.AllowSubmit ? "-submit" : "-nosubmit");
}
CommandLine.Append(Parameters.Name);
if (!String.IsNullOrEmpty(Parameters.Arguments))
{
CommandLine.AppendFormat(" {0}", Parameters.Arguments);
}
if (TelemetryFile != null)
{
CommandLine.AppendFormat(" -Telemetry={0}", CommandUtils.MakePathSafeToUseWithCommandLine(TelemetryFile.FullName));
}
try
{
CommandUtils.RunUAT(CommandUtils.CmdEnv, CommandLine.ToString());
}
catch (CommandUtils.CommandFailedException)
{
return(false);
}
// Merge in any new telemetry data that was produced
if (Parameters.MergeTelemetryWithPrefix != null)
{
TelemetryData NewTelemetry;
if (TelemetryData.TryRead(TelemetryFile.FullName, out NewTelemetry))
{
CommandUtils.Telemetry.Merge(Parameters.MergeTelemetryWithPrefix, NewTelemetry);
}
}
return(true);
}
void StageBootstrapExecutable(DeploymentContext SC, string ExeName, FileReference TargetFile, StagedFileReference StagedRelativeTargetPath, string StagedArguments)
{
FileReference InputFile = FileReference.Combine(SC.LocalRoot, "Engine", "Binaries", SC.PlatformDir, String.Format("BootstrapPackagedGame-{0}-Shipping.exe", SC.PlatformDir));
if (FileReference.Exists(InputFile))
{
// Create the new bootstrap program
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
DirectoryReference.CreateDirectory(IntermediateDir);
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, ExeName);
CommandUtils.CopyFile(InputFile.FullName, IntermediateFile.FullName);
CommandUtils.SetFileAttributes(IntermediateFile.FullName, ReadOnly: false);
// currently the icon updating doesn't run under mono
if (UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64 ||
UnrealBuildTool.BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win32)
{
// Get the icon from the build directory if possible
GroupIconResource GroupIcon = null;
if (FileReference.Exists(FileReference.Combine(SC.ProjectRoot, "Build/Windows/Application.ico")))
{
GroupIcon = GroupIconResource.FromIco(FileReference.Combine(SC.ProjectRoot, "Build/Windows/Application.ico").FullName);
}
if (GroupIcon == null)
{
GroupIcon = GroupIconResource.FromExe(TargetFile.FullName);
}
// Update the resources in the new file
using (ModuleResourceUpdate Update = new ModuleResourceUpdate(IntermediateFile.FullName, false))
{
const int IconResourceId = 101;
if (GroupIcon != null)
{
Update.SetIcons(IconResourceId, GroupIcon);
}
const int ExecFileResourceId = 201;
Update.SetData(ExecFileResourceId, ResourceType.RawData, Encoding.Unicode.GetBytes(StagedRelativeTargetPath + "\0"));
const int ExecArgsResourceId = 202;
Update.SetData(ExecArgsResourceId, ResourceType.RawData, Encoding.Unicode.GetBytes(StagedArguments + "\0"));
}
}
// Copy it to the staging directory
SC.StageFile(StagedFileType.SystemNonUFS, IntermediateFile, new StagedFileReference(ExeName));
}
}
protected override bool PerformPrequisites()
{
if (TaskOptions.HasFlag(DDCTaskOptions.ColdDDC))
{
StoredEnvVars.Clear();
CachePaths.Clear();
// We put our temp DDC paths in here
DirectoryReference BasePath = DirectoryReference.Combine(CommandUtils.EngineDirectory, "BenchmarkDDC");
IEnumerable <string> DDCEnvVars = new string[] { GetXPlatformEnvironmentKey("UE-BootDataCachePath"), GetXPlatformEnvironmentKey("UE-LocalDataCachePath") };
if (TaskOptions.HasFlag(DDCTaskOptions.KeepMemoryDDC))
{
DDCEnvVars = DDCEnvVars.Where(E => !E.Contains("UE-Boot"));
}
// get all current environment vars and set them to our temp dir
foreach (var Key in DDCEnvVars)
{
// save current key
StoredEnvVars.Add(Key, Environment.GetEnvironmentVariable(Key));
// create a new dir for this key
DirectoryReference Dir = DirectoryReference.Combine(BasePath, Key);
if (DirectoryReference.Exists(Dir))
{
DirectoryReference.Delete(Dir, true);
}
DirectoryReference.CreateDirectory(Dir);
// save this dir and set it as the env var
CachePaths.Add(Dir);
Environment.SetEnvironmentVariable(Key, Dir.FullName);
}
// remove project files
DirectoryReference ProjectDDC = DirectoryReference.Combine(ProjectFile.Directory, "DerivedDataCache");
CommandUtils.DeleteDirectory_NoExceptions(ProjectDDC.FullName);
// remove S3 files
DirectoryReference S3DDC = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "S3DDC");
CommandUtils.DeleteDirectory_NoExceptions(S3DDC.FullName);
}
return(base.PerformPrequisites());
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Find the matching files
FileReference[] PdbFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.BinaryFiles, TagNameToFileSet).Where(x => x.HasExtension(".pdb")).ToArray();
// Find all the matching source files
FileReference[] SourceFiles = ResolveFilespec(CommandUtils.RootDirectory, Parameters.SourceFiles, TagNameToFileSet).ToArray();
// Get the PDBSTR.EXE path, using the latest SDK version we can find.
FileReference PdbStrExe;
if (!TryGetPdbStrExe("v10.0", out PdbStrExe) && !TryGetPdbStrExe("v8.1", out PdbStrExe) && !TryGetPdbStrExe("v8.0", out PdbStrExe))
{
CommandUtils.LogError("Couldn't find PDBSTR.EXE in any Windows SDK installation");
return(false);
}
// Get the path to the generated SRCSRV.INI file
FileReference SrcSrvIni = FileReference.Combine(CommandUtils.RootDirectory, "Engine", "Intermediate", "SrcSrv.ini");
DirectoryReference.CreateDirectory(SrcSrvIni.Directory);
// Generate the SRCSRV.INI file
using (StreamWriter Writer = new StreamWriter(SrcSrvIni.FullName))
{
Writer.WriteLine("SRCSRV: ini------------------------------------------------");
Writer.WriteLine("VERSION=1");
Writer.WriteLine("VERCTRL=Perforce");
Writer.WriteLine("SRCSRV: variables------------------------------------------");
Writer.WriteLine("SRCSRVTRG=%sdtrg%");
Writer.WriteLine("SRCSRVCMD=%sdcmd%");
Writer.WriteLine("SDCMD=p4.exe print -o %srcsrvtrg% \"{0}/%var2%@{1}\"", Parameters.Branch.TrimEnd('/'), Parameters.Change);
Writer.WriteLine("SDTRG=%targ%\\{0}\\{1}\\%fnbksl%(%var2%)", Parameters.Branch.Replace('/', '+'), Parameters.Change);
Writer.WriteLine("SRCSRV: source files ---------------------------------------");
foreach (FileReference SourceFile in SourceFiles)
{
string RelativeSourceFile = SourceFile.MakeRelativeTo(CommandUtils.RootDirectory);
Writer.WriteLine("{0}*{1}", SourceFile.FullName, RelativeSourceFile.Replace('\\', '/'));
}
Writer.WriteLine("SRCSRV: end------------------------------------------------");
}
// Execute PDBSTR on the PDB files in parallel.
bool[] Results = new bool[PdbFiles.Length];
Parallel.For(0, PdbFiles.Length, (Idx, State) => { Results[Idx] = ExecuteTool(PdbStrExe, PdbFiles[Idx], SrcSrvIni, State); });
return(Results.All(x => x));
}
/// <summary>
/// Creates a zip file containing the given input files
/// </summary>
/// <param name="ZipFileName">Filename for the zip</param>
/// <param name="Filter">Filter which selects files to be included in the zip</param>
/// <param name="BaseDirectory">Base directory to store relative paths in the zip file to</param>
/// <param name="CopyModeOnly">No compression will be done. Only acts like a container. The default value is set to false.</param>
internal static void InternalZipFiles(FileReference ZipFileName, DirectoryReference BaseDirectory, FileFilter Filter, int CompressionLevel = 0)
{
// Ionic.Zip.Zip64Option.Always option produces broken archives on Mono, so we use system zip tool instead
if (Utils.IsRunningOnMono)
{
DirectoryReference.CreateDirectory(ZipFileName.Directory);
CommandUtils.PushDir(BaseDirectory.FullName);
string FilesList = "";
foreach (FileReference FilteredFile in Filter.ApplyToDirectory(BaseDirectory, true))
{
FilesList += " \"" + FilteredFile.MakeRelativeTo(BaseDirectory) + "\"";
if (FilesList.Length > 32000)
{
CommandUtils.RunAndLog(CommandUtils.CmdEnv, "zip", "-g -q \"" + ZipFileName + "\"" + FilesList);
FilesList = "";
}
}
if (FilesList.Length > 0)
{
CommandUtils.RunAndLog(CommandUtils.CmdEnv, "zip", "-g -q \"" + ZipFileName + "\"" + FilesList);
}
CommandUtils.PopDir();
}
else
{
using (Ionic.Zip.ZipFile Zip = new Ionic.Zip.ZipFile())
{
Zip.UseZip64WhenSaving = Ionic.Zip.Zip64Option.Always;
Zip.CompressionLevel = (Ionic.Zlib.CompressionLevel)CompressionLevel;
if (Zip.CompressionLevel == Ionic.Zlib.CompressionLevel.Level0)
{
Zip.CompressionMethod = Ionic.Zip.CompressionMethod.None;
}
foreach (FileReference FilteredFile in Filter.ApplyToDirectory(BaseDirectory, true))
{
Zip.AddFile(FilteredFile.FullName, Path.GetDirectoryName(FilteredFile.MakeRelativeTo(BaseDirectory)));
}
CommandUtils.CreateDirectory(Path.GetDirectoryName(ZipFileName.FullName));
Zip.Save(ZipFileName.FullName);
}
}
}
FileReference CreateHostProject(FileReference HostProjectFile, FileReference PluginFile)
{
DirectoryReference HostProjectDir = HostProjectFile.Directory;
DirectoryReference.CreateDirectory(HostProjectDir);
// Create the new project descriptor
File.WriteAllText(HostProjectFile.FullName, "{ \"FileVersion\": 3, \"Plugins\": [ { \"Name\": \"" + PluginFile.GetFileNameWithoutExtension() + "\", \"Enabled\": true } ] }");
// Get the plugin directory in the host project, and copy all the files in
DirectoryReference HostProjectPluginDir = DirectoryReference.Combine(HostProjectDir, "Plugins", PluginFile.GetFileNameWithoutExtension());
CommandUtils.ThreadedCopyFiles(PluginFile.Directory.FullName, HostProjectPluginDir.FullName);
CommandUtils.DeleteDirectory(true, DirectoryReference.Combine(HostProjectPluginDir, "Intermediate").FullName);
// Return the path to the plugin file in the host project
return(FileReference.Combine(HostProjectPluginDir, PluginFile.GetFileName()));
}
public override int Execute()
{
// Create the repo
Repository Repo = CreateOrLoadRepository(ServerAndPort, UserName, BaseDir, bOverwrite);
// Delete any old log files
FileReference LogFile = FileReference.Combine(BaseDir, "Logs", "Log.txt");
DirectoryReference.CreateDirectory(LogFile.Directory);
BackupLogFile(LogFile, TimeSpan.FromDays(3));
if (bVerbose)
{
Log.OutputLevel = LogEventType.Verbose;
}
Trace.Listeners.Add(new TextWriterTraceListener(new StreamWriter(LogFile.FullName), "LogTraceListener"));
Execute(Repo);
return(0);
}
/// <summary>
/// Export the build graph to a Json file for parsing by Horde
/// </summary>
/// <param name="File">Output file to write</param>
public void ExportForHorde(FileReference File)
{
DirectoryReference.CreateDirectory(File.Directory);
using (JsonWriter JsonWriter = new JsonWriter(File.FullName))
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteArrayStart("Groups");
foreach (Agent Agent in Agents)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteArrayStart("Nodes");
foreach (Node Node in Agent.Nodes)
{
JsonWriter.WriteObjectStart();
JsonWriter.WriteValue("Name", Node.Name);
JsonWriter.WriteValue("Group", Agent.Name);
JsonWriter.WriteValue("RunEarly", Node.bRunEarly);
JsonWriter.WriteValue("Exclusive", true);
JsonWriter.WriteArrayStart("InputDependencies");
foreach (string InputDependency in Node.GetDirectInputDependencies().Select(x => x.Name))
{
JsonWriter.WriteValue(InputDependency);
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteArrayStart("OrderDependencies");
foreach (string OrderDependency in Node.GetDirectOrderDependencies().Select(x => x.Name))
{
JsonWriter.WriteValue(OrderDependency);
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
JsonWriter.WriteArrayEnd();
JsonWriter.WriteObjectEnd();
}
}
public override void ExecuteBuild()
{
// Parse all the arguments
string TargetName = ParseRequiredStringParam("Name");
string PlatformName = ParseOptionalStringParam("Platform");
UnrealTargetPlatform Platform;
if (UnrealTargetPlatform.TryParse(PlatformName, out Platform))
{
Platform = HostPlatform.Current.HostEditorPlatform;
}
UnrealTargetConfiguration Configuration = ParseOptionalEnumParam <UnrealTargetConfiguration>("Configuration") ?? UnrealTargetConfiguration.Development;
string Architecture = ParseOptionalStringParam("Architecture");
FileReference ProjectFile = ParseOptionalFileReferenceParam("Project");
DirectoryReference ToDir = ParseRequiredDirectoryReferenceParam("To");
// Read the receipt
FileReference ReceiptFile = TargetReceipt.GetDefaultPath(DirectoryReference.FromFile(ProjectFile) ?? EngineDirectory, TargetName, Platform, Configuration, Architecture);
if (!FileReference.Exists(ReceiptFile))
{
throw new AutomationException("Unable to find '{0}'", ReceiptFile);
}
TargetReceipt Receipt = TargetReceipt.Read(ReceiptFile);
// Enumerate all the files we want to move
List <FileReference> FilesToMove = new List <FileReference>();
FilesToMove.Add(ReceiptFile);
FilesToMove.AddRange(Receipt.BuildProducts.Select(x => x.Path));
// Move all the files to the output folder
DirectoryReference.CreateDirectory(ToDir);
CommandUtils.DeleteDirectoryContents(ToDir.FullName);
foreach (FileReference SourceFile in FilesToMove)
{
FileReference TargetFile = FileReference.Combine(ToDir, SourceFile.MakeRelativeTo(RootDirectory));
LogInformation("Copying {0} to {1}", SourceFile, TargetFile);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
}
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which these build products belong to</param>
/// <param name="OutputName">The output name of the node.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string OutputName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using (TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if (bRemote)
{
// Create the shared directory for this node
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
SharedNodeDir.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedNodeDir, LocalNodeDir, OutputName);
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, OutputName));
return(Manifest);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the base directory
DirectoryReference BaseDir = ResolveDirectory(Parameters.BaseDir);
// Get the output directory
DirectoryReference OutputDir = ResolveDirectory(Parameters.OutputDir);
// Find the matching files
FileReference[] SourceFiles = ResolveFilespec(BaseDir, Parameters.Files, TagNameToFileSet).OrderBy(x => x.FullName).ToArray();
// Create the matching target files
FileReference[] TargetFiles = SourceFiles.Select(x => FileReference.Combine(OutputDir, x.MakeRelativeTo(BaseDir))).ToArray();
// Run the stripping command
Platform TargetPlatform = Platform.GetPlatform(Parameters.Platform);
for (int Idx = 0; Idx < SourceFiles.Length; Idx++)
{
DirectoryReference.CreateDirectory(TargetFiles[Idx].Directory);
if (SourceFiles[Idx] == TargetFiles[Idx])
{
CommandUtils.Log("Stripping symbols: {0}", SourceFiles[Idx].FullName);
}
else
{
CommandUtils.Log("Stripping symbols: {0} -> {1}", SourceFiles[Idx].FullName, TargetFiles[Idx].FullName);
}
TargetPlatform.StripSymbols(SourceFiles[Idx], TargetFiles[Idx]);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target files to the set of build products
BuildProducts.UnionWith(TargetFiles);
return(true);
}
public void Dispose()
{
// Try to write the output file in a transactional way; write it to a temporary file and rename it.
DirectoryReference.CreateDirectory(OutputFile.Directory);
InputData Data = new InputData();
Data.Jobs.Add(Job);
FileReference TempOutputFile = new FileReference(OutputFile.FullName + ".incoming");
using (MemoryStream Stream = new MemoryStream())
{
DataContractJsonSerializer InputFileDataSerializer = new DataContractJsonSerializer(typeof(InputData));
InputFileDataSerializer.WriteObject(Stream, Data);
FileReference.WriteAllBytes(TempOutputFile, Stream.ToArray());
}
FileReference.Delete(OutputFile);
FileReference.Move(TempOutputFile, OutputFile);
}
public bool CopyBuild(DirectoryReference InstallPath)
{
CommandUtils.LogInformation("Copying shared cooked build from stage directory: {0} to {1}", Path.FullName, InstallPath.FullName);
// Delete existing
if (DirectoryReference.Exists(InstallPath))
{
DirectoryReference.Delete(InstallPath, true);
}
DirectoryReference.CreateDirectory(InstallPath);
// Copy new
if (!CommandUtils.CopyDirectory_NoExceptions(Path.FullName, InstallPath.FullName))
{
CommandUtils.LogWarning("Failed to copy {0} -> {1}", Path.FullName, InstallPath.FullName);
return false;
}
FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(InstallPath.FullName, SyncedBuildFileName));
FileReference.WriteAllLines(SyncedBuildFile, new string[] { CL.ToString(), Path.FullName });
return true;
}
void StageBootstrapExecutable(DeploymentContext SC, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments)
{
// create a temp script file location
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, ExeName);
DirectoryReference.CreateDirectory(IntermediateDir);
// make sure slashes are good
StagedRelativeTargetPath = StagedRelativeTargetPath.Replace("\\", "/");
// make contents
StringBuilder Script = new StringBuilder();
string EOL = "\n";
Script.Append("#!/bin/sh" + EOL);
// allow running from symlinks
Script.AppendFormat("UE4_TRUE_SCRIPT_NAME=$(echo \\\"$0\\\" | xargs readlink -f)" + EOL);
Script.AppendFormat("UE4_PROJECT_ROOT=$(dirname \"$UE4_TRUE_SCRIPT_NAME\")" + EOL);
Script.AppendFormat("chmod +x \"$UE4_PROJECT_ROOT/{0}\"" + EOL, StagedRelativeTargetPath);
Script.AppendFormat("\"$UE4_PROJECT_ROOT/{0}\" {1} $@ " + EOL, StagedRelativeTargetPath, StagedArguments);
// write out the
FileReference.WriteAllText(IntermediateFile, Script.ToString());
if (Utils.IsRunningOnMono)
{
var Result = CommandUtils.Run("sh", string.Format("-c 'chmod +x \\\"{0}\\\"'", IntermediateFile));
if (Result.ExitCode != 0)
{
throw new AutomationException(string.Format("Failed to chmod \"{0}\"", IntermediateFile));
}
}
SC.StageFile(StagedFileType.NonUFS, IntermediateFile, new StagedFileReference(ExeName));
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job.</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include.</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
string FileText = Parameters.Text;
// If any files or tagsets are provided, add them to the text output.
if (!String.IsNullOrEmpty(Parameters.Files))
{
if (!string.IsNullOrWhiteSpace(FileText))
{
FileText += Environment.NewLine;
}
HashSet <FileReference> Files = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet);
if (Files.Any())
{
FileText += string.Join(Environment.NewLine, Files.Select(f => f.FullName));
}
}
// Make sure output folder exists.
if (!DirectoryReference.Exists(Parameters.File.Directory))
{
DirectoryReference.CreateDirectory(Parameters.File.Directory);
}
if (Parameters.Append)
{
CommandUtils.LogInformation(string.Format("Appending text to file '{0}': {1}", Parameters.File, FileText));
FileReference.AppendAllText(Parameters.File, Environment.NewLine + FileText);
}
else
{
CommandUtils.LogInformation(string.Format("Writing text to file '{0}': {1}", Parameters.File, FileText));
FileReference.WriteAllText(Parameters.File, FileText);
}
}
/// <summary>
/// Generate HTML documentation for all the tasks
/// </summary>
/// <param name="NameToTask">Map of task name to implementation</param>
/// <param name="OutputFile">Output file</param>
static void GenerateDocumentation(Dictionary <string, ScriptTask> NameToTask, FileReference OutputFile)
{
// Find all the assemblies containing tasks
Assembly[] TaskAssemblies = NameToTask.Values.Select(x => x.ParametersClass.Assembly).Distinct().ToArray();
// Read documentation for each of them
Dictionary <string, XmlElement> MemberNameToElement = new Dictionary <string, XmlElement>();
foreach (Assembly TaskAssembly in TaskAssemblies)
{
string XmlFileName = Path.ChangeExtension(TaskAssembly.Location, ".xml");
if (File.Exists(XmlFileName))
{
// Read the document
XmlDocument Document = new XmlDocument();
Document.Load(XmlFileName);
// Parse all the members, and add them to the map
foreach (XmlElement Element in Document.SelectNodes("/doc/members/member"))
{
string Name = Element.GetAttribute("name");
MemberNameToElement.Add(Name, Element);
}
}
}
// Create the output directory
if (FileReference.Exists(OutputFile))
{
FileReference.MakeWriteable(OutputFile);
}
else
{
DirectoryReference.CreateDirectory(OutputFile.Directory);
}
// Write the output file
LogInformation("Writing {0}...", OutputFile);
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
Writer.WriteLine("<html>");
Writer.WriteLine(" <head>");
Writer.WriteLine(" <style>");
Writer.WriteLine(" table { border-collapse: collapse }");
Writer.WriteLine(" table, th, td { border: 1px solid black; }");
Writer.WriteLine(" </style>");
Writer.WriteLine(" </head>");
Writer.WriteLine(" <body>");
Writer.WriteLine(" <h1>BuildGraph Tasks</h1>");
foreach (string TaskName in NameToTask.Keys.OrderBy(x => x))
{
// Get the task object
ScriptTask Task = NameToTask[TaskName];
// Get the documentation for this task
XmlElement TaskElement;
if (MemberNameToElement.TryGetValue("T:" + Task.TaskClass.FullName, out TaskElement))
{
// Write the task heading
Writer.WriteLine(" <h2>{0}</h2>", TaskName);
Writer.WriteLine(" <p>{0}</p>", TaskElement.SelectSingleNode("summary").InnerXml.Trim());
// Start the parameter table
Writer.WriteLine(" <table>");
Writer.WriteLine(" <tr>");
Writer.WriteLine(" <th>Attribute</th>");
Writer.WriteLine(" <th>Type</th>");
Writer.WriteLine(" <th>Usage</th>");
Writer.WriteLine(" <th>Description</th>");
Writer.WriteLine(" </tr>");
// Document the parameters
foreach (string ParameterName in Task.NameToParameter.Keys)
{
// Get the parameter data
ScriptTaskParameter Parameter = Task.NameToParameter[ParameterName];
// Get the documentation for this parameter
XmlElement ParameterElement;
if (MemberNameToElement.TryGetValue("F:" + Parameter.FieldInfo.DeclaringType.FullName + "." + Parameter.Name, out ParameterElement))
{
string TypeName = Parameter.FieldInfo.FieldType.Name;
if (Parameter.ValidationType != TaskParameterValidationType.Default)
{
StringBuilder NewTypeName = new StringBuilder(Parameter.ValidationType.ToString());
for (int Idx = 1; Idx < NewTypeName.Length; Idx++)
{
if (Char.IsLower(NewTypeName[Idx - 1]) && Char.IsUpper(NewTypeName[Idx]))
{
NewTypeName.Insert(Idx, ' ');
}
}
TypeName = NewTypeName.ToString();
}
Writer.WriteLine(" <tr>");
Writer.WriteLine(" <td>{0}</td>", ParameterName);
Writer.WriteLine(" <td>{0}</td>", TypeName);
Writer.WriteLine(" <td>{0}</td>", Parameter.bOptional? "Optional" : "Required");
Writer.WriteLine(" <td>{0}</td>", ParameterElement.SelectSingleNode("summary").InnerXml.Trim());
Writer.WriteLine(" </tr>");
}
}
// Always include the "If" attribute
Writer.WriteLine(" <tr>");
Writer.WriteLine(" <td>If</td>");
Writer.WriteLine(" <td>Condition</td>");
Writer.WriteLine(" <td>Optional</td>");
Writer.WriteLine(" <td>Whether to execute this task. It is ignored if this condition evaluates to false.</td>");
Writer.WriteLine(" </tr>");
// Close the table
Writer.WriteLine(" <table>");
}
}
Writer.WriteLine(" </body>");
Writer.WriteLine("</html>");
}
}
/// <summary>
/// Generate HTML documentation for all the tasks
/// </summary>
/// <param name="NameToTask">Map of task name to implementation</param>
/// <param name="OutputFile">Output file</param>
static void GenerateDocumentation(Dictionary <string, ScriptTask> NameToTask, FileReference OutputFile)
{
// Find all the assemblies containing tasks
Assembly[] TaskAssemblies = NameToTask.Values.Select(x => x.ParametersClass.Assembly).Distinct().ToArray();
// Read documentation for each of them
Dictionary <string, XmlElement> MemberNameToElement = new Dictionary <string, XmlElement>();
foreach (Assembly TaskAssembly in TaskAssemblies)
{
string XmlFileName = Path.ChangeExtension(TaskAssembly.Location, ".xml");
if (File.Exists(XmlFileName))
{
// Read the document
XmlDocument Document = new XmlDocument();
Document.Load(XmlFileName);
// Parse all the members, and add them to the map
foreach (XmlElement Element in Document.SelectNodes("/doc/members/member"))
{
string Name = Element.GetAttribute("name");
MemberNameToElement.Add(Name, Element);
}
}
}
// Create the output directory
DirectoryReference.CreateDirectory(OutputFile.Directory);
FileReference.MakeWriteable(OutputFile);
Log("Writing {0}...", OutputFile);
// Parse the engine version
BuildVersion Version;
if (!BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version))
{
throw new AutomationException("Couldn't read Build.version");
}
// Write the output file
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
Writer.WriteLine("Availability: NoPublish");
Writer.WriteLine("Title: BuildGraph Predefined Tasks");
Writer.WriteLine("Crumbs: %ROOT%, Programming, Programming/Development, Programming/Development/BuildGraph, Programming/Development/BuildGraph/BuildGraphScriptTasks");
Writer.WriteLine("Description: This is a procedurally generated markdown page.");
Writer.WriteLine("version: {0}.{1}", Version.MajorVersion, Version.MinorVersion);
Writer.WriteLine("parent:Programming/Development/BuildGraph/BuildGraphScriptTasks");
Writer.WriteLine();
foreach (string TaskName in NameToTask.Keys.OrderBy(x => x))
{
// Get the task object
ScriptTask Task = NameToTask[TaskName];
// Get the documentation for this task
XmlElement TaskElement;
if (MemberNameToElement.TryGetValue("T:" + Task.TaskClass.FullName, out TaskElement))
{
// Write the task heading
Writer.WriteLine("### {0}", TaskName);
Writer.WriteLine();
Writer.WriteLine(ConvertToMarkdown(TaskElement.SelectSingleNode("summary")));
Writer.WriteLine();
// Document the parameters
List <string[]> Rows = new List <string[]>();
foreach (string ParameterName in Task.NameToParameter.Keys)
{
// Get the parameter data
ScriptTaskParameter Parameter = Task.NameToParameter[ParameterName];
// Get the documentation for this parameter
XmlElement ParameterElement;
if (MemberNameToElement.TryGetValue("F:" + Parameter.FieldInfo.DeclaringType.FullName + "." + Parameter.Name, out ParameterElement))
{
string TypeName = Parameter.FieldInfo.FieldType.Name;
if (Parameter.ValidationType != TaskParameterValidationType.Default)
{
StringBuilder NewTypeName = new StringBuilder(Parameter.ValidationType.ToString());
for (int Idx = 1; Idx < NewTypeName.Length; Idx++)
{
if (Char.IsLower(NewTypeName[Idx - 1]) && Char.IsUpper(NewTypeName[Idx]))
{
NewTypeName.Insert(Idx, ' ');
}
}
TypeName = NewTypeName.ToString();
}
string[] Columns = new string[4];
Columns[0] = ParameterName;
Columns[1] = TypeName;
Columns[2] = Parameter.bOptional? "Optional" : "Required";
Columns[3] = ConvertToMarkdown(ParameterElement.SelectSingleNode("summary"));
Rows.Add(Columns);
}
}
// Always include the "If" attribute
string[] IfColumns = new string[4];
IfColumns[0] = "If";
IfColumns[1] = "Condition";
IfColumns[2] = "Optional";
IfColumns[3] = "Whether to execute this task. It is ignored if this condition evaluates to false.";
Rows.Add(IfColumns);
// Get the width of each column
int[] Widths = new int[4];
for (int Idx = 0; Idx < 4; Idx++)
{
Widths[Idx] = Rows.Max(x => x[Idx].Length);
}
// Format the markdown table
string Format = String.Format("| {{0,-{0}}} | {{1,-{1}}} | {{2,-{2}}} | {{3,-{3}}} |", Widths[0], Widths[1], Widths[2], Widths[3]);
Writer.WriteLine(Format, "", "", "", "");
Writer.WriteLine(Format, new string('-', Widths[0]), new string('-', Widths[1]), new string('-', Widths[2]), new string('-', Widths[3]));
for (int Idx = 0; Idx < Rows.Count; Idx++)
{
Writer.WriteLine(Format, Rows[Idx][0], Rows[Idx][1], Rows[Idx][2], Rows[Idx][3]);
}
// Blank line before next task
Writer.WriteLine();
}
}
}
}
/// <summary>
/// Main command entry point
/// </summary>
/// <param name="Arguments">The command line arguments</param>
public override void Exec(CommandLineArguments Arguments)
{
// Parse the arguments
bool bClean = Arguments.HasOption("-Clean");
string PerforcePort = Arguments.GetStringOrDefault("-P4Port=", null);
string PerforceUser = Arguments.GetStringOrDefault("-P4User="******"-InputFile=", null);
FileReference StateFile = Arguments.GetFileReference("-StateFile=");
string ServerUrl = Arguments.GetStringOrDefault("-Server=", null);
bool bKeepHistory = Arguments.HasOption("-KeepHistory");
bool bReadOnly = Arguments.HasOption("-ReadOnly");
DirectoryReference SaveUnmatchedDir = Arguments.GetDirectoryReferenceOrDefault("-SaveUnmatched=", null);
Arguments.CheckAllArgumentsUsed();
// Build a mapping from category to matching
Dictionary <string, PatternMatcher> CategoryNameToMatcher = new Dictionary <string, PatternMatcher>();
foreach (PatternMatcher Matcher in Matchers)
{
CategoryNameToMatcher[Matcher.Category] = Matcher;
}
// Complete any interrupted operation to update the state file
CompleteStateTransaction(StateFile);
// Read the persistent data file
BuildHealthState State;
if (!bClean && FileReference.Exists(StateFile))
{
Log.TraceInformation("Reading persistent data from {0}", StateFile);
State = DeserializeJson <BuildHealthState>(StateFile);
}
else
{
Log.TraceInformation("Creating new persistent data");
State = new BuildHealthState();
}
// Fixup any issues loaded from disk
foreach (BuildHealthIssue Issue in State.Issues)
{
if (Issue.References == null)
{
Issue.References = new SortedSet <string>();
}
}
// Create the Perforce connection
PerforceConnection Perforce = new PerforceConnection(PerforcePort, PerforceUser, null);
// Process the input data
if (InputFile != null)
{
// Parse the input file
Log.TraceInformation("Reading build results from {0}", InputFile);
InputData InputData = DeserializeJson <InputData>(InputFile);
// Parse all the builds and add them to the persistent data
List <InputJob> InputJobs = InputData.Jobs.OrderBy(x => x.Change).ThenBy(x => x.Stream).ToList();
Stopwatch Timer = Stopwatch.StartNew();
foreach (InputJob InputJob in InputJobs)
{
// Add a new build for each job step
foreach (InputJobStep InputJobStep in InputJob.Steps)
{
BuildHealthJobStep NewBuild = new BuildHealthJobStep(InputJob.Change, InputJob.Name, InputJob.Url, InputJobStep.Name, InputJobStep.Url, null);
State.AddBuild(InputJob.Stream, NewBuild);
}
// Add all the job steps
List <InputJobStep> InputJobSteps = InputJob.Steps.OrderBy(x => x.Name).ToList();
foreach (InputJobStep InputJobStep in InputJobSteps)
{
if (InputJobStep.Diagnostics != null && InputJobStep.Diagnostics.Count > 0)
{
AddStep(Perforce, State, InputJob, InputJobStep);
}
}
// Remove any steps which are empty
InputJob.Steps.RemoveAll(x => x.Diagnostics == null || x.Diagnostics.Count == 0);
}
InputJobs.RemoveAll(x => x.Steps.Count == 0);
Log.TraceInformation("Added jobs in {0}s", Timer.Elapsed.TotalSeconds);
// If there are any unmatched issues, save out the current state and remaining input
if (SaveUnmatchedDir != null && InputJobs.Count > 0)
{
DirectoryReference.CreateDirectory(SaveUnmatchedDir);
if (FileReference.Exists(StateFile))
{
FileReference.Copy(StateFile, FileReference.Combine(SaveUnmatchedDir, "State.json"), true);
}
SerializeJson(FileReference.Combine(SaveUnmatchedDir, "Input.json"), InputData);
}
// Try to find the next successful build for each stream, so we can close it as part of updating the server
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
BuildHealthIssue Issue = State.Issues[Idx];
foreach (string Stream in Issue.Streams.Keys)
{
Dictionary <string, BuildHealthJobHistory> StepNameToHistory = Issue.Streams[Stream];
foreach (string StepName in StepNameToHistory.Keys)
{
BuildHealthJobHistory IssueHistory = StepNameToHistory[StepName];
if (IssueHistory.FailedBuilds.Count > 0 && IssueHistory.NextSuccessfulBuild == null)
{
// Find the successful build after this change
BuildHealthJobStep LastFailedBuild = IssueHistory.FailedBuilds[IssueHistory.FailedBuilds.Count - 1];
IssueHistory.NextSuccessfulBuild = State.FindBuildAfter(Stream, LastFailedBuild.Change, StepName);
}
}
}
}
// Find the change two days before the latest change being added
if (InputData.Jobs.Count > 0 && !bKeepHistory)
{
// Find all the unique change numbers for each stream
SortedSet <int> ChangeNumbers = new SortedSet <int>();
foreach (List <BuildHealthJobStep> Builds in State.Streams.Values)
{
ChangeNumbers.UnionWith(Builds.Select(x => x.Change));
}
// Get the latest change record
int LatestChangeNumber = InputData.Jobs.Min(x => x.Change);
ChangeRecord LatestChangeRecord = Perforce.GetChange(GetChangeOptions.None, LatestChangeNumber).Data;
// Step forward through all the changelists until we get to one we don't want to delete
int DeleteChangeNumber = -1;
foreach (int ChangeNumber in ChangeNumbers)
{
ChangeRecord ChangeRecord = Perforce.GetChange(GetChangeOptions.None, ChangeNumber).Data;
if (ChangeRecord.Date > LatestChangeRecord.Date - TimeSpan.FromDays(2))
{
break;
}
DeleteChangeNumber = ChangeNumber;
}
// Remove any builds we no longer want to track
foreach (List <BuildHealthJobStep> Builds in State.Streams.Values)
{
Builds.RemoveAll(x => x.Change <= DeleteChangeNumber);
}
}
}
// Mark any issues as resolved
foreach (BuildHealthIssue Issue in State.Issues)
{
if (Issue.IsResolved())
{
if (!Issue.ResolvedAt.HasValue)
{
Issue.ResolvedAt = DateTime.UtcNow;
}
}
else
{
if (Issue.ResolvedAt.HasValue)
{
Issue.ResolvedAt = null;
}
}
}
// If we're in read-only mode, don't write anything out
if (bReadOnly)
{
return;
}
// Save the persistent data
Log.TraceInformation("Writing persistent data to {0}", StateFile);
DirectoryReference.CreateDirectory(StateFile.Directory);
WriteState(StateFile, State);
// Synchronize with the server
if (ServerUrl != null)
{
// Post any issue updates
foreach (BuildHealthIssue Issue in State.Issues)
{
PatternMatcher Matcher;
if (!CategoryNameToMatcher.TryGetValue(Issue.Category, out Matcher))
{
continue;
}
string Summary = Matcher.GetSummary(Issue);
if (Issue.Id == -1)
{
Log.TraceInformation("Adding issue: {0}", Issue);
if (Issue.PendingWatchers.Count == 0)
{
Log.TraceWarning("(No possible causers)");
}
CommandTypes.AddIssue IssueBody = new CommandTypes.AddIssue();
IssueBody.Project = Issue.Project;
IssueBody.Summary = Summary;
if (Issue.PendingWatchers.Count == 1)
{
IssueBody.Owner = Issue.PendingWatchers.First();
}
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues", ServerUrl), "POST", IssueBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add issue");
}
Issue.Id = ParseHttpResponse <CommandTypes.AddIssueResponse>(Response).Id;
}
Issue.PostedSummary = Summary;
WriteState(StateFile, State);
}
else if (Issue.PostedSummary == null || !String.Equals(Issue.PostedSummary, Summary, StringComparison.Ordinal))
{
Log.TraceInformation("Updating issue {0}", Issue.Id);
CommandTypes.UpdateIssue IssueBody = new CommandTypes.UpdateIssue();
IssueBody.Summary = Summary;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}", ServerUrl, Issue.Id), "PUT", IssueBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add issue");
}
}
Issue.PostedSummary = Summary;
WriteState(StateFile, State);
}
}
// Add any new builds associated with issues
Dictionary <string, long> JobStepUrlToId = new Dictionary <string, long>(StringComparer.Ordinal);
foreach (BuildHealthIssue Issue in State.Issues)
{
foreach (KeyValuePair <string, Dictionary <string, BuildHealthJobHistory> > StreamPair in Issue.Streams)
{
foreach (BuildHealthJobHistory StreamHistory in StreamPair.Value.Values)
{
foreach (BuildHealthJobStep Build in StreamHistory.Builds)
{
if (!Build.bPostedToServer)
{
Log.TraceInformation("Adding {0} to issue {1}", Build.JobStepUrl, Issue.Id);
CommandTypes.AddBuild AddBuild = new CommandTypes.AddBuild();
AddBuild.Stream = StreamPair.Key;
AddBuild.Change = Build.Change;
AddBuild.JobName = Build.JobName;
AddBuild.JobUrl = Build.JobUrl;
AddBuild.JobStepName = Build.JobStepName;
AddBuild.JobStepUrl = Build.JobStepUrl;
AddBuild.ErrorUrl = Build.ErrorUrl;
AddBuild.Outcome = (Build == StreamHistory.PrevSuccessfulBuild || Build == StreamHistory.NextSuccessfulBuild)? CommandTypes.Outcome.Success : CommandTypes.Outcome.Error;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/builds", ServerUrl, Issue.Id), "POST", AddBuild))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add build");
}
Build.Id = ParseHttpResponse <CommandTypes.AddBuildResponse>(Response).Id;
}
Build.bPostedToServer = true;
WriteState(StateFile, State);
}
if (Build.Id != -1)
{
JobStepUrlToId[Build.JobStepUrl] = Build.Id;
}
}
}
}
}
// Add any new diagnostics
foreach (BuildHealthIssue Issue in State.Issues)
{
foreach (BuildHealthDiagnostic Diagnostic in Issue.Diagnostics)
{
if (!Diagnostic.bPostedToServer)
{
string Summary = Diagnostic.Message;
const int MaxLength = 40;
if (Summary.Length > MaxLength)
{
Summary = Summary.Substring(0, MaxLength).TrimEnd();
}
Log.TraceInformation("Adding diagnostic '{0}' to issue {1}", Summary, Issue.Id);
CommandTypes.AddDiagnostic AddDiagnostic = new CommandTypes.AddDiagnostic();
long BuildId;
if (Diagnostic.JobStepUrl != null && JobStepUrlToId.TryGetValue(Diagnostic.JobStepUrl, out BuildId))
{
AddDiagnostic.BuildId = BuildId;
}
else
{
Console.WriteLine("ERROR");
}
AddDiagnostic.Message = Diagnostic.Message;
AddDiagnostic.Url = Diagnostic.ErrorUrl;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/diagnostics", ServerUrl, Issue.Id), "POST", AddDiagnostic))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add build");
}
}
Diagnostic.bPostedToServer = true;
WriteState(StateFile, State);
}
}
}
// Close any issues which are complete
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
BuildHealthIssue Issue = State.Issues[Idx];
if (Issue.ResolvedAt.HasValue != Issue.bPostedResolved)
{
Log.TraceInformation("Setting issue {0} resolved flag to {1}", Issue.Id, Issue.ResolvedAt.HasValue);
CommandTypes.UpdateIssue UpdateBody = new CommandTypes.UpdateIssue();
UpdateBody.Resolved = Issue.ResolvedAt.HasValue;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}", ServerUrl, Issue.Id), "PUT", UpdateBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to delete issue");
}
}
Issue.bPostedResolved = Issue.ResolvedAt.HasValue;
WriteState(StateFile, State);
}
}
// Update watchers on any open builds
foreach (BuildHealthIssue Issue in State.Issues)
{
while (Issue.PendingWatchers.Count > 0)
{
CommandTypes.Watcher Watcher = new CommandTypes.Watcher();
Watcher.UserName = Issue.PendingWatchers.First();
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/watchers", ServerUrl, Issue.Id), "POST", Watcher))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add watcher");
}
}
Issue.PendingWatchers.Remove(Watcher.UserName);
Issue.Watchers.Add(Watcher.UserName);
WriteState(StateFile, State);
}
}
}
// Remove any issues which have been resolved for 24 hours. We have to keep information about issues that have been fixed for some time; we may be updating the same job
// multiple times while other steps are running, and we don't want to keep opening new issues for it. Also, it can take time for changes to propagate between streams.
DateTime RemoveIssueTime = DateTime.UtcNow - TimeSpan.FromHours(24.0);
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
BuildHealthIssue Issue = State.Issues[Idx];
if (Issue.ResolvedAt.HasValue && Issue.ResolvedAt.Value < RemoveIssueTime)
{
State.Issues.RemoveAt(Idx--);
WriteState(StateFile, State);
continue;
}
}
// TODO: VERIFY ISSUES ARE CLOSED
}
public static void TakeLock(DirectoryReference LockDirectory, TimeSpan Timeout, System.Action Callback)
{
string LockFilePath = Path.Combine(LockDirectory.FullName, ".lock");
FileStream Stream = null;
DateTime StartTime = DateTime.Now;
DateTime Deadline = StartTime.Add(Timeout);
try
{
DirectoryReference.CreateDirectory(LockDirectory);
for (int Iterations = 0; ; ++Iterations)
{
// Attempt to create the lock file. Ignore any IO exceptions. Stream will be null if this fails.
try { Stream = new FileStream(LockFilePath, FileMode.Create, FileAccess.Write, FileShare.Read, 4096, FileOptions.DeleteOnClose); }
catch (IOException) { }
if (Stream != null)
{
// If we have a stream, we've taken the lock.
try
{
// Write the machine name to the file.
Stream.Write(Encoding.UTF8.GetBytes(Environment.MachineName));
Stream.Flush();
break;
}
catch
{
throw new AutomationException("Failed to write to the lock file '{0}'.", LockFilePath);
}
}
// We've failed to take the lock. Throw an exception if the timeout has elapsed.
// Otherwise print a log message and retry.
var CurrentTime = DateTime.Now;
if (CurrentTime >= Deadline)
{
throw new AutomationException("Couldn't create lock file '{0}' after {1} seconds.", LockFilePath, CurrentTime.Subtract(StartTime).TotalSeconds);
}
if (Iterations == 0)
{
CommandUtils.Log("Waiting for lock file '{0}' to be removed...", LockFilePath);
}
else if ((Iterations % 30) == 0)
{
CommandUtils.Log("Still waiting for lock file '{0}' after {1} seconds.", LockFilePath, CurrentTime.Subtract(StartTime).TotalSeconds);
}
// Wait for a while before retrying.
Thread.Sleep(1000);
}
// Invoke the user callback now that we own the lock.
Callback();
}
finally
{
// Always dispose the lock file stream if we took the lock.
// The file will delete on close.
if (Stream != null)
{
Stream.Dispose();
Stream = null;
}
}
}
public override void ExecuteBuild()
{
int WorkingCL = -1;
FileReference PluginFile = null;
string ProjectFileName = ParseParamValue("Project");
if (ProjectFileName == null)
{
ProjectFileName = CombinePaths(CmdEnv.LocalRoot, "SimpleGame", "SimpleGame.uproject");
}
LogInformation(ProjectFileName);
ProjectParams Params = GetParams(this, ProjectFileName, out PluginFile);
// Check whether folder already exists so we know if we can delete it later
string PlatformStageDir = Path.Combine(Params.StageDirectoryParam, "WindowsNoEditor");
bool bPreExistingStageDir = Directory.Exists(PlatformStageDir);
PluginDescriptor Plugin = PluginDescriptor.FromFile(PluginFile);
FileReference ProjectFile = new FileReference(ProjectFileName);
// Add Plugin to folders excluded for nativization in config file
FileReference UserEditorIni = new FileReference(Path.Combine(Path.GetDirectoryName(ProjectFileName), "Config", "UserEditor.ini"));
bool bPreExistingUserEditorIni = FileReference.Exists(UserEditorIni);
if (!bPreExistingUserEditorIni)
{
// Expect this most of the time so we will create and clean up afterwards
DirectoryReference.CreateDirectory(UserEditorIni.Directory);
CommandUtils.WriteAllText(UserEditorIni.FullName, "");
}
const string ConfigSection = "BlueprintNativizationSettings";
const string ConfigKey = "ExcludedFolderPaths";
string ConfigValue = "/" + PluginFile.GetFileNameWithoutAnyExtensions() + "/";
ConfigFile UserEditorConfig = new ConfigFile(UserEditorIni);
ConfigFileSection BPNSection = UserEditorConfig.FindOrAddSection(ConfigSection);
bool bUpdateConfigFile = !BPNSection.Lines.Exists(x => String.Equals(x.Key, ConfigKey, StringComparison.OrdinalIgnoreCase) && String.Equals(x.Value, ConfigValue, StringComparison.OrdinalIgnoreCase));
if (bUpdateConfigFile)
{
BPNSection.Lines.Add(new ConfigLine(ConfigLineAction.Add, ConfigKey, ConfigValue));
UserEditorConfig.Write(UserEditorIni);
}
Project.Cook(Params);
if (!bPreExistingUserEditorIni)
{
FileReference.Delete(UserEditorIni);
}
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
// Get path to where the plugin was staged
string StagedPluginDir = Path.Combine(PlatformStageDir, Path.GetFileNameWithoutExtension(ProjectFileName), PluginFile.Directory.MakeRelativeTo(ProjectFile.Directory));
string ZipFile = Path.Combine(Params.StageDirectoryParam, PluginFile.GetFileNameWithoutAnyExtensions());
CommandUtils.DeleteFile(ZipFile);
System.IO.Compression.ZipFile.CreateFromDirectory(StagedPluginDir, ZipFile + ".zip");
if (!bPreExistingStageDir)
{
CommandUtils.DeleteDirectory(PlatformStageDir);
}
}
private void WriteSchemeFile(string TargetName, string TargetGuid, string BuildTargetGuid, string IndexTargetGuid, bool bHasEditorConfiguration, string GameProjectPath)
{
string DefaultConfiguration = bHasEditorConfiguration && !XcodeProjectFileGenerator.bGeneratingRunIOSProject && !XcodeProjectFileGenerator.bGeneratingRunTVOSProject ? "Development Editor" : "Development";
var Content = new StringBuilder();
Content.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + ProjectFileGenerator.NewLine);
Content.Append("<Scheme" + ProjectFileGenerator.NewLine);
Content.Append(" LastUpgradeVersion = \"0710\"" + ProjectFileGenerator.NewLine);
Content.Append(" version = \"1.3\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildAction" + ProjectFileGenerator.NewLine);
Content.Append(" parallelizeBuildables = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" buildImplicitDependencies = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildActionEntries>" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildActionEntry" + ProjectFileGenerator.NewLine);
Content.Append(" buildForTesting = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" buildForRunning = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" buildForProfiling = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" buildForArchiving = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" buildForAnalyzing = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildableReference" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableIdentifier = \"primary\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintIdentifier = \"" + TargetGuid + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableName = \"" + TargetName + ".app\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintName = \"" + TargetName + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" ReferencedContainer = \"container:" + TargetName + ".xcodeproj\">" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildableReference>" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildActionEntry>" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildActionEntries>" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildAction>" + ProjectFileGenerator.NewLine);
Content.Append(" <TestAction" + ProjectFileGenerator.NewLine);
Content.Append(" buildConfiguration = \"" + DefaultConfiguration + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" selectedDebuggerIdentifier = \"Xcode.DebuggerFoundation.Debugger.LLDB\"" + ProjectFileGenerator.NewLine);
Content.Append(" selectedLauncherIdentifier = \"Xcode.DebuggerFoundation.Launcher.LLDB\"" + ProjectFileGenerator.NewLine);
Content.Append(" shouldUseLaunchSchemeArgsEnv = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" <Testables>" + ProjectFileGenerator.NewLine);
Content.Append(" </Testables>" + ProjectFileGenerator.NewLine);
Content.Append(" <MacroExpansion>" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildableReference" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableIdentifier = \"primary\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintIdentifier = \"" + TargetGuid + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableName = \"" + TargetName + ".app\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintName = \"" + TargetName + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" ReferencedContainer = \"container:" + TargetName + ".xcodeproj\">" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildableReference>" + ProjectFileGenerator.NewLine);
Content.Append(" </MacroExpansion>" + ProjectFileGenerator.NewLine);
Content.Append(" <AdditionalOptions>" + ProjectFileGenerator.NewLine);
Content.Append(" </AdditionalOptions>" + ProjectFileGenerator.NewLine);
Content.Append(" </TestAction>" + ProjectFileGenerator.NewLine);
Content.Append(" <LaunchAction" + ProjectFileGenerator.NewLine);
Content.Append(" buildConfiguration = \"" + DefaultConfiguration + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" selectedDebuggerIdentifier = \"Xcode.DebuggerFoundation.Debugger.LLDB\"" + ProjectFileGenerator.NewLine);
Content.Append(" selectedLauncherIdentifier = \"Xcode.DebuggerFoundation.Launcher.LLDB\"" + ProjectFileGenerator.NewLine);
Content.Append(" launchStyle = \"0\"" + ProjectFileGenerator.NewLine);
Content.Append(" useCustomWorkingDirectory = \"NO\"" + ProjectFileGenerator.NewLine);
Content.Append(" ignoresPersistentStateOnLaunch = \"NO\"" + ProjectFileGenerator.NewLine);
Content.Append(" debugDocumentVersioning = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" debugServiceExtension = \"internal\"" + ProjectFileGenerator.NewLine);
Content.Append(" allowLocationSimulation = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildableProductRunnable" + ProjectFileGenerator.NewLine);
Content.Append(" runnableDebuggingMode = \"0\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildableReference" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableIdentifier = \"primary\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintIdentifier = \"" + TargetGuid + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableName = \"" + TargetName + ".app\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintName = \"" + TargetName + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" ReferencedContainer = \"container:" + TargetName + ".xcodeproj\">" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildableReference>" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildableProductRunnable>" + ProjectFileGenerator.NewLine);
if (bHasEditorConfiguration && TargetName != "UE4")
{
Content.Append(" <CommandLineArguments>" + ProjectFileGenerator.NewLine);
if (IsForeignProject)
{
Content.Append(" <CommandLineArgument" + ProjectFileGenerator.NewLine);
Content.Append(" argument = \""" + GameProjectPath + ""\"" + ProjectFileGenerator.NewLine);
Content.Append(" isEnabled = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" </CommandLineArgument>" + ProjectFileGenerator.NewLine);
}
else
{
Content.Append(" <CommandLineArgument" + ProjectFileGenerator.NewLine);
Content.Append(" argument = \"" + TargetName + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" isEnabled = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" </CommandLineArgument>" + ProjectFileGenerator.NewLine);
}
Content.Append(" </CommandLineArguments>" + ProjectFileGenerator.NewLine);
}
Content.Append(" <AdditionalOptions>" + ProjectFileGenerator.NewLine);
Content.Append(" </AdditionalOptions>" + ProjectFileGenerator.NewLine);
Content.Append(" </LaunchAction>" + ProjectFileGenerator.NewLine);
Content.Append(" <ProfileAction" + ProjectFileGenerator.NewLine);
Content.Append(" buildConfiguration = \"" + DefaultConfiguration + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" shouldUseLaunchSchemeArgsEnv = \"YES\"" + ProjectFileGenerator.NewLine);
Content.Append(" savedToolIdentifier = \"\"" + ProjectFileGenerator.NewLine);
Content.Append(" useCustomWorkingDirectory = \"NO\"" + ProjectFileGenerator.NewLine);
Content.Append(" debugDocumentVersioning = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildableProductRunnable" + ProjectFileGenerator.NewLine);
Content.Append(" runnableDebuggingMode = \"0\">" + ProjectFileGenerator.NewLine);
Content.Append(" <BuildableReference" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableIdentifier = \"primary\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintIdentifier = \"" + TargetGuid + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" BuildableName = \"" + TargetName + ".app\"" + ProjectFileGenerator.NewLine);
Content.Append(" BlueprintName = \"" + TargetName + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" ReferencedContainer = \"container:" + TargetName + ".xcodeproj\">" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildableReference>" + ProjectFileGenerator.NewLine);
Content.Append(" </BuildableProductRunnable>" + ProjectFileGenerator.NewLine);
Content.Append(" </ProfileAction>" + ProjectFileGenerator.NewLine);
Content.Append(" <AnalyzeAction" + ProjectFileGenerator.NewLine);
Content.Append(" buildConfiguration = \"" + DefaultConfiguration + "\">" + ProjectFileGenerator.NewLine);
Content.Append(" </AnalyzeAction>" + ProjectFileGenerator.NewLine);
Content.Append(" <ArchiveAction" + ProjectFileGenerator.NewLine);
Content.Append(" buildConfiguration = \"" + DefaultConfiguration + "\"" + ProjectFileGenerator.NewLine);
Content.Append(" revealArchiveInOrganizer = \"YES\">" + ProjectFileGenerator.NewLine);
Content.Append(" </ArchiveAction>" + ProjectFileGenerator.NewLine);
Content.Append("</Scheme>" + ProjectFileGenerator.NewLine);
DirectoryReference SchemesDir = new DirectoryReference(ProjectFilePath.FullName + "/xcshareddata/xcschemes");
if (!SchemesDir.Exists())
{
SchemesDir.CreateDirectory();
}
string SchemeFilePath = SchemesDir + "/" + TargetName + ".xcscheme";
File.WriteAllText(SchemeFilePath, Content.ToString(), new UTF8Encoding());
Content.Clear();
Content.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + ProjectFileGenerator.NewLine);
Content.Append("<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">" + ProjectFileGenerator.NewLine);
Content.Append("<plist version=\"1.0\">" + ProjectFileGenerator.NewLine);
Content.Append("<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t<key>SchemeUserState</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<key>" + TargetName + ".xcscheme_^#shared#^_</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<key>orderHint</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<integer>" + SchemeOrderHint.ToString() + "</integer>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t</dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t</dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t<key>SuppressBuildableAutocreation</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<key>" + TargetGuid + "</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<key>primary</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<true/>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t</dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<key>" + BuildTargetGuid + "</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<key>primary</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<true/>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t</dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<key>" + IndexTargetGuid + "</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t<dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<key>primary</key>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t\t<true/>" + ProjectFileGenerator.NewLine);
Content.Append("\t\t</dict>" + ProjectFileGenerator.NewLine);
Content.Append("\t</dict>" + ProjectFileGenerator.NewLine);
Content.Append("</dict>" + ProjectFileGenerator.NewLine);
Content.Append("</plist>" + ProjectFileGenerator.NewLine);
DirectoryReference ManagementFileDir = new DirectoryReference(ProjectFilePath.FullName + "/xcuserdata/" + Environment.UserName + ".xcuserdatad/xcschemes");
if (!ManagementFileDir.Exists())
{
ManagementFileDir.CreateDirectory();
}
string ManagementFilePath = ManagementFileDir + "/xcschememanagement.plist";
File.WriteAllText(ManagementFilePath, Content.ToString(), new UTF8Encoding());
SchemeOrderHint++;
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
/// <returns>True if the task succeeded</returns>
public override bool Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the project path, and check it exists
FileReference ProjectFile = null;
if (Parameters.Project != null)
{
ProjectFile = ResolveFile(Parameters.Project);
if (!FileReference.Exists(ProjectFile))
{
CommandUtils.LogError("Couldn't find project '{0}'", ProjectFile.FullName);
return(false);
}
}
// Get the directories used for staging this project
DirectoryReference SourceEngineDir = CommandUtils.EngineDirectory;
DirectoryReference SourceProjectDir = (ProjectFile == null)? SourceEngineDir : ProjectFile.Directory;
// Get the output directories. We flatten the directory structure on output.
DirectoryReference TargetDir = ResolveDirectory(Parameters.ToDir);
DirectoryReference TargetEngineDir = DirectoryReference.Combine(TargetDir, "Engine");
DirectoryReference TargetProjectDir = DirectoryReference.Combine(TargetDir, ProjectFile.GetFileNameWithoutExtension());
// Get the path to the receipt
FileReference ReceiptFileName = TargetReceipt.GetDefaultPath(SourceProjectDir, Parameters.Target, Parameters.Platform, Parameters.Configuration, Parameters.Architecture);
// Try to load it
TargetReceipt Receipt;
if (!TargetReceipt.TryRead(ReceiptFileName, SourceEngineDir, SourceProjectDir, out Receipt))
{
CommandUtils.LogError("Couldn't read receipt '{0}'", ReceiptFileName);
return(false);
}
// Stage all the build products needed at runtime
HashSet <FileReference> SourceFiles = new HashSet <FileReference>();
foreach (BuildProduct BuildProduct in Receipt.BuildProducts.Where(x => x.Type != BuildProductType.StaticLibrary && x.Type != BuildProductType.ImportLibrary))
{
SourceFiles.Add(BuildProduct.Path);
}
foreach (RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies.Where(x => x.Type != StagedFileType.UFS))
{
SourceFiles.Add(RuntimeDependency.Path);
}
// Get all the target files
List <FileReference> TargetFiles = new List <FileReference>();
foreach (FileReference SourceFile in SourceFiles)
{
// Get the destination file to copy to, mapping to the new engine and project directories as appropriate
FileReference TargetFile;
if (SourceFile.IsUnderDirectory(SourceEngineDir))
{
TargetFile = FileReference.Combine(TargetEngineDir, SourceFile.MakeRelativeTo(SourceEngineDir));
}
else
{
TargetFile = FileReference.Combine(TargetProjectDir, SourceFile.MakeRelativeTo(SourceProjectDir));
}
// Fixup the case of the output file. Would expect Platform.DeployLowerCaseFilenames() to return true here, but seems not to be the case.
if (Parameters.Platform == UnrealTargetPlatform.PS4)
{
TargetFile = FileReference.Combine(TargetDir, TargetFile.MakeRelativeTo(TargetDir).ToLowerInvariant());
}
// Only copy the output file if it doesn't already exist. We can stage multiple targets to the same output directory.
if (Parameters.Overwrite || !FileReference.Exists(TargetFile))
{
DirectoryReference.CreateDirectory(TargetFile.Directory);
CommandUtils.CopyFile(SourceFile.FullName, TargetFile.FullName);
}
// Add it to the list of target files
TargetFiles.Add(TargetFile);
}
// Apply the optional tag to the build products
foreach (string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFiles);
}
// Add the target file to the list of build products
BuildProducts.UnionWith(TargetFiles);
return(true);
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if(PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!PluginFile.Exists())
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (PackageDir.Exists())
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
PackageDir.CreateDirectory();
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!SourceFilterFile.Exists())
{
List<string> Lines = new List<string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
SourceFilterFile.Directory.CreateDirectory();
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.Log("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile, false);
// Compile the plugin for all the target platforms
List<UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List<UnrealTargetPlatform>() : new List<UnrealTargetPlatform> { BuildHostPlatform.Current.Platform };
List<UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform).Where(x => IsCodeTargetPlatform(BuildHostPlatform.Current.Platform, x)).ToList();
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, "");
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir);
// Remove the host project
if(!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
public override void ExecuteBuild()
{
// Get the plugin filename
string PluginParam = ParseParamValue("Plugin");
if (PluginParam == null)
{
throw new AutomationException("Missing -Plugin=... argument");
}
// Check it exists
FileReference PluginFile = new FileReference(PluginParam);
if (!FileReference.Exists(PluginFile))
{
throw new AutomationException("Plugin '{0}' not found", PluginFile.FullName);
}
// Get the output directory
string PackageParam = ParseParamValue("Package");
if (PackageParam == null)
{
throw new AutomationException("Missing -Package=... argument");
}
// Option for verifying that all include directive s
bool bStrictIncludes = ParseParam("StrictIncludes");
// Whether to use VS2019 for compiling all targets. By default, we currently use 2017 for compiling static libraries for maximum compatibility.
bool bVS2019 = ParseParam("VS2019");
// Make sure the packaging directory is valid
DirectoryReference PackageDir = new DirectoryReference(PackageParam);
if (PluginFile.IsUnderDirectory(PackageDir))
{
throw new AutomationException("Packaged plugin output directory must be different to source");
}
if (PackageDir.IsUnderDirectory(DirectoryReference.Combine(CommandUtils.RootDirectory, "Engine")))
{
throw new AutomationException("Output directory for packaged plugin must be outside engine directory");
}
// Clear the output directory of existing stuff
if (DirectoryReference.Exists(PackageDir))
{
CommandUtils.DeleteDirectoryContents(PackageDir.FullName);
}
else
{
DirectoryReference.CreateDirectory(PackageDir);
}
// Create a placeholder FilterPlugin.ini with instructions on how to use it
FileReference SourceFilterFile = FileReference.Combine(PluginFile.Directory, "Config", "FilterPlugin.ini");
if (!FileReference.Exists(SourceFilterFile))
{
List <string> Lines = new List <string>();
Lines.Add("[FilterPlugin]");
Lines.Add("; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and");
Lines.Add("; may include \"...\", \"*\", and \"?\" wildcards to match directories, files, and individual characters respectively.");
Lines.Add(";");
Lines.Add("; Examples:");
Lines.Add("; /README.txt");
Lines.Add("; /Extras/...");
Lines.Add("; /Binaries/ThirdParty/*.dll");
DirectoryReference.CreateDirectory(SourceFilterFile.Directory);
CommandUtils.WriteAllLines_NoExceptions(SourceFilterFile.FullName, Lines.ToArray());
}
// Create a host project for the plugin. For script generator plugins, we need to have UHT be able to load it, which can only happen if it's enabled in a project.
FileReference HostProjectFile = FileReference.Combine(PackageDir, "HostProject", "HostProject.uproject");
FileReference HostProjectPluginFile = CreateHostProject(HostProjectFile, PluginFile);
// Read the plugin
CommandUtils.LogInformation("Reading plugin from {0}...", HostProjectPluginFile);
PluginDescriptor Plugin = PluginDescriptor.FromFile(HostProjectPluginFile);
// Get the arguments for the compile
StringBuilder AdditionalArgs = new StringBuilder();
if (bStrictIncludes)
{
CommandUtils.LogInformation("Building with precompiled headers and unity disabled");
AdditionalArgs.Append(" -NoPCH -NoSharedPCH -DisableUnity");
}
// Compile the plugin for all the target platforms
List <UnrealTargetPlatform> HostPlatforms = ParseParam("NoHostPlatform")? new List <UnrealTargetPlatform>() : new List <UnrealTargetPlatform> {
BuildHostPlatform.Current.Platform
};
List <UnrealTargetPlatform> TargetPlatforms = GetTargetPlatforms(this, BuildHostPlatform.Current.Platform);
FileReference[] BuildProducts = CompilePlugin(HostProjectFile, HostProjectPluginFile, Plugin, HostPlatforms, TargetPlatforms, AdditionalArgs.ToString(), bVS2019);
// Package up the final plugin data
PackagePlugin(HostProjectPluginFile, BuildProducts, PackageDir, ParseParam("unversioned"));
// Remove the host project
if (!ParseParam("NoDeleteHostProject"))
{
CommandUtils.DeleteDirectory(HostProjectFile.Directory.FullName);
}
}
public static void TakeLock(DirectoryReference LockDirectory, TimeSpan Timeout, System.Action Callback)
{
string LockFilePath = Path.Combine(LockDirectory.FullName, ".lock");
FileStream Stream = null;
DateTime StartTime = DateTime.Now;
DateTime Deadline = StartTime.Add(Timeout);
try
{
LockDirectory.CreateDirectory();
for (int Iterations = 0; ; ++Iterations)
{
// Attempt to create the lock file. Ignore any IO exceptions. Stream will be null if this fails.
try { Stream = new FileStream(LockFilePath, FileMode.Create, FileAccess.Write, FileShare.Read, 4096, FileOptions.DeleteOnClose); }
catch (IOException) { }
if (Stream != null)
{
// If we have a stream, we've taken the lock.
try
{
// Write the machine name to the file.
Stream.Write(Encoding.UTF8.GetBytes(Environment.MachineName));
Stream.Flush();
break;
}
catch
{
throw new AutomationException("Failed to write to the lock file '{0}'.", LockFilePath);
}
}
// We've failed to take the lock. Throw an exception if the timeout has elapsed.
// Otherwise print a log message and retry.
var CurrentTime = DateTime.Now;
if (CurrentTime >= Deadline)
{
throw new AutomationException("Couldn't create lock file '{0}' after {1} seconds.", LockFilePath, CurrentTime.Subtract(StartTime).TotalSeconds);
}
if (Iterations == 0)
{
CommandUtils.Log("Waiting for lock file '{0}' to be removed...", LockFilePath);
}
else if ((Iterations % 30) == 0)
{
CommandUtils.LogWarning("Still waiting for lock file '{0}' after {1} seconds.", LockFilePath, CurrentTime.Subtract(StartTime).TotalSeconds);
}
// Wait for a while before retrying.
Thread.Sleep(1000);
}
// Invoke the user callback now that we own the lock.
Callback();
}
finally
{
// Always dispose the lock file stream if we took the lock.
// The file will delete on close.
if (Stream != null)
{
Stream.Dispose();
Stream = null;
}
}
}
//#nv begin #Blast Linux build
void StageBootstrapExecutable(DeploymentContext SC, StageTarget Target, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments) //@third party code - NVSTUDIOS Set LD_LIBRARY_PATH
//nv end
{
// create a temp script file location
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, ExeName);
DirectoryReference.CreateDirectory(IntermediateDir);
// make sure slashes are good
StagedRelativeTargetPath = StagedRelativeTargetPath.Replace("\\", "/");
// make contents
StringBuilder Script = new StringBuilder();
string EOL = "\n";
Script.Append("#!/bin/sh" + EOL);
// allow running from symlinks
Script.AppendFormat("UE4_TRUE_SCRIPT_NAME=$(echo \\\"$0\\\" | xargs readlink -f)" + EOL);
Script.AppendFormat("UE4_PROJECT_ROOT=$(dirname \"$UE4_TRUE_SCRIPT_NAME\")" + EOL);
Script.AppendFormat("chmod +x \"$UE4_PROJECT_ROOT/{0}\"" + EOL, StagedRelativeTargetPath);
//#nv begin #Blast Linux build
//The Blast .so files are not loaded by dlopen so we we need to setup the search paths
//Really UE should be doing this for all dependent libraries, but they usually statically link
HashSet <string> LDLibraryPaths = new HashSet <string>();
DirectoryReference TargetFileDir = (new FileReference(TargetFile)).Directory;
DirectoryReference SourceEngineDir = SC.LocalRoot;
DirectoryReference SourceProjectDir = SC.ProjectRoot;
foreach (var RuntimeDependency in Target.Receipt.RuntimeDependencies)
{
foreach (FileReference File in CommandUtils.ResolveFilespec(CommandUtils.RootDirectory, RuntimeDependency.Path.FullName, new string[] { }))
{
if (FileReference.Exists(File) && File.GetExtension().Equals(".so", StringComparison.OrdinalIgnoreCase))
{
string FileRelativePath = null;
DirectoryReference SharedLibFolder = File.Directory;
if (SharedLibFolder.IsUnderDirectory(SourceProjectDir))
{
FileRelativePath = Path.Combine(SharedLibFolder.MakeRelativeTo(SourceProjectDir), SC.RelativeProjectRootForStage.ToString());
}
else if (SharedLibFolder.IsUnderDirectory(SourceEngineDir))
{
FileRelativePath = SharedLibFolder.MakeRelativeTo(SourceEngineDir);
}
if (FileRelativePath != null)
{
FileRelativePath = Path.Combine("$UE4_PROJECT_ROOT", FileRelativePath);
FileRelativePath = FileRelativePath.Replace("\\", "/");
//Escape spaces
FileRelativePath = FileRelativePath.Replace(" ", @"\ ");
LDLibraryPaths.Add(FileRelativePath);
}
}
}
}
if (LDLibraryPaths.Count > 0)
{
Script.AppendFormat("export LD_LIBRARY_PATH={0}" + EOL, string.Join(":", LDLibraryPaths));
}
//nv end
Script.AppendFormat("\"$UE4_PROJECT_ROOT/{0}\" {1} $@ " + EOL, StagedRelativeTargetPath, StagedArguments);
// write out the
FileReference.WriteAllText(IntermediateFile, Script.ToString());
if (Utils.IsRunningOnMono)
{
var Result = CommandUtils.Run("sh", string.Format("-c 'chmod +x \\\"{0}\\\"'", IntermediateFile));
if (Result.ExitCode != 0)
{
throw new AutomationException(string.Format("Failed to chmod \"{0}\"", IntermediateFile));
}
}
SC.StageFile(StagedFileType.NonUFS, IntermediateFile, new StagedFileReference(ExeName));
}
/// <summary>
/// Gather compile time telemetry for the given files
/// </summary>
/// <param name="FileToCompileEnvironment">Mapping of source file to the environment used to compile it</param>
/// <param name="WorkingDir">The working directory for output files</param>
/// <param name="NumSamples">Number of samples to take</param>
/// <param name="MaxParallel">Maximum number of tasks to run in parallel.</param>
/// <param name="Log">Log writer</param>
public static void Generate(DirectoryReference InputDir, DirectoryReference WorkingDir, Dictionary <SourceFile, CompileEnvironment> FileToCompileEnvironment, int NumSamples, int Shard, int NumShards, int MaxParallel, LineBasedTextWriter Log)
{
Stopwatch Timer = Stopwatch.StartNew();
// Create an intermediate directory
DirectoryReference IntermediateDir = DirectoryReference.Combine(WorkingDir, "Timing");
IntermediateDir.CreateDirectory();
// Map of unique fragment to timing data
Dictionary <SourceFragment, FragmentTimingData> FragmentToTimingData = new Dictionary <SourceFragment, FragmentTimingData>();
// Map of unique fragment key to timing data
Dictionary <string, FragmentTimingData> DigestToTimingData = new Dictionary <string, FragmentTimingData>();
// List of all the sequences to time
HashSet <string> UniqueNames = new HashSet <string>();
foreach (KeyValuePair <SourceFile, CompileEnvironment> Pair in FileToCompileEnvironment)
{
// Find all the fragments in this file
List <SourceFragment> Fragments = new List <SourceFragment>();
List <Tuple <int, SourceFile> > IncludeHistory = new List <Tuple <int, SourceFile> >();
Pair.Key.FindIncludedFragments(Fragments, IncludeHistory, new HashSet <SourceFile>());
// Create a sequence for each unique fragment
FragmentTimingData PrevTimingData = null;
for (int Idx = 0; Idx < Fragments.Count; Idx++)
{
FragmentTimingData TimingData = null;
if (!FragmentToTimingData.ContainsKey(Fragments[Idx]) || (Idx + 1 < Fragments.Count && !FragmentToTimingData.ContainsKey(Fragments[Idx + 1])))
{
// Create a sequence for this fragment
SourceFragment LastFragment = Fragments[Idx];
// Create a unique key for this sequence by concatenating all the fragment names
string Digest = Utility.ComputeDigest(String.Join("\n", Fragments.Take(Idx + 1).Select(x => x.Location.FullName)));
// Try to get an existing sequence for this key, otherwise create a new one;
if (!DigestToTimingData.TryGetValue(Digest, out TimingData))
{
// Find a unique name for this sequence
string UniqueName = LastFragment.Location.GetFileName();
for (int NameIdx = 2; !UniqueNames.Add(UniqueName); NameIdx++)
{
UniqueName = String.Format("{0}_{1}{2}", LastFragment.Location.GetFileNameWithoutExtension(), NameIdx, LastFragment.Location.GetExtension());
}
// Add the object for this sequence
FileReference IntermediateFile = FileReference.Combine(IntermediateDir, UniqueName);
TimingData = new FragmentTimingData(UniqueName, Digest, PrevTimingData, Fragments.Take(Idx + 1).ToArray(), IncludeHistory, IntermediateFile, Pair.Value);
DigestToTimingData.Add(Digest, TimingData);
}
// Add it to the unique mapping of fragments
if (!FragmentToTimingData.ContainsKey(LastFragment))
{
FragmentToTimingData[LastFragment] = TimingData;
}
}
PrevTimingData = TimingData;
}
}
// Read any existing shard timing data in the output folder
foreach (FileReference IntermediateFile in IntermediateDir.EnumerateFileReferences("*.csv"))
{
string[] Lines = File.ReadAllLines(IntermediateFile.FullName);
foreach (string Line in Lines.Skip(1))
{
string[] Tokens = Line.Split(',');
if (Tokens.Length == 5)
{
FragmentTimingData TimingData;
if (DigestToTimingData.TryGetValue(Tokens[1], out TimingData) && TimingData.Samples.Count < NumSamples)
{
FragmentTimingSample Sample = new FragmentTimingSample();
Sample.TotalTime = Double.Parse(Tokens[2]);
Sample.FrontendTime = Double.Parse(Tokens[3]);
Sample.BackendTime = Double.Parse(Tokens[4]);
TimingData.Samples.Add(Sample);
}
}
}
}
// Find all the remaining fragments, and repeat each one by the number of times it has to be executed
List <FragmentTimingData> FilteredFragments = DigestToTimingData.Values.ToList();
FilteredFragments.RemoveAll(x => (int)(Math.Abs((long)x.Digest.GetHashCode()) % NumShards) != (Shard - 1));
// Get the initial number of compile times for each fragment. We avoid saving before this number.
List <int> InitialCompileCount = FilteredFragments.Select(x => x.Samples.Count).ToList();
// Create all the actions to execute
List <Action> Actions = new List <Action>();
foreach (FragmentTimingData Fragment in FilteredFragments)
{
FragmentTimingData FragmentCopy = Fragment;
for (int SampleIdx = Fragment.Samples.Count; SampleIdx < NumSamples; SampleIdx++)
{
int SampleIdxCopy = SampleIdx;
Actions.Add(() => FragmentCopy.Compile(IntermediateDir, SampleIdxCopy));
}
}
// Randomize the order to ensure that compile times are not consistently affected by other files being compiled simultaneously.
Random Random = new Random();
Actions = Actions.OrderBy(x => Random.Next()).ToList();
// Compile them all
if (Actions.Count > 0)
{
Utility.ParallelForWithStatus("Compiling fragments...", 0, Actions.Count, new ParallelOptions {
MaxDegreeOfParallelism = MaxParallel
}, Idx => Actions[Idx](), Log);
}
// Write out the results
if (NumShards > 1)
{
// If we're running a sharded build, write out intermediate files containing the results
FileReference OutputFile = FileReference.Combine(IntermediateDir, String.Format("Shard{0}.csv", Shard));
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
Writer.WriteLine("Name,Digest,TotalTime,FrontendTime,BackendTime");
for (int Idx = 0; Idx < FilteredFragments.Count; Idx++)
{
FragmentTimingData FilteredFragment = FilteredFragments[Idx];
for (int SampleIdx = InitialCompileCount[Idx]; SampleIdx < FilteredFragment.Samples.Count; SampleIdx++)
{
FragmentTimingSample Sample = FilteredFragment.Samples[SampleIdx];
Writer.WriteLine("{0},{1},{2},{3},{4}", FilteredFragment.UniqueName, FilteredFragment.Digest, Sample.TotalTime, Sample.FrontendTime, Sample.BackendTime);
}
}
}
}
else
{
// Write out the fragment report
FileReference FragmentReport = FileReference.Combine(WorkingDir, "Timing.csv");
Log.WriteLine("Writing {0}...", FragmentReport);
using (StreamWriter Writer = new StreamWriter(FragmentReport.FullName))
{
// Write the header
Writer.Write("Fragment,MinLine,MaxLine");
// Write the labels for each sample type
string[] Types = new string[] { "Total", "Frontend", "Backend" };
for (int Idx = 0; Idx < Types.Length; Idx++)
{
for (int SampleIdx = 0; SampleIdx < NumSamples; SampleIdx++)
{
Writer.Write(",{0}{1}", Types[Idx], SampleIdx + 1);
}
Writer.Write(",{0}Min,{0}Max,{0}Avg,{0}Exc", Types[Idx]);
}
Writer.WriteLine();
// Write all the results
Func <FragmentTimingSample, double>[] TimeFieldDelegates = new Func <FragmentTimingSample, double>[] { x => x.TotalTime, x => x.FrontendTime, x => x.BackendTime };
foreach (FragmentTimingData TimingData in FragmentToTimingData.Values)
{
Writer.Write("{0},{1},{2}", TimingData.Fragment.Location.GetFileName(), TimingData.Fragment.MarkupMin + 1, TimingData.Fragment.MarkupMax + 1);
foreach (Func <FragmentTimingSample, double> TimeFieldDelegate in TimeFieldDelegates)
{
foreach (FragmentTimingSample Sample in TimingData.Samples)
{
Writer.Write(",{0:0.000}", TimeFieldDelegate(Sample));
}
Writer.Write(",{0:0.000}", TimingData.Samples.Min(x => TimeFieldDelegate(x)));
Writer.Write(",{0:0.000}", TimingData.Samples.Max(x => TimeFieldDelegate(x)));
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)));
if (TimingData.PrevFragmentData == null)
{
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)));
}
else
{
Writer.Write(",{0:0.000}", TimingData.Samples.Average(x => TimeFieldDelegate(x)) - TimingData.PrevFragmentData.Samples.Average(x => TimeFieldDelegate(x)));
}
}
Writer.WriteLine();
}
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet<FileReference> BuildProducts, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Parse all the source patterns
FilePattern SourcePattern = new FilePattern(CommandUtils.RootDirectory, Parameters.From);
// Parse the target pattern
FilePattern TargetPattern = new FilePattern(CommandUtils.RootDirectory, Parameters.To);
// Apply the filter to the source files
HashSet<FileReference> Files = null;
if(!String.IsNullOrEmpty(Parameters.Files))
{
SourcePattern = SourcePattern.AsDirectoryPattern();
Files = ResolveFilespec(SourcePattern.BaseDirectory, Parameters.Files, TagNameToFileSet);
}
// Build the file mapping
Dictionary<FileReference, FileReference> TargetFileToSourceFile = FilePattern.CreateMapping(Files, SourcePattern, TargetPattern);
// Check we got some files
if(TargetFileToSourceFile.Count == 0)
{
CommandUtils.Log("No files found matching '{0}'", SourcePattern);
return;
}
// If the target is on a network share, retry creating the first directory until it succeeds
DirectoryReference FirstTargetDirectory = TargetFileToSourceFile.First().Key.Directory;
if(!DirectoryReference.Exists(FirstTargetDirectory))
{
const int MaxNumRetries = 15;
for(int NumRetries = 0;;NumRetries++)
{
try
{
DirectoryReference.CreateDirectory(FirstTargetDirectory);
if(NumRetries == 1)
{
Log.TraceInformation("Created target directory {0} after 1 retry.", FirstTargetDirectory);
}
else if(NumRetries > 1)
{
Log.TraceInformation("Created target directory {0} after {1} retries.", FirstTargetDirectory, NumRetries);
}
break;
}
catch(Exception Ex)
{
if(NumRetries == 0)
{
Log.TraceInformation("Unable to create directory '{0}' on first attempt. Retrying {1} times...", FirstTargetDirectory, MaxNumRetries);
}
Log.TraceLog(" {0}", Ex);
if(NumRetries >= 15)
{
throw new AutomationException(Ex, "Unable to create target directory '{0}' after {1} retries.", FirstTargetDirectory, NumRetries);
}
Thread.Sleep(2000);
}
}
}
// Copy them all
KeyValuePair<FileReference, FileReference>[] FilePairs = TargetFileToSourceFile.ToArray();
CommandUtils.Log("Copying {0} file{1} from {2} to {3}...", FilePairs.Length, (FilePairs.Length == 1)? "" : "s", SourcePattern.BaseDirectory, TargetPattern.BaseDirectory);
CommandUtils.ThreadedCopyFiles(FilePairs.Select(x => x.Value.FullName).ToList(), FilePairs.Select(x => x.Key.FullName).ToList(), bQuiet: true);
// Update the list of build products
BuildProducts.UnionWith(TargetFileToSourceFile.Keys);
// Apply the optional output tag to them
foreach(string TagName in FindTagNamesFromList(Parameters.Tag))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(TargetFileToSourceFile.Keys);
}
}
