/// <summary>
/// Download multiple files from the remote Mac
/// </summary>
/// <param name="Files">List of local files to download</param>
void DownloadFiles(IEnumerable <FileReference> Files)
{
List <FileReference>[] FileGroups = new List <FileReference> [Mappings.Count];
for (int Idx = 0; Idx < Mappings.Count; Idx++)
{
FileGroups[Idx] = new List <FileReference>();
}
foreach (FileReference File in Files)
{
int MappingIdx = Mappings.FindIndex(x => File.IsUnderDirectory(x.LocalDirectory));
if (MappingIdx == -1)
{
throw new BuildException("File for download '{0}' is not under the engine or project directory.", File);
}
FileGroups[MappingIdx].Add(File);
}
for (int Idx = 0; Idx < Mappings.Count; Idx++)
{
if (FileGroups[Idx].Count > 0)
{
FileReference DownloadListLocation = FileReference.Combine(UnrealBuildTool.EngineDirectory, "Intermediate", "Rsync", "Download.txt");
DirectoryReference.CreateDirectory(DownloadListLocation.Directory);
FileReference.WriteAllLines(DownloadListLocation, FileGroups[Idx].Select(x => x.MakeRelativeTo(Mappings[Idx].LocalDirectory).Replace('\\', '/')));
List <string> Arguments = new List <string>(CommonRsyncArguments);
Arguments.Add(String.Format("--files-from=\"{0}\"", GetLocalCygwinPath(DownloadListLocation)));
Arguments.Add(String.Format("\"{0}@{1}\":'{2}/'", UserName, ServerName, Mappings[Idx].RemoteDirectory));
Arguments.Add(String.Format("\"{0}/\"", GetLocalCygwinPath(Mappings[Idx].LocalDirectory)));
int Result = Rsync(String.Join(" ", Arguments));
if (Result != 0)
{
throw new BuildException("Unable to download files from remote Mac (exit code {0})", Result);
}
}
}
}
/// <summary>
/// Patches a set of actions for use with live coding. The new action list will output object files to a different location.
/// </summary>
/// <param name="Actions">Set of actions</param>
/// <param name="OriginalFileToPatchedFile">Dictionary that receives a map of original object file to patched object file</param>
public static void PatchActionGraphForLiveCoding(IEnumerable <Action> Actions, Dictionary <FileReference, FileReference> OriginalFileToPatchedFile)
{
foreach (Action Action in Actions)
{
if (Action.ActionType == ActionType.Compile)
{
if (!Action.CommandPath.GetFileName().Equals("cl-filter.exe", StringComparison.OrdinalIgnoreCase))
{
throw new BuildException("Unable to patch action graph - unexpected executable in compile action ({0})", Action.CommandPath);
}
List <string> Arguments = Utils.ParseArgumentList(Action.CommandArguments);
// Find the index of the cl-filter argument delimiter
int DelimiterIdx = Arguments.IndexOf("--");
if (DelimiterIdx == -1)
{
throw new BuildException("Unable to patch action graph - missing '--' delimiter to cl-filter");
}
// Fix the dependencies path
const string DependenciesPrefix = "-dependencies=";
int DependenciesIdx = 0;
for (;; DependenciesIdx++)
{
if (DependenciesIdx == DelimiterIdx)
{
throw new BuildException("Unable to patch action graph - missing '{0}' argument to cl-filter", DependenciesPrefix);
}
else if (Arguments[DependenciesIdx].StartsWith(DependenciesPrefix, StringComparison.OrdinalIgnoreCase))
{
break;
}
}
FileReference OldDependenciesFile = new FileReference(Arguments[DependenciesIdx].Substring(DependenciesPrefix.Length));
FileItem OldDependenciesFileItem = Action.ProducedItems.First(x => x.Location == OldDependenciesFile);
Action.ProducedItems.Remove(OldDependenciesFileItem);
FileReference NewDependenciesFile = OldDependenciesFile.ChangeExtension(".lc.response");
FileItem NewDependenciesFileItem = FileItem.GetItemByFileReference(NewDependenciesFile);
Action.ProducedItems.Add(NewDependenciesFileItem);
Arguments[DependenciesIdx] = DependenciesPrefix + NewDependenciesFile.FullName;
// Fix the response file
int ResponseFileIdx = DelimiterIdx + 1;
for (; ; ResponseFileIdx++)
{
if (ResponseFileIdx == Arguments.Count)
{
throw new BuildException("Unable to patch action graph - missing response file argument to cl-filter");
}
else if (Arguments[ResponseFileIdx].StartsWith("@", StringComparison.Ordinal))
{
break;
}
}
FileReference OldResponseFile = new FileReference(Arguments[ResponseFileIdx].Substring(1));
FileReference NewResponseFile = new FileReference(OldResponseFile.FullName + ".lc");
const string OutputFilePrefix = "/Fo";
string[] ResponseLines = FileReference.ReadAllLines(OldResponseFile);
for (int Idx = 0; Idx < ResponseLines.Length; Idx++)
{
string ResponseLine = ResponseLines[Idx];
if (ResponseLine.StartsWith(OutputFilePrefix, StringComparison.Ordinal))
{
FileReference OldOutputFile = new FileReference(ResponseLine.Substring(3).Trim('\"'));
FileItem OldOutputFileItem = Action.ProducedItems.First(x => x.Location == OldOutputFile);
Action.ProducedItems.Remove(OldOutputFileItem);
FileReference NewOutputFile = OldOutputFile.ChangeExtension(".lc.obj");
FileItem NewOutputFileItem = FileItem.GetItemByFileReference(NewOutputFile);
Action.ProducedItems.Add(NewOutputFileItem);
OriginalFileToPatchedFile[OldOutputFile] = NewOutputFile;
ResponseLines[Idx] = OutputFilePrefix + "\"" + NewOutputFile.FullName + "\"";
break;
}
}
FileReference.WriteAllLines(NewResponseFile, ResponseLines);
Arguments[ResponseFileIdx] = "@" + NewResponseFile.FullName;
// Update the final arguments
Action.CommandArguments = Utils.FormatCommandLine(Arguments);
}
}
}
/// <summary>
/// Dynamically compiles an assembly for the specified source file and loads that assembly into the application's
/// current domain. If an assembly has already been compiled and is not out of date, then it will be loaded and
/// no compilation is necessary.
/// </summary>
/// <param name="OutputAssemblyPath">Full path to the assembly to be created</param>
/// <param name="SourceFileNames">List of source file name</param>
/// <param name="ReferencedAssembies"></param>
/// <param name="PreprocessorDefines"></param>
/// <param name="DoNotCompile"></param>
/// <param name="TreatWarningsAsErrors"></param>
/// <returns>The assembly that was loaded</returns>
public static Assembly CompileAndLoadAssembly(FileReference OutputAssemblyPath, HashSet <FileReference> SourceFileNames, List <string> ReferencedAssembies = null, List <string> PreprocessorDefines = null, bool DoNotCompile = false, bool TreatWarningsAsErrors = false)
{
// Check to see if the resulting assembly is compiled and up to date
FileReference AssemblySourcesListFilePath = FileReference.Combine(OutputAssemblyPath.Directory, Path.GetFileNameWithoutExtension(OutputAssemblyPath.FullName) + "SourceFiles.txt");
bool bNeedsCompilation = false;
if (!DoNotCompile)
{
bNeedsCompilation = RequiresCompilation(SourceFileNames, AssemblySourcesListFilePath, OutputAssemblyPath);
}
// Load the assembly to ensure it is correct
Assembly CompiledAssembly = null;
if (!bNeedsCompilation)
{
try
{
// Load the previously-compiled assembly from disk
CompiledAssembly = Assembly.LoadFile(OutputAssemblyPath.FullName);
}
catch (FileLoadException Ex)
{
Log.TraceInformation(String.Format("Unable to load the previously-compiled assembly file '{0}'. Unreal Build Tool will try to recompile this assembly now. (Exception: {1})", OutputAssemblyPath, Ex.Message));
bNeedsCompilation = true;
}
catch (BadImageFormatException Ex)
{
Log.TraceInformation(String.Format("Compiled assembly file '{0}' appears to be for a newer CLR version or is otherwise invalid. Unreal Build Tool will try to recompile this assembly now. (Exception: {1})", OutputAssemblyPath, Ex.Message));
bNeedsCompilation = true;
}
catch (FileNotFoundException)
{
throw new BuildException("Precompiled rules assembly '{0}' does not exist.", OutputAssemblyPath);
}
catch (Exception Ex)
{
throw new BuildException(Ex, "Error while loading previously-compiled assembly file '{0}'. (Exception: {1})", OutputAssemblyPath, Ex.Message);
}
}
// Compile the assembly if me
if (bNeedsCompilation)
{
using (Timeline.ScopeEvent(String.Format("Compiling rules assembly ({0})", OutputAssemblyPath.GetFileName())))
{
CompiledAssembly = CompileAssembly(OutputAssemblyPath, SourceFileNames, ReferencedAssembies, PreprocessorDefines, TreatWarningsAsErrors);
}
// Save out a list of all the source files we compiled. This is so that we can tell if whole files were added or removed
// since the previous time we compiled the assembly. In that case, we'll always want to recompile it!
FileReference.WriteAllLines(AssemblySourcesListFilePath, SourceFileNames.Select(x => x.FullName));
}
#if !NET_CORE
// Load the assembly into our app domain
try
{
AppDomain.CurrentDomain.Load(CompiledAssembly.GetName());
}
catch (Exception Ex)
{
throw new BuildException(Ex, "Unable to load the compiled build assembly '{0}' into our application's domain. (Exception: {1})", OutputAssemblyPath, Ex.Message);
}
#endif
return(CompiledAssembly);
}
/// <summary>
/// Determine what needs to be built for a target
/// </summary>
/// <param name="BuildConfiguration">The build configuration</param>
/// <param name="TargetDescriptor">Target being built</param>
/// <param name="Makefile">Makefile generated for this target</param>
/// <returns>Set of actions to execute</returns>
static HashSet <Action> GetActionsForTarget(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, TargetMakefile Makefile)
{
// Create the action graph
ActionGraph.Link(Makefile.Actions);
// Get the hot-reload mode
HotReloadMode HotReloadMode = TargetDescriptor.HotReloadMode;
if (HotReloadMode == HotReloadMode.Default)
{
if (TargetDescriptor.HotReloadModuleNameToSuffix.Count > 0 && TargetDescriptor.ForeignPlugin == null)
{
HotReloadMode = HotReloadMode.FromEditor;
}
else if (BuildConfiguration.bAllowHotReloadFromIDE && HotReload.ShouldDoHotReloadFromIDE(BuildConfiguration, TargetDescriptor))
{
HotReloadMode = HotReloadMode.FromIDE;
}
else
{
HotReloadMode = HotReloadMode.Disabled;
}
}
// Guard against a live coding session for this target being active
if (HotReloadMode != HotReloadMode.LiveCoding && TargetDescriptor.ForeignPlugin == null && HotReload.IsLiveCodingSessionActive(Makefile))
{
throw new BuildException("Unable to start regular build while Live Coding is active. Press Ctrl+Alt+F11 to trigger a Live Coding compile.");
}
// Get the root prerequisite actions
List <Action> PrerequisiteActions = GatherPrerequisiteActions(TargetDescriptor, Makefile);
// Get the path to the hot reload state file for this target
FileReference HotReloadStateFile = global::UnrealBuildTool.HotReloadState.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, TargetDescriptor.Architecture);
// Apply the previous hot reload state
HotReloadState HotReloadState = null;
if (HotReloadMode == HotReloadMode.Disabled)
{
// Make sure we're not doing a partial build from the editor (eg. compiling a new plugin)
if (TargetDescriptor.ForeignPlugin == null && TargetDescriptor.SingleFileToCompile == null)
{
// Delete the previous state file
HotReload.DeleteTemporaryFiles(HotReloadStateFile);
}
}
else
{
// Read the previous state file and apply it to the action graph
if (FileReference.Exists(HotReloadStateFile))
{
HotReloadState = HotReloadState.Load(HotReloadStateFile);
}
else
{
HotReloadState = new HotReloadState();
}
// Apply the old state to the makefile
HotReload.ApplyState(HotReloadState, Makefile);
// If we want a specific suffix on any modules, apply that now. We'll track the outputs later, but the suffix has to be forced (and is always out of date if it doesn't exist).
HotReload.PatchActionGraphWithNames(PrerequisiteActions, TargetDescriptor.HotReloadModuleNameToSuffix, Makefile);
// Re-link the action graph
ActionGraph.Link(PrerequisiteActions);
}
// Create the dependencies cache
CppDependencyCache CppDependencies;
using (Timeline.ScopeEvent("Reading dependency cache"))
{
CppDependencies = CppDependencyCache.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Configuration, Makefile.TargetType, TargetDescriptor.Architecture);
}
// Create the action history
ActionHistory History;
using (Timeline.ScopeEvent("Reading action history"))
{
History = ActionHistory.CreateHierarchy(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, Makefile.TargetType, TargetDescriptor.Architecture);
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
HashSet <Action> TargetActionsToExecute;
if (TargetDescriptor.SingleFileToCompile == null)
{
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
}
else
{
TargetActionsToExecute = new HashSet <Action>(PrerequisiteActions);
}
// Additional processing for hot reload
if (HotReloadMode == HotReloadMode.LiveCoding)
{
// Make sure we're not overwriting any lazy-loaded modules
if (TargetDescriptor.LiveCodingModules != null)
{
// Read the list of modules that we're allowed to build
string[] Lines = FileReference.ReadAllLines(TargetDescriptor.LiveCodingModules);
// Parse it out into a set of filenames
HashSet <string> AllowedOutputFileNames = new HashSet <string>(FileReference.Comparer);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim();
if (TrimLine.Length > 0)
{
AllowedOutputFileNames.Add(Path.GetFileName(TrimLine));
}
}
// Find all the binaries that we're actually going to build
HashSet <FileReference> OutputFiles = new HashSet <FileReference>();
foreach (Action Action in TargetActionsToExecute)
{
if (Action.ActionType == ActionType.Link)
{
OutputFiles.UnionWith(Action.ProducedItems.Where(x => x.HasExtension(".exe") || x.HasExtension(".dll")).Select(x => x.Location));
}
}
// Find all the files that will be built that aren't allowed
List <FileReference> ProtectedOutputFiles = OutputFiles.Where(x => !AllowedOutputFileNames.Contains(x.GetFileName())).ToList();
if (ProtectedOutputFiles.Count > 0)
{
FileReference.WriteAllLines(new FileReference(TargetDescriptor.LiveCodingModules.FullName + ".out"), ProtectedOutputFiles.Select(x => x.ToString()));
foreach (FileReference ProtectedOutputFile in ProtectedOutputFiles)
{
Log.TraceInformation("Module {0} is not currently enabled for Live Coding", ProtectedOutputFile);
}
throw new CompilationResultException(CompilationResult.Canceled);
}
}
// Filter the prerequisite actions down to just the compile actions, then recompute all the actions to execute
PrerequisiteActions = new List <Action>(TargetActionsToExecute.Where(x => x.ActionType == ActionType.Compile));
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Update the action graph with these new paths
Dictionary <FileReference, FileReference> OriginalFileToPatchedFile = new Dictionary <FileReference, FileReference>();
HotReload.PatchActionGraphForLiveCoding(PrerequisiteActions, OriginalFileToPatchedFile);
// Get a new list of actions to execute now that the graph has been modified
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Output the Live Coding manifest
if (TargetDescriptor.LiveCodingManifest != null)
{
HotReload.WriteLiveCodingManifest(TargetDescriptor.LiveCodingManifest, Makefile.Actions, OriginalFileToPatchedFile);
}
}
else if (HotReloadMode == HotReloadMode.FromEditor || HotReloadMode == HotReloadMode.FromIDE)
{
// Patch action history for hot reload when running in assembler mode. In assembler mode, the suffix on the output file will be
// the same for every invocation on that makefile, but we need a new suffix each time.
// For all the hot-reloadable modules that may need a unique suffix appended, build a mapping from output item to all the output items in that module. We can't
// apply a suffix to one without applying a suffix to all of them.
Dictionary <FileItem, FileItem[]> HotReloadItemToDependentItems = new Dictionary <FileItem, FileItem[]>();
foreach (string HotReloadModuleName in Makefile.HotReloadModuleNames)
{
int ModuleSuffix;
if (!TargetDescriptor.HotReloadModuleNameToSuffix.TryGetValue(HotReloadModuleName, out ModuleSuffix) || ModuleSuffix == -1)
{
FileItem[] ModuleOutputItems;
if (Makefile.ModuleNameToOutputItems.TryGetValue(HotReloadModuleName, out ModuleOutputItems))
{
foreach (FileItem ModuleOutputItem in ModuleOutputItems)
{
HotReloadItemToDependentItems[ModuleOutputItem] = ModuleOutputItems;
}
}
}
}
// Expand the list of actions to execute to include everything that references any files with a new suffix. Unlike a regular build, we can't ignore
// dependencies on import libraries under the assumption that a header would change if the API changes, because the dependency will be on a different DLL.
HashSet <FileItem> FilesRequiringSuffix = new HashSet <FileItem>(TargetActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => HotReloadItemToDependentItems.ContainsKey(x)));
for (int LastNumFilesWithNewSuffix = 0; FilesRequiringSuffix.Count > LastNumFilesWithNewSuffix;)
{
LastNumFilesWithNewSuffix = FilesRequiringSuffix.Count;
foreach (Action PrerequisiteAction in PrerequisiteActions)
{
if (!TargetActionsToExecute.Contains(PrerequisiteAction))
{
foreach (FileItem ProducedItem in PrerequisiteAction.ProducedItems)
{
FileItem[] DependentItems;
if (HotReloadItemToDependentItems.TryGetValue(ProducedItem, out DependentItems))
{
TargetActionsToExecute.Add(PrerequisiteAction);
FilesRequiringSuffix.UnionWith(DependentItems);
}
}
}
}
}
// Build a list of file mappings
Dictionary <FileReference, FileReference> OldLocationToNewLocation = new Dictionary <FileReference, FileReference>();
foreach (FileItem FileRequiringSuffix in FilesRequiringSuffix)
{
FileReference OldLocation = FileRequiringSuffix.Location;
FileReference NewLocation = HotReload.ReplaceSuffix(OldLocation, HotReloadState.NextSuffix);
OldLocationToNewLocation[OldLocation] = NewLocation;
}
// Update the action graph with these new paths
HotReload.PatchActionGraph(PrerequisiteActions, OldLocationToNewLocation);
// Get a new list of actions to execute now that the graph has been modified
TargetActionsToExecute = ActionGraph.GetActionsToExecute(Makefile.Actions, PrerequisiteActions, CppDependencies, History, BuildConfiguration.bIgnoreOutdatedImportLibraries);
// Build a mapping of all file items to their original
Dictionary <FileReference, FileReference> HotReloadFileToOriginalFile = new Dictionary <FileReference, FileReference>();
foreach (KeyValuePair <FileReference, FileReference> Pair in HotReloadState.OriginalFileToHotReloadFile)
{
HotReloadFileToOriginalFile[Pair.Value] = Pair.Key;
}
foreach (KeyValuePair <FileReference, FileReference> Pair in OldLocationToNewLocation)
{
FileReference OriginalLocation;
if (!HotReloadFileToOriginalFile.TryGetValue(Pair.Key, out OriginalLocation))
{
OriginalLocation = Pair.Key;
}
HotReloadFileToOriginalFile[Pair.Value] = OriginalLocation;
}
// Now filter out all the hot reload files and update the state
foreach (Action Action in TargetActionsToExecute)
{
foreach (FileItem ProducedItem in Action.ProducedItems)
{
FileReference OriginalLocation;
if (HotReloadFileToOriginalFile.TryGetValue(ProducedItem.Location, out OriginalLocation))
{
HotReloadState.OriginalFileToHotReloadFile[OriginalLocation] = ProducedItem.Location;
HotReloadState.TemporaryFiles.Add(ProducedItem.Location);
}
}
}
// Increment the suffix for the next iteration
if (TargetActionsToExecute.Count > 0)
{
HotReloadState.NextSuffix++;
}
// Save the new state
HotReloadState.Save(HotReloadStateFile);
// Prevent this target from deploying
Makefile.bDeployAfterCompile = false;
}
return(TargetActionsToExecute);
}
public void Write(FileReference File)
{
Lines[Lines.Count - 1] = Lines[Lines.Count - 1].TrimEnd(',');
FileReference.WriteAllLines(File, Lines.ToArray());
}
