/// <summary>
/// Gets the default location to stage an input file
/// </summary>
/// <param name="InputFile">Location of the file in the file system</param>
/// <returns>Staged file location</returns>
public StagedFileReference GetStagedFileLocation(FileReference InputFile)
{
StagedFileReference OutputFile;
if (InputFile.IsUnderDirectory(ProjectRoot))
{
OutputFile = StagedFileReference.Combine(RelativeProjectRootForStage, InputFile.MakeRelativeTo(ProjectRoot));
}
else if (InputFile.HasExtension(".uplugin"))
{
DirectoryReference EnterpriseRoot = DirectoryReference.Combine(EngineRoot, "..", "Enterprise"); // Enterprise plugins aren't under the project additional plugin directories, so they shouldn't be remapped
if (InputFile.IsUnderDirectory(EngineRoot) || InputFile.IsUnderDirectory(EnterpriseRoot))
{
OutputFile = new StagedFileReference(InputFile.MakeRelativeTo(LocalRoot));
}
else
{
// This is a plugin that lives outside of the Engine/Plugins or Game/Plugins directory so needs to be remapped for staging/packaging
// We need to remap C:\SomePath\PluginName\PluginName.uplugin to RemappedPlugins\PluginName\PluginName.uplugin
OutputFile = new StagedFileReference(String.Format("RemappedPlugins/{0}/{1}", InputFile.GetFileNameWithoutExtension(), InputFile.GetFileName()));
}
}
else if (InputFile.IsUnderDirectory(LocalRoot))
{
OutputFile = new StagedFileReference(InputFile.MakeRelativeTo(LocalRoot));
}
else
{
throw new AutomationException("Can't deploy {0} because it doesn't start with {1} or {2}", InputFile, ProjectRoot, LocalRoot);
}
return(OutputFile);
}
bool IsFileForOtherPlatform(FileReference InputFile)
{
foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
bool bMatchesIniPlatform = (InputFile.HasExtension(".ini") && Plat == StageTargetPlatform.IniPlatformType); // filter ini files for the ini file platform
bool bMatchesTargetPlatform = (Plat == StageTargetPlatform.PlatformType || Plat == UnrealTargetPlatform.Unknown); // filter platform files for the target platform
if (!bMatchesIniPlatform && !bMatchesTargetPlatform)
{
FileSystemName PlatformName = new FileSystemName(Plat.ToString());
if (InputFile.IsUnderDirectory(ProjectRoot))
{
if (InputFile.ContainsName(PlatformName, ProjectRoot))
{
return(true);
}
}
else if (InputFile.IsUnderDirectory(LocalRoot))
{
if (InputFile.ContainsName(PlatformName, LocalRoot))
{
return(true);
}
}
}
}
return(false);
}
/// <summary>
/// Stage a single file to its default location
/// </summary>
/// <param name="FileType">The type of file being staged</param>
/// <param name="InputFile">Path to the file</param>
public void StageFile(StagedFileType FileType, FileReference InputFile)
{
StagedFileReference OutputFile;
if (InputFile.IsUnderDirectory(ProjectRoot))
{
OutputFile = StagedFileReference.Combine(RelativeProjectRootForStage, InputFile.MakeRelativeTo(ProjectRoot));
}
else if (InputFile.HasExtension(".uplugin"))
{
if (InputFile.IsUnderDirectory(EngineRoot))
{
OutputFile = new StagedFileReference(InputFile.MakeRelativeTo(LocalRoot));
}
else
{
// This is a plugin that lives outside of the Engine/Plugins or Game/Plugins directory so needs to be remapped for staging/packaging
// We need to remap C:\SomePath\PluginName\PluginName.uplugin to RemappedPlugins\PluginName\PluginName.uplugin
OutputFile = new StagedFileReference(String.Format("RemappedPlugins/{0}/{1}", InputFile.GetFileNameWithoutExtension(), InputFile.GetFileName()));
}
}
else if (InputFile.IsUnderDirectory(LocalRoot))
{
OutputFile = new StagedFileReference(InputFile.MakeRelativeTo(LocalRoot));
}
else
{
throw new AutomationException("Can't deploy {0} because it doesn't start with {1} or {2}", InputFile, ProjectRoot, LocalRoot);
}
StageFile(FileType, InputFile, OutputFile);
}
/// <summary>
/// Starts compiling the provided project file and returns the process. Caller should check HasExited
/// or call WaitForExit() to get results
/// </summary>
/// <param name="ProjectFile"></param>
/// <returns></returns>
private static bool CompileAutomationProject(FileReference ProjectFile, Dictionary <string, string> Properties)
{
if (!ProjectFile.HasExtension(".csproj"))
{
throw new AutomationException(String.Format("Unable to build Project {0}. Not a valid .csproj file.", ProjectFile));
}
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException(String.Format("Unable to build Project {0}. Project file not found.", ProjectFile));
}
string CmdLine = String.Format("\"{0}\" /verbosity:quiet /nologo /target:Build {1} /p:TreatWarningsAsErrors=false /p:NoWarn=\"612,618,672,1591\" /p:BuildProjectReferences=true",
ProjectFile, GetMsBuildPropertyArguments(Properties));
// Compile the project
IProcessResult Result = CommandUtils.Run(CommandUtils.CmdEnv.MsBuildExe, CmdLine);
if (Result.ExitCode != 0)
{
throw new AutomationException(String.Format("Failed to build \"{0}\":{1}{2}", ProjectFile, Environment.NewLine, Result.Output));
}
else
{
// Remove .Automation.csproj and copy to target dir
Log.TraceVerbose("Successfully compiled {0}", ProjectFile);
}
return(Result.ExitCode == 0);
}
bool IsSymbolFile(FileReference File)
{
if (File.HasExtension(".dll") || File.HasExtension(".exe") || File.HasExtension(".pdb"))
{
return(true);
}
if (File.HasExtension(".dl_") || File.HasExtension(".ex_") || File.HasExtension(".pd_"))
{
return(true);
}
return(false);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}