/// <summary>
/// Clone another compile environment
/// </summary>
/// <param name="Other">The compile environment to copy</param>
public CompileEnvironment(CompileEnvironment Other)
{
Compiler = Other.Compiler;
CompilerType = Other.CompilerType;
Options = new List <CompileOption>(Other.Options);
Definitions = new List <string>(Other.Definitions);
IncludePaths = new List <DirectoryReference>(Other.IncludePaths);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}
/// <summary>
/// Reads an exported XGE task list
/// </summary>
/// <param name="TaskListFile">File to read from</param>
/// <param name="FileToEnvironment">Mapping from source file to compile environment</param>
public static void ReadTaskList(FileReference TaskListFile, DirectoryReference BaseDir, out Dictionary <FileReference, CompileEnvironment> FileToEnvironment)
{
XmlDocument Document = new XmlDocument();
Document.Load(TaskListFile.FullName);
// Read the standard include paths from the INCLUDE environment variable in the script
List <DirectoryReference> StandardIncludePaths = new List <DirectoryReference>();
foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Variables/Variable"))
{
XmlAttribute NameAttr = Node.Attributes["Name"];
if (NameAttr != null && String.Compare(NameAttr.InnerText, "INCLUDE") == 0)
{
foreach (string IncludePath in Node.Attributes["Value"].InnerText.Split(';'))
{
StandardIncludePaths.Add(new DirectoryReference(IncludePath));
}
}
}
// Read all the individual compiles
Dictionary <FileReference, CompileEnvironment> NewFileToEnvironment = new Dictionary <FileReference, CompileEnvironment>();
foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Tools/Tool"))
{
XmlAttribute ToolPathAttr = Node.Attributes["Path"];
if (ToolPathAttr != null)
{
// Get the full path to the tool
FileReference ToolLocation = new FileReference(ToolPathAttr.InnerText);
// Get the compiler type
CompilerType CompilerType;
if (GetCompilerType(ToolLocation, out CompilerType))
{
string Name = Node.Attributes["Name"].InnerText;
string Params = Node.Attributes["Params"].InnerText;
// Construct the compile environment
CompileEnvironment Environment = new CompileEnvironment(ToolLocation, CompilerType);
// Parse a list of tokens
List <string> Tokens = new List <string>();
ParseArgumentTokens(CompilerType, Params, Tokens);
// Parse it into a list of options, removing any that don't apply
List <FileReference> SourceFiles = new List <FileReference>();
List <FileReference> OutputFiles = new List <FileReference>();
for (int Idx = 0; Idx < Tokens.Count; Idx++)
{
if (Tokens[Idx] == "/Fo" || Tokens[Idx] == "/Fp" || Tokens[Idx] == "-o")
{
OutputFiles.Add(new FileReference(Tokens[++Idx]));
}
else if (Tokens[Idx].StartsWith("/Fo") || Tokens[Idx].StartsWith("/Fp"))
{
OutputFiles.Add(new FileReference(Tokens[Idx].Substring(3)));
}
else if (Tokens[Idx] == "/D" || Tokens[Idx] == "-D")
{
Environment.Definitions.Add(Tokens[++Idx]);
}
else if (Tokens[Idx].StartsWith("/D"))
{
Environment.Definitions.Add(Tokens[Idx].Substring(2));
}
else if (Tokens[Idx] == "/I" || Tokens[Idx] == "-I")
{
Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[++Idx].Replace("//", "/")).FullName.ToLowerInvariant()));
}
else if (Tokens[Idx].StartsWith("-I"))
{
Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[Idx].Substring(2).Replace("//", "/")).FullName.ToLowerInvariant()));
}
else if (Tokens[Idx].StartsWith("/FI"))
{
Environment.ForceIncludeFiles.Add(new FileReference(Tokens[Idx].Substring(3)));
}
else if (Tokens[Idx] == "-include")
{
Environment.ForceIncludeFiles.Add(new FileReference(Tokens[++Idx]));
}
else if (Tokens[Idx] == "-Xclang" || Tokens[Idx] == "-target" || Tokens[Idx] == "--target" || Tokens[Idx] == "-x" || Tokens[Idx] == "-o")
{
Environment.Options.Add(new CompileOption(Tokens[Idx], Tokens[++Idx]));
}
else if (Tokens[Idx].StartsWith("/") || Tokens[Idx].StartsWith("-"))
{
Environment.Options.Add(new CompileOption(Tokens[Idx], null));
}
else
{
SourceFiles.Add(FileReference.Combine(BaseDir, Tokens[Idx]));
}
}
// Make sure we're not compiling a precompiled header
if (!OutputFiles.Any(x => x.HasExtension(".gch")) && !Environment.Options.Any(x => x.Name.StartsWith("/Yc")))
{
// Add all the standard include paths
Environment.IncludePaths.AddRange(StandardIncludePaths);
// Add to the output map if there are any source files. Use the extension to distinguish between a source file and linker invocation on Clang.
foreach (FileReference SourceFile in SourceFiles)
{
if (!SourceFile.HasExtension(".a"))
{
NewFileToEnvironment.Add(SourceFile, Environment);
}
}
}
}
}
}
FileToEnvironment = NewFileToEnvironment;
}
