/// <summary> /// Clone another compile environment /// </summary> /// <param name="Other">The compile environment to copy</param> public CompileEnvironment(CompileEnvironment Other) { Compiler = Other.Compiler; CompilerType = Other.CompilerType; Options = new List <CompileOption>(Other.Options); Definitions = new List <string>(Other.Definitions); IncludePaths = new List <DirectoryReference>(Other.IncludePaths); }
/// <summary> /// Constructor /// </summary> /// <param name="Type">The type of probe to create</param> /// <param name="Node">The node to optimize</param> /// <param name="IntermediateDir">Directory for intermediate files</param> /// <param name="UniqueName">Unique name prefix for all temporary files</param> public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName) { this.Type = Type; this.IntermediateDir = IntermediateDir; this.Fragments = Fragments; this.LastFragment = Fragments[Fragments.Length - 1]; this.UniqueName = UniqueName; this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state")); this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt"); this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt"); // Get the file to use for trying to compile different permutations FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation"); // Create the response file FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response"); CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment); if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang) { WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/'))); } else { WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI"); WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName)); WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null)); } WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile); string ResponseFileDigest = Utility.ComputeDigest(ResponseFile); // Keep track of the include stack, so we can format the flat fragment list with context int IncludeHistoryIdx = 0; List <SourceFile> IncludeStack = new List <SourceFile>(); // Create the script for the probe List <Tuple <int, string> > Lines = new List <Tuple <int, string> >(); for (int Idx = 0; Idx < Fragments.Length; Idx++) { SourceFragment Fragment = Fragments[Idx]; // Figure out which tag it's bound to int Tag = (Fragment.File.Counterpart == null)? Idx : -1; // Update the stack for new includes while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx) { if (IncludeHistory[IncludeHistoryIdx].Item2 == null) { SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1]; IncludeStack.RemoveAt(IncludeStack.Count - 1); Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName))); IncludeHistoryIdx++; } else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null) { IncludeHistoryIdx += 2; } else { SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2; Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName))); IncludeStack.Add(IncludeFile); IncludeHistoryIdx++; } } // Get the indent at this point string Indent = new string(' ', (IncludeStack.Count - 0) * 4); // Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments // if the heuristic for detecting when to use them doesn't work. if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0) { foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols) { if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration)) { Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration)); } } } // Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x)); if (BaseSystemIncludePath != null) { Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath)))); } else { Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName))); } } // Create the new task string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray(); string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray(); Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName); AddDependency(Worker, Fragments, Fragments.Length - 1); // Log the referenced symbols if (LastFragment.ReferencedSymbols.Count > 0) { List <string> ReferenceLog = new List <string>(); ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location)); foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name)) { Symbol ReferencedSymbol = Pair.Key; ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location)); } ReferenceLog.Add(""); Worker.SummaryLog.InsertRange(0, ReferenceLog); } // Check to see if an existing version of the task exists which we can continue if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists()) { // Try to read the old task SequenceWorker OldWorker; try { OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName); } catch (Exception) { OldWorker = null; } // If it succeeded, compare it to the new task if (OldWorker != null) { SequenceWorker NewWorker = Worker; // Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task HashSet <string> IgnoreFragments = new HashSet <string>(); for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++) { if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount) { IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]); } } IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x])); // Compute digests for the old and new tasks string OldDigest = CreateDigest(OldWorker, IgnoreFragments); string NewDigest = CreateDigest(NewWorker, IgnoreFragments); if (OldDigest == NewDigest) { // Build a map of fragment file names to their new index Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>(); for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++) { string FragmentFileName = FragmentFileNames[Idx]; FragmentFileNameToNewIndex[FragmentFileName] = Idx; } // Add known dependencies to the new worker foreach (int OldKnownDependency in OldWorker.KnownDependencies) { string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency]; int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName]; NewWorker.KnownDependencies.Add(NewKnownDependency); } // Update the remaining count. All these fragments must match, because they're not part of the ignore list. NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount; } } } // If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare. FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location; if (MainFileLocation.HasExtension(".cpp")) { string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h"); for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++) { if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0) { AddDependency(Worker, Fragments, FragmentIdx); } } } // Update the finished fragment if we're done, otherwise clear out all the intermediate files if (Worker.RemainingFragmentCount == 0) { SetCompletedDependencies(); } else { Worker.Serialize(TaskStateFile.FullName); PermutationFile.Delete(); SummaryLogFile.Delete(); CompileLogFile.Delete(); } }
/// <summary> /// Reads an exported XGE task list /// </summary> /// <param name="TaskListFile">File to read from</param> /// <param name="FileToEnvironment">Mapping from source file to compile environment</param> public static void ReadTaskList(FileReference TaskListFile, DirectoryReference BaseDir, out Dictionary <FileReference, CompileEnvironment> FileToEnvironment) { XmlDocument Document = new XmlDocument(); Document.Load(TaskListFile.FullName); // Read the standard include paths from the INCLUDE environment variable in the script List <DirectoryReference> StandardIncludePaths = new List <DirectoryReference>(); foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Variables/Variable")) { XmlAttribute NameAttr = Node.Attributes["Name"]; if (NameAttr != null && String.Compare(NameAttr.InnerText, "INCLUDE") == 0) { foreach (string IncludePath in Node.Attributes["Value"].InnerText.Split(';')) { StandardIncludePaths.Add(new DirectoryReference(IncludePath)); } } } // Read all the individual compiles Dictionary <FileReference, CompileEnvironment> NewFileToEnvironment = new Dictionary <FileReference, CompileEnvironment>(); foreach (XmlNode Node in Document.SelectNodes("BuildSet/Environments/Environment/Tools/Tool")) { XmlAttribute ToolPathAttr = Node.Attributes["Path"]; if (ToolPathAttr != null) { // Get the full path to the tool FileReference ToolLocation = new FileReference(ToolPathAttr.InnerText); // Get the compiler type CompilerType CompilerType; if (GetCompilerType(ToolLocation, out CompilerType)) { string Name = Node.Attributes["Name"].InnerText; string Params = Node.Attributes["Params"].InnerText; // Construct the compile environment CompileEnvironment Environment = new CompileEnvironment(ToolLocation, CompilerType); // Parse a list of tokens List <string> Tokens = new List <string>(); ParseArgumentTokens(CompilerType, Params, Tokens); // Parse it into a list of options, removing any that don't apply List <FileReference> SourceFiles = new List <FileReference>(); List <FileReference> OutputFiles = new List <FileReference>(); for (int Idx = 0; Idx < Tokens.Count; Idx++) { if (Tokens[Idx] == "/Fo" || Tokens[Idx] == "/Fp" || Tokens[Idx] == "-o") { OutputFiles.Add(new FileReference(Tokens[++Idx])); } else if (Tokens[Idx].StartsWith("/Fo") || Tokens[Idx].StartsWith("/Fp")) { OutputFiles.Add(new FileReference(Tokens[Idx].Substring(3))); } else if (Tokens[Idx] == "/D" || Tokens[Idx] == "-D") { Environment.Definitions.Add(Tokens[++Idx]); } else if (Tokens[Idx].StartsWith("/D")) { Environment.Definitions.Add(Tokens[Idx].Substring(2)); } else if (Tokens[Idx] == "/I" || Tokens[Idx] == "-I") { Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[++Idx].Replace("//", "/")).FullName.ToLowerInvariant())); } else if (Tokens[Idx].StartsWith("-I")) { Environment.IncludePaths.Add(new DirectoryReference(DirectoryReference.Combine(BaseDir, Tokens[Idx].Substring(2).Replace("//", "/")).FullName.ToLowerInvariant())); } else if (Tokens[Idx].StartsWith("/FI")) { Environment.ForceIncludeFiles.Add(new FileReference(Tokens[Idx].Substring(3))); } else if (Tokens[Idx] == "-include") { Environment.ForceIncludeFiles.Add(new FileReference(Tokens[++Idx])); } else if (Tokens[Idx] == "-Xclang" || Tokens[Idx] == "-target" || Tokens[Idx] == "--target" || Tokens[Idx] == "-x" || Tokens[Idx] == "-o") { Environment.Options.Add(new CompileOption(Tokens[Idx], Tokens[++Idx])); } else if (Tokens[Idx].StartsWith("/") || Tokens[Idx].StartsWith("-")) { Environment.Options.Add(new CompileOption(Tokens[Idx], null)); } else { SourceFiles.Add(FileReference.Combine(BaseDir, Tokens[Idx])); } } // Make sure we're not compiling a precompiled header if (!OutputFiles.Any(x => x.HasExtension(".gch")) && !Environment.Options.Any(x => x.Name.StartsWith("/Yc"))) { // Add all the standard include paths Environment.IncludePaths.AddRange(StandardIncludePaths); // Add to the output map if there are any source files. Use the extension to distinguish between a source file and linker invocation on Clang. foreach (FileReference SourceFile in SourceFiles) { if (!SourceFile.HasExtension(".a")) { NewFileToEnvironment.Add(SourceFile, Environment); } } } } } } FileToEnvironment = NewFileToEnvironment; }