/// <summary>
/// Creates a string representing a context-free representation of the task, for using in comparing whether the task parameters have changed.
/// </summary>
/// <param name="Worker">The task to generate a digest for</param>
/// <param name="IgnoreFragments">Set of fragments which can be ignored</param>
/// <returns>String digest of the task</returns>
static string CreateDigest(SequenceWorker Worker, HashSet <string> IgnoreFragments)
{
StringWriter Writer = new StringWriter();
Writer.WriteLine("ResponseFile: {0}", Worker.ResponseFileDigest);
Writer.WriteLine("Fragments:");
for (int Idx = 0; Idx < Worker.FragmentFileNames.Length; Idx++)
{
string FragmentFileName = Worker.FragmentFileNames[Idx];
if (!IgnoreFragments.Contains(FragmentFileName))
{
Writer.WriteLine(" {0}={1}", FragmentFileName, Worker.FragmentDigests[Idx]);
}
}
Writer.WriteLine("Script:");
foreach (Tuple <int, string> Line in Worker.Lines)
{
if (Line.Item1 < 0)
{
Writer.WriteLine(" None={0}", Line.Item2);
}
else if (!IgnoreFragments.Contains(Worker.FragmentFileNames[Line.Item1]))
{
Writer.WriteLine(" {0}={1}", Worker.FragmentFileNames[Line.Item1], Line.Item2);
}
}
return(Writer.ToString());
}
/// <summary>
/// Deserialize a worker from a file
/// </summary>
/// <param name="FileName">The filename to deserialize from</param>
/// <returns>The deserialized worker instance</returns>
public static SequenceWorker Deserialize(string FileName)
{
SequenceWorker Worker = new SequenceWorker();
using (FileStream Stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (BinaryReader Reader = new BinaryReader(Stream, Encoding.UTF8, true))
{
Worker.PermutationFileName = Reader.ReadString();
Worker.ResponseFileName = Reader.ReadString();
Worker.ResponseFileDigest = Reader.ReadString();
Worker.FragmentCount = Reader.ReadInt32();
Worker.RemainingFragmentCount = Reader.ReadInt32();
Worker.CompileCount = Reader.ReadInt32();
Worker.CompilerExe = Reader.ReadString();
Worker.bSingleStepMode = Reader.ReadBoolean();
Worker.bResult = Reader.ReadBoolean();
int NumKnownDependencies = Reader.ReadInt32();
for (int Idx = 0; Idx < NumKnownDependencies; Idx++)
{
Worker.KnownDependencies.Add(Reader.ReadInt32());
}
Worker.FragmentFileNames = new string[Reader.ReadInt32()];
for (int Idx = 0; Idx < Worker.FragmentFileNames.Length; Idx++)
{
Worker.FragmentFileNames[Idx] = Reader.ReadString();
}
Worker.FragmentDigests = new string[Reader.ReadInt32()];
for (int Idx = 0; Idx < Worker.FragmentDigests.Length; Idx++)
{
Worker.FragmentDigests[Idx] = Reader.ReadString();
}
Worker.Lines = new Tuple <int, string> [Reader.ReadInt32()];
for (int Idx = 0; Idx < Worker.Lines.Length; Idx++)
{
int Tag = Reader.ReadInt32();
string Line = Reader.ReadString();
Worker.Lines[Idx] = Tuple.Create(Tag, Line);
}
int NumSummaryLogLines = Reader.ReadInt32();
for (int Idx = 0; Idx < NumSummaryLogLines; Idx++)
{
Worker.SummaryLog.Add(Reader.ReadString());
}
int NumCompileLogLines = Reader.ReadInt32();
for (int Idx = 0; Idx < NumCompileLogLines; Idx++)
{
Worker.CompileLog.Add(Reader.ReadString());
}
}
}
return(Worker);
}
/// <summary>
/// Runs the FindDependencyTask with the given state file
/// </summary>
/// <param name="StateFile">Path to the state file</param>
/// <param name="Writer">Writer for any messages</param>
/// <returns>Zero on success</returns>
static int FindNextDependency(string StateFile, LineBasedTextWriter Writer)
{
SequenceWorker Task = SequenceWorker.Deserialize(StateFile);
Task.FindNextDependency(Writer);
Task.Serialize(StateFile + ".out");
return(0);
}
/// <summary>
/// Checks that the given permutation compiles
/// </summary>
/// <param name="StateFile">Path to the state file</param>
/// <param name="Writer">Writer for any messages</param>
/// <returns>Zero on success</returns>
static int Verify(string StateFile, TextWriter Writer)
{
SequenceWorker Task = SequenceWorker.Deserialize(StateFile);
Task.Verify(Writer);
Task.Serialize(StateFile + ".out");
return(0);
}
/// <summary>
/// Adds a known dependency to the given task
/// </summary>
/// <param name="Worker">The task to add the dependency to</param>
/// <param name="Fragments">The array of fragments</param>
/// <param name="FragmentIdx">Index of the dependency</param>
static void AddDependency(SequenceWorker Worker, SourceFragment[] Fragments, int FragmentIdx)
{
// Add the dependency
Worker.KnownDependencies.Add(FragmentIdx);
// Check for any macro definitions that the given fragment depends on
SourceFragment Fragment = Fragments[FragmentIdx];
foreach (Symbol ReferencedSymbol in Fragment.ReferencedSymbols.Keys)
{
if (ReferencedSymbol.Type == SymbolType.Macro)
{
int ReferencedFragmentIdx = Array.IndexOf(Fragments, ReferencedSymbol.Fragment);
if (ReferencedFragmentIdx != -1 && ReferencedFragmentIdx < FragmentIdx)
{
Worker.KnownDependencies.Add(ReferencedFragmentIdx);
}
}
}
// Force dependencies onto pinned fragments. This can make a difference when templated functions are implemented in pinned headers
// but declared in another (eg. LegacyText.h). The file will compile fine without the implementations being present, expecting them to be linked
// into another object file, but fail when they are included due to being pinned later because MSVC only parses the token stream then.
for (int MarkupIdx = Fragment.MarkupMin; MarkupIdx < Fragment.MarkupMax; MarkupIdx++)
{
PreprocessorMarkup Markup = Fragment.File.Markup[MarkupIdx];
if (Markup.Type == PreprocessorMarkupType.Include && Markup.IncludedFile != null && (Markup.IncludedFile.Flags & SourceFileFlags.Pinned) != 0 && Markup.IncludedFile.Counterpart == null)
{
foreach (SourceFragment PinnedFragment in Markup.IncludedFile.Fragments)
{
int PinnedFragmentIdx = Array.IndexOf(Fragments, PinnedFragment);
if (PinnedFragmentIdx != -1 && PinnedFragmentIdx < FragmentIdx)
{
AddDependency(Worker, Fragments, PinnedFragmentIdx);
}
}
}
}
}
/// <summary>
/// Wait for this worker to complete
/// </summary>
/// <param name="Writer">Writer for log output</param>
/// <returns>Return code from the thread</returns>
public SequenceProbeResult Join(LineBasedTextWriter Writer)
{
// Finish the task instance
BufferedTextWriter BufferedWriter = new BufferedTextWriter();
int ExitCode = ActiveInstance.Join(BufferedWriter);
ActiveInstance.Dispose();
ActiveInstance = null;
// Read the new state
FileReference OutputFile = new FileReference(TaskStateFile.FullName + ".out");
SequenceWorker NewWorker = SequenceWorker.Deserialize(OutputFile.FullName);
OutputFile.Delete();
// Make sure the exit code reflects the failure state. XGE can sometimes fail transferring back.
if (ExitCode == 0 && !NewWorker.bResult)
{
ExitCode = -1;
}
// If it's a hard failure, print the compile log to the regular log
if (ExitCode == -1)
{
Writer.WriteLine("Failed to compile {0}, exit code {1}:", UniqueName, ExitCode);
foreach (string Line in NewWorker.CompileLog)
{
Writer.WriteLine(" > {0}", Line.Replace("error:", "err:"));
}
}
// Annotate the log data if this is from a failed attempt. It still may be useful for debugging purposes.
if (ExitCode != 0)
{
NewWorker.SummaryLog.Insert(0, String.Format("ExitCode={0}", ExitCode));
NewWorker.SummaryLog = NewWorker.SummaryLog.Select(x => "FAIL > " + x).ToList();
NewWorker.CompileLog.Insert(0, String.Format("ExitCode={0}", ExitCode));
NewWorker.CompileLog = NewWorker.CompileLog.Select(x => "FAIL > " + x).ToList();
}
// Append the log data back to the local output
File.AppendAllLines(SummaryLogFile.FullName, NewWorker.SummaryLog);
NewWorker.SummaryLog.Clear();
File.AppendAllLines(CompileLogFile.FullName, NewWorker.CompileLog);
NewWorker.CompileLog.Clear();
// If we failed, return the
if (ExitCode != 0)
{
if (ExitCode == -1)
{
Writer.WriteLine("Warning: Failed to compile {0}, exit code {1}. Aborting.", UniqueName, ExitCode);
return(SequenceProbeResult.Failed);
}
else
{
Writer.WriteLine("Warning: Failed to compile {0}; exit code {1}. Will retry.", UniqueName, ExitCode);
return(SequenceProbeResult.FailedAllowRetry);
}
}
// Update the task
Worker = NewWorker;
// Check if this is just an incremental update
if (Type == SequenceProbeType.Verify)
{
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Return that we're done
return(SequenceProbeResult.Completed);
}
else if (Type == SequenceProbeType.Optimize)
{
if (Worker.RemainingFragmentCount > 0)
{
// Get the top-most fragment - the one we've just established is a dependency for this leaf node - and add it to the list of known dependencies
SourceFragment NextFragment = Fragments[Worker.RemainingFragmentCount - 1];
AddDependency(Worker, Fragments, Worker.RemainingFragmentCount - 1);
Worker.SummaryLog.Add(String.Format(" [Added {0}: {1}]", Worker.RemainingFragmentCount - 1, Fragments[Worker.RemainingFragmentCount - 1].Location));
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Otherwise, return that we've just updated
return(SequenceProbeResult.Updated);
}
else
{
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Return that we're done
SetCompletedDependencies();
return(SequenceProbeResult.Completed);
}
}
else
{
throw new NotImplementedException();
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}
