/// <summary>
/// Try to skip over a template header
/// </summary>
/// <param name="OriginalReader">Tokens to parse</param>
/// <param name="Fragment">Fragment containing these tokens</param>
bool ReadTypedefHeader(TokenReader OriginalReader, SourceFragment Fragment)
{
TokenReader Reader = new TokenReader(OriginalReader);
// Check for the typedef keyword
Token PreviousToken = Reader.Current;
if (Reader.Current.Text != "typedef" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Scan to the next semicolon
while (Reader.MoveNext(TokenReaderContext.IgnoreNewlines) && Reader.Current.Text != ";" && Reader.Current.Text != "{")
{
PreviousToken = Reader.Current;
}
// Ignore 'typedef struct' and 'typedef union' declarations.
if (Reader.Current.Text == "{")
{
return(false);
}
// Try to add a symbol for the previous token. If it already exists, replace it.
if (PreviousToken.Type == TokenType.Identifier && Rules.AllowSymbol(PreviousToken.Text))
{
AddSymbol(PreviousToken.Text, SymbolType.Typedef, null, Fragment, OriginalReader.TokenLocation);
}
// Move the original reader past the declaration
OriginalReader.Set(Reader);
return(true);
}
/// <summary>
/// Parse declarations from the reader
/// </summary>
/// <param name="File">The file containing the declarations</param>
/// <param name="MarkupIdx">The markup to parse declarations from</param>
/// <param name="Scope">The scope depth; the number of unmatched opening brace tokens</param>
void ParseDeclarations(SourceFile File, SourceFragment Fragment, PreprocessorMarkup Markup, ref int Scope)
{
TokenReader Reader = new TokenReader(File.Text, Markup.Location, Markup.EndLocation);
if (Reader.MoveNext())
{
for (;;)
{
if (Scope > 0 || (!ReadEnumClassHeader(Reader, Fragment) && !ReadClassOrStructHeader(Reader, Fragment) && !ReadTypedefHeader(Reader, Fragment) && !ReadTemplateClassOrStructHeader(Reader, Fragment) && !SkipTemplateHeader(Reader)))
{
// Update the scope depth
if (Reader.Current.Text == "{")
{
Scope++;
}
else if (Reader.Current.Text == "}" && --Scope < 0)
{
throw new Exception(String.Format("Unbalanced '}}' while parsing '{0}'", File.Location));
}
// Move to the next token
if (!Reader.MoveNext())
{
break;
}
}
}
}
}
/// <summary>
/// Creates an output file which represents the same includes as the inpu tfile
/// </summary>
/// <param name="InputFile">The input file that this output file corresponds to</param>
/// <param name="Includes">The active set of includes parsed for this file</param>
/// <param name="Log">Writer for log messages</param>
public static OutputFile CreatePassthroughOutputFile(SourceFile InputFile, List <OutputFileInclude> Includes, LineBasedTextWriter Log)
{
// Write the state
InputFile.LogVerbose("InputFile={0}", InputFile.Location.FullName);
InputFile.LogVerbose("Duplicate.");
// Reduce the list of includes to those that are required.
HashSet <SourceFragment> Dependencies = new HashSet <SourceFragment>();
for (int FragmentIdx = InputFile.Fragments.Length - 1, IncludeIdx = Includes.Count - 1; FragmentIdx >= 0; FragmentIdx--)
{
// Update the dependency lists for this fragment
SourceFragment InputFragment = InputFile.Fragments[FragmentIdx];
if (InputFragment.Dependencies != null)
{
Dependencies.UnionWith(InputFragment.Dependencies);
}
Dependencies.Remove(InputFragment);
// Scan backwards through the list of includes, expanding each include to those which are required
int MarkupMin = (FragmentIdx == 0)? -1 : InputFragment.MarkupMin;
for (; IncludeIdx >= 0 && Includes[IncludeIdx].MarkupIdx >= MarkupMin; IncludeIdx--)
{
OutputFileInclude Include = Includes[IncludeIdx];
Include.FinalFiles.Add(Include.TargetFile);
Dependencies.ExceptWith(Include.TargetFile.IncludedFragments);
Dependencies.UnionWith(Include.TargetFile.Dependencies);
}
}
// Create the optimized file
return(new OutputFile(InputFile, Includes, new HashSet <SourceFragment>(), new List <Symbol>()));
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Name">Symbol name</param>
/// <param name="Type">Type of the symbol</param>
/// <param name="ForwardDeclaration">Text for a forward declaration of this symbol</param>
/// <param name="Fragment">File that this symbol is defined in</param>
/// <param name="Location">Location within the file that this symbol is declared</param>
public Symbol(string Name, SymbolType Type, string ForwardDeclaration, SourceFragment Fragment, TextLocation Location)
{
this.Name = Name;
this.Type = Type;
this.ForwardDeclaration = ForwardDeclaration;
this.Fragment = Fragment;
this.Location = Location;
}
/// <summary>
/// Try to read a class or struct header
/// </summary>
/// <param name="Reader">Tokens to parse</param>
/// <param name="Fragment">Fragment containing these tokens</param>
bool ReadClassOrStructHeader(TokenReader OriginalReader, SourceFragment Fragment)
{
TokenReader Reader = new TokenReader(OriginalReader);
// Make sure it's the right type
SymbolType Type;
if (Reader.Current.Text == "class")
{
Type = SymbolType.Class;
}
else if (Reader.Current.Text == "struct")
{
Type = SymbolType.Struct;
}
else
{
return(false);
}
// Move to the name
string ClassOrStructKeyword = Reader.Current.Text;
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Skip over an optional DLL export declaration
if (Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API") && !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Read the class name
string Name = Reader.Current.Text;
if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
if (Reader.Current.Text != ":" && Reader.Current.Text != "{")
{
return(false);
}
// Create the symbol
if (Rules.AllowSymbol(Name))
{
string ForwardDeclaration = String.Format("{0} {1};", ClassOrStructKeyword, Name);
AddSymbol(Name, Type, ForwardDeclaration, Fragment, OriginalReader.TokenLocation);
}
// Move the original reader past the declaration
OriginalReader.Set(Reader);
return(true);
}
/// <summary>
/// Parse an "enum class" declaration, and add a symbol for it
/// </summary>
/// <param name="Reader">Tokens to be parsed. On success, this is assigned to a new </param>
/// <param name="Fragment">Fragment containing these tokens</param>
bool ReadEnumClassHeader(TokenReader OriginalReader, SourceFragment Fragment)
{
TokenReader Reader = new TokenReader(OriginalReader);
// Read the 'enum class' tokens
if (Reader.Current.Text != "enum" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
if (Reader.Current.Text != "class" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Read the name, make sure we haven't read a definition for it already, and check it's an enum declaration
string Name = Reader.Current.Text;
if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
if (Reader.Current.Text != ";" && Reader.Current.Text != ":" && Reader.Current.Text != "{")
{
return(false);
}
// Build the forward declaration for it
StringBuilder ForwardDeclaration = new StringBuilder();
ForwardDeclaration.AppendFormat("enum class {0}", Name);
while (Reader.Current.Text != ";" && Reader.Current.Text != "{")
{
// Append the next token
if (Reader.Current.HasLeadingSpace)
{
ForwardDeclaration.Append(" ");
}
ForwardDeclaration.Append(Reader.Current.Text);
// Try to move to the next token
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
}
ForwardDeclaration.Append(";");
// Create a symbol for it if it's an actual definition rather than a forward declaration
if (Reader.Current.Text == "{" && Rules.AllowSymbol(Name))
{
AddSymbol(Name, SymbolType.Enumeration, ForwardDeclaration.ToString(), Fragment, OriginalReader.TokenLocation);
}
// Update the original reader to be the new location
OriginalReader.Set(Reader);
return(true);
}
/// <summary>
/// Determines whether a symbol reference should be ignored, because it's to an item defined above in the same file
/// </summary>
/// <param name="Symbol">The referenced symbol</param>
/// <param name="Fragment">The current fragment being parsed</param>
/// <param name="Location">Position within the current fragment</param>
/// <returns>True if the symbol should be ignored</returns>
static bool IgnoreSymbol(Symbol Symbol, SourceFragment Fragment, TextLocation Location)
{
if (Symbol.Fragment.File == Fragment.File && Location > Symbol.Location)
{
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// Update the list of known dependencies to include dependencies of our already known dependencies
/// </summary>
/// <returns>True if the dependencies were updated, false if a dependency was encountered which is not available to this fragment</returns>
void UpdateDependencies(LineBasedTextWriter Log)
{
// Update the task to account for any new dependencies that may have been found. Loop back through all the fragments so we include
// dependencies we find along the way.
for (int FragmentIdx = Worker.FragmentCount - 1; FragmentIdx > 0; FragmentIdx--)
{
SourceFragment Fragment = Fragments[FragmentIdx];
if (Worker.KnownDependencies.Contains(FragmentIdx) && Fragment.Dependencies != null)
{
// Include the dependencies of this fragment in our dependencies list
List <SourceFragment> MissingDependencies = new List <SourceFragment>();
foreach (SourceFragment Dependency in Fragment.Dependencies)
{
int DependencyIdx = Array.IndexOf(Fragments, Dependency);
if (DependencyIdx == -1 || DependencyIdx > FragmentIdx)
{
MissingDependencies.Add(Dependency);
}
else
{
Worker.KnownDependencies.Add(DependencyIdx);
}
}
// If any were missing (because they were calculated by a different derivation), provide a useful diagnostic. Otherwise re-queue the worker
// to find the next dependency.
if (MissingDependencies.Count > 0 && MissingDependencies.Any(x => !WarnedMissingDependencies.Contains(x)))
{
StringBuilder Message = new StringBuilder();
Log.WriteLine("warning: could not find dependencies of '{0}':", Fragment.Location.FullName);
foreach (SourceFragment MissingDependency in MissingDependencies)
{
Log.WriteLine(" {0}", MissingDependency.Location);
}
Log.WriteLine("In fragments for '{0}':", LastFragment.Location);
for (int Idx = 0; Idx < Worker.RemainingFragmentCount; Idx++)
{
Log.WriteLine(" {0,3}: {1}", Idx, Fragments[Idx].Location.FullName);
}
for (int Idx = Worker.RemainingFragmentCount; Idx < Fragments.Length; Idx++)
{
Log.WriteLine(" {0,3}: ({1})", Idx, Fragments[Idx].Location.FullName);
}
WarnedMissingDependencies.UnionWith(MissingDependencies);
}
}
}
}
/// <summary>
/// Adds a symbol to the lookup. If the symbol already exists and is marked as a different type, mark it as ambiguous.
/// </summary>
/// <param name="Name">The symbol name</param>
/// <param name="Type">Type of the symbol</param>
/// <param name="ForwardDeclaration">Code used to declare a forward declaration</param>
/// <param name="Fragment">The current fragment</param>
/// <param name="Location">Location within the file that this symbol is declared</param>
Symbol AddSymbol(string Name, SymbolType Type, string ForwardDeclaration, SourceFragment Fragment, TextLocation Location)
{
// Try to find an existing symbol which matches
foreach (Symbol ExistingSymbol in Lookup.WithKey(Name))
{
if (ExistingSymbol.Type == Type && ExistingSymbol.Fragment == Fragment)
{
return(ExistingSymbol);
}
}
// Otherwise create a new one
Symbol NewSymbol = new Symbol(Name, Type, ForwardDeclaration, Fragment, Location);
Lookup.Add(Name, NewSymbol);
return(NewSymbol);
}
/// <summary>
/// Adds a known dependency to the given task
/// </summary>
/// <param name="Worker">The task to add the dependency to</param>
/// <param name="Fragments">The array of fragments</param>
/// <param name="FragmentIdx">Index of the dependency</param>
static void AddDependency(SequenceWorker Worker, SourceFragment[] Fragments, int FragmentIdx)
{
// Add the dependency
Worker.KnownDependencies.Add(FragmentIdx);
// Check for any macro definitions that the given fragment depends on
SourceFragment Fragment = Fragments[FragmentIdx];
foreach (Symbol ReferencedSymbol in Fragment.ReferencedSymbols.Keys)
{
if (ReferencedSymbol.Type == SymbolType.Macro)
{
int ReferencedFragmentIdx = Array.IndexOf(Fragments, ReferencedSymbol.Fragment);
if (ReferencedFragmentIdx != -1 && ReferencedFragmentIdx < FragmentIdx)
{
Worker.KnownDependencies.Add(ReferencedFragmentIdx);
}
}
}
// Force dependencies onto pinned fragments. This can make a difference when templated functions are implemented in pinned headers
// but declared in another (eg. LegacyText.h). The file will compile fine without the implementations being present, expecting them to be linked
// into another object file, but fail when they are included due to being pinned later because MSVC only parses the token stream then.
for (int MarkupIdx = Fragment.MarkupMin; MarkupIdx < Fragment.MarkupMax; MarkupIdx++)
{
PreprocessorMarkup Markup = Fragment.File.Markup[MarkupIdx];
if (Markup.Type == PreprocessorMarkupType.Include && Markup.IncludedFile != null && (Markup.IncludedFile.Flags & SourceFileFlags.Pinned) != 0 && Markup.IncludedFile.Counterpart == null)
{
foreach (SourceFragment PinnedFragment in Markup.IncludedFile.Fragments)
{
int PinnedFragmentIdx = Array.IndexOf(Fragments, PinnedFragment);
if (PinnedFragmentIdx != -1 && PinnedFragmentIdx < FragmentIdx)
{
AddDependency(Worker, Fragments, PinnedFragmentIdx);
}
}
}
}
}
/// <summary>
/// Finds all the imported symbols from the given fragment
/// </summary>
/// <param name="Fragment">Fragment to find symbols for</param>
/// <returns>Array of unique symbols for this fragment, which are not already forward-declared</returns>
public Dictionary <Symbol, SymbolReferenceType> FindReferences(SourceFragment Fragment)
{
Dictionary <Symbol, SymbolReferenceType> References = new Dictionary <Symbol, SymbolReferenceType>();
TextBuffer Text = Fragment.File.Text;
if (Text != null && Fragment.MarkupMax > Fragment.MarkupMin)
{
TokenReader Reader = new TokenReader(Fragment.File.Text, Fragment.MinLocation, Fragment.MaxLocation);
for (bool bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); bMoveNext;)
{
// Read the current token, and immediately move to the next so that we can lookahead if we need to
Token Token = Reader.Current;
bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines);
// Check if the previous token is a symbol
if (Token.Type == TokenType.Identifier)
{
if (Token.Text == "class" || Token.Text == "struct")
{
// Skip over "class", "struct" and "enum class" declarations and forward declarations. They're not actually referencing another symbol.
if (bMoveNext && Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API"))
{
bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines);
}
if (bMoveNext && Reader.Current.Type == TokenType.Identifier)
{
bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines);
}
}
else if (IsSmartPointerClass(Token.Text) && bMoveNext && Reader.Current.Text == "<")
{
// Smart pointer types which do not require complete declarations
bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines);
if (bMoveNext && Reader.Current.Type == TokenType.Identifier)
{
foreach (Symbol MatchingSymbol in Lookup.WithKey(Reader.Current.Text))
{
if ((MatchingSymbol.Type == SymbolType.Class || MatchingSymbol.Type == SymbolType.Struct) && !References.ContainsKey(MatchingSymbol) && !IgnoreSymbol(MatchingSymbol, Fragment, Reader.TokenLocation))
{
References[MatchingSymbol] = SymbolReferenceType.Opaque;
}
}
bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines);
}
}
else
{
// Otherwise check what type of symbol it is (if any), and whether it's referenced in an opaque way (though a pointer/reference or so on).
foreach (Symbol MatchingSymbol in Lookup.WithKey(Token.Text))
{
if (IgnoreSymbol(MatchingSymbol, Fragment, Reader.TokenLocation))
{
continue;
}
else if (MatchingSymbol.Type == SymbolType.Macro)
{
// Add the symbol if it doesn't already exist
if (!References.ContainsKey(MatchingSymbol))
{
References.Add(MatchingSymbol, SymbolReferenceType.RequiresDefinition);
}
}
else if (MatchingSymbol.Type == SymbolType.Enumeration)
{
// Check whether we're using the type in an opaque way, or we actually require the definition
if (!bMoveNext || Reader.Current.Text == "::")
{
References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition;
}
else if (!References.ContainsKey(MatchingSymbol))
{
References[MatchingSymbol] = SymbolReferenceType.Opaque;
}
}
else if (MatchingSymbol.Type == SymbolType.Class || MatchingSymbol.Type == SymbolType.Struct)
{
// Check whether we're declaring a pointer to this type, and just need a forward declaration
if (!bMoveNext || (Reader.Current.Text != "*" && Reader.Current.Text != "&"))
{
References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition;
}
else if (!References.ContainsKey(MatchingSymbol))
{
References[MatchingSymbol] = SymbolReferenceType.Opaque;
}
}
else if (MatchingSymbol.Type == SymbolType.TemplateClass || MatchingSymbol.Type == SymbolType.TemplateStruct)
{
// Check whether we're declaring a pointer to this type, and just need a forward declaration
TokenReader Lookahead = new TokenReader(Reader);
bool bLookaheadMoveNext = bMoveNext;
if (!bMoveNext || !SkipTemplateArguments(Lookahead, ref bLookaheadMoveNext) || (Lookahead.Current.Text != "*" && Lookahead.Current.Text != "&"))
{
References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition;
}
else if (!References.ContainsKey(MatchingSymbol))
{
References[MatchingSymbol] = SymbolReferenceType.Opaque;
}
}
}
}
}
}
}
return(References);
}
/// <summary>
/// Creates an optimized output file
/// </summary>
/// <param name="InputFile">The input file that this output file corresponds to</param>
/// <param name="HeaderFile">The corresponding header file</param>
/// <param name="PreviousFiles">List of files parsed before this one</param>
/// <param name="Includes">The active set of includes parsed for this file</param>
/// <param name="InputFileStack">The active include stack</param>
/// <param name="FwdSymbolToHeader"></param>
/// <param name="bMakeStandalone">Whether to make this output file standalone</param>
/// <param name="Log">Writer for log messages</param>
public static OutputFile CreateOptimizedOutputFile(SourceFile InputFile, OutputFile HeaderFile, List <OutputFile> PreviousFiles, List <OutputFileInclude> Includes, List <SourceFile> InputFileStack, Dictionary <Symbol, OutputFile> FwdSymbolToHeader, bool bMakeStandalone, LineBasedTextWriter Log)
{
Debug.Assert(HeaderFile == null || (InputFile.Flags & SourceFileFlags.TranslationUnit) != 0);
// Write the state
InputFile.LogVerbose("InputFile={0}", InputFile.Location.FullName);
InputFile.LogVerbose("InputFile.Flags={0}", InputFile.Flags.ToString());
if (HeaderFile != null)
{
InputFile.LogVerbose("HeaderFile={0}", HeaderFile.InputFile.Location.FullName);
InputFile.LogVerbose("HeaderFile.Flags={0}", HeaderFile.InputFile.Flags.ToString());
}
InputFile.LogVerbose("");
for (int Idx = 0; Idx < InputFileStack.Count; Idx++)
{
InputFile.LogVerbose("InputFileStack[{0}]={1}", Idx, InputFileStack[Idx].Location.FullName);
}
InputFile.LogVerbose("");
for (int Idx = 0; Idx < PreviousFiles.Count; Idx++)
{
InputFile.LogVerbose("PreviousFiles[{0}]={1}", Idx, PreviousFiles[Idx].InputFile.Location.FullName);
}
InputFile.LogVerbose("");
for (int Idx = 0; Idx < Includes.Count; Idx++)
{
}
InputFile.LogVerbose("");
for (int Idx = 0; Idx < Includes.Count; Idx++)
{
OutputFileInclude Include = Includes[Idx];
InputFile.LogVerbose("Includes[{0}]={1}", Idx, Includes[Idx].TargetFile.InputFile.Location.FullName);
foreach (SourceFragment Fragment in Include.FinalFiles.SelectMany(x => x.IncludedFragments))
{
InputFile.LogVerbose("Includes[{0}].FinalFiles.IncludedFragments={1}", Idx, Fragment);
}
}
// Traverse through all the included headers, figuring out the first unique include for each file and fragment
HashSet <OutputFile> VisitedFiles = new HashSet <OutputFile>();
HashSet <SourceFragment> VisitedFragments = new HashSet <SourceFragment>();
// Go through the standalone headers first
OutputFile MonolithicHeader = null;
if (HeaderFile == null && (InputFile.Flags & SourceFileFlags.Standalone) != 0 && (InputFile.Flags & SourceFileFlags.External) == 0 && (InputFile.Flags & SourceFileFlags.Aggregate) == 0)
{
// Insert a dummy include to receive all the inserted headers
OutputFileInclude ImplicitInclude = new OutputFileInclude(-1, null);
ImplicitInclude.ExpandedReferences = new List <OutputFileReference>();
Includes.Insert(0, ImplicitInclude);
// Determine which monolithic header to use
IEnumerable <OutputFile> PotentialMonolithicHeaders = PreviousFiles.Union(Includes.Select(x => x.TargetFile).Where(x => x != null).SelectMany(x => x.IncludedFiles));
if (InputFile.Module != null && InputFile.Module.PublicDependencyModules.Union(InputFile.Module.PrivateDependencyModules).Any(x => x.Name == "Core"))
{
MonolithicHeader = PotentialMonolithicHeaders.FirstOrDefault(x => (x.InputFile.Flags & SourceFileFlags.IsCoreMinimal) != 0);
}
else
{
MonolithicHeader = PotentialMonolithicHeaders.FirstOrDefault(x => (x.InputFile.Flags & SourceFileFlags.IsCoreTypes) != 0);
}
// Update the dependencies to treat all the contents of a monolithic header as pinned
if (MonolithicHeader != null)
{
SourceFragment[] UniqueFragments = MonolithicHeader.IncludedFragments.Except(VisitedFragments).ToArray();
ImplicitInclude.ExpandedReferences.Add(new OutputFileReference(MonolithicHeader, UniqueFragments));
VisitedFragments.UnionWith(UniqueFragments);
VisitedFiles.Add(MonolithicHeader);
}
// Insert all the forward declaration headers, but only treat them as supplying the forward declarations themselves. They may happen to include
// some utility classes (eg. TSharedPtr), and we don't want to include an unrelated header to satisfy that dependency.
foreach (OutputFile FwdHeader in FwdSymbolToHeader.Values)
{
FindExpandedReferences(FwdHeader, ImplicitInclude.ExpandedReferences, VisitedFiles, VisitedFragments, true);
}
// Add all the other files
if (bMakeStandalone)
{
foreach (OutputFile PreviousFile in PreviousFiles)
{
if ((InputFile.Flags & SourceFileFlags.Standalone) != 0 && (PreviousFile.InputFile.Flags & SourceFileFlags.Inline) == 0 && (PreviousFile.InputFile.Flags & SourceFileFlags.Pinned) == 0 && VisitedFiles.Add(PreviousFile))
{
SourceFragment[] UniqueFragments = PreviousFile.IncludedFragments.Except(VisitedFragments).ToArray();
ImplicitInclude.ExpandedReferences.Add(new OutputFileReference(PreviousFile, UniqueFragments));
VisitedFragments.UnionWith(UniqueFragments);
}
}
}
}
// Figure out a list of files which are uniquely reachable through each include. Force an include of the matching header as the first thing.
OutputFileReference ForcedHeaderFileReference = null;
foreach (OutputFileInclude Include in Includes)
{
if (Include.ExpandedReferences == null)
{
Include.ExpandedReferences = new List <OutputFileReference>();
if (Include == Includes[0] && HeaderFile != null)
{
ForcedHeaderFileReference = new OutputFileReference(HeaderFile, HeaderFile.IncludedFragments);
Include.ExpandedReferences.Add(ForcedHeaderFileReference);
VisitedFragments.UnionWith(HeaderFile.IncludedFragments);
}
FindExpandedReferences(Include.TargetFile, Include.ExpandedReferences, VisitedFiles, VisitedFragments, true);
}
}
// Find all the symbols which are referenced by this file
HashSet <SourceFragment> FragmentsWithReferencedSymbols = new HashSet <SourceFragment>();
foreach (SourceFragment Fragment in InputFile.Fragments)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (ReferencedSymbol.Value == SymbolReferenceType.RequiresDefinition)
{
FragmentsWithReferencedSymbols.Add(ReferencedSymbol.Key.Fragment);
}
}
}
// Aggregate headers are designed to explicitly include headers from the current module. Expand out a list of them, so they can be included when encountered.
HashSet <OutputFile> ExplicitIncludes = new HashSet <OutputFile>();
if ((InputFile.Flags & SourceFileFlags.Aggregate) != 0)
{
foreach (OutputFileInclude Include in Includes)
{
ExplicitIncludes.UnionWith(Include.ExpandedReferences.Where(x => x.File.InputFile.Location.IsUnderDirectory(InputFile.Location.Directory)).Select(x => x.File));
}
foreach (OutputFileInclude Include in Includes)
{
ExplicitIncludes.Remove(Include.TargetFile);
}
}
// Create the list of remaining dependencies for this file, and add any forward declarations
HashSet <SourceFragment> Dependencies = new HashSet <SourceFragment>();
List <Symbol> ForwardDeclarations = new List <Symbol>();
AddForwardDeclarations(InputFile, ForwardDeclarations, Dependencies, FwdSymbolToHeader);
// Reduce the list of includes to those that are required.
for (int FragmentIdx = InputFile.Fragments.Length - 1, IncludeIdx = Includes.Count - 1; FragmentIdx >= 0; FragmentIdx--)
{
// Update the dependency lists for this fragment
SourceFragment InputFragment = InputFile.Fragments[FragmentIdx];
if (InputFragment.Dependencies != null)
{
Dependencies.UnionWith(InputFragment.Dependencies);
}
Dependencies.Remove(InputFragment);
// Scan backwards through the list of includes, expanding each include to those which are required
int MarkupMin = (FragmentIdx == 0)? -1 : InputFragment.MarkupMin;
for (; IncludeIdx >= 0 && Includes[IncludeIdx].MarkupIdx >= MarkupMin; IncludeIdx--)
{
OutputFileInclude Include = Includes[IncludeIdx];
// Always include the same header for aggregates
if ((InputFile.Flags & SourceFileFlags.Aggregate) != 0)
{
Include.FinalFiles.Insert(0, Include.TargetFile);
Dependencies.ExceptWith(Include.TargetFile.IncludedFragments);
Dependencies.UnionWith(Include.TargetFile.Dependencies);
}
// Include any indirectly included files
for (int Idx = Include.ExpandedReferences.Count - 1; Idx >= 0; Idx--)
{
// Make sure we haven't already added it above
OutputFileReference Reference = Include.ExpandedReferences[Idx];
if (!Include.FinalFiles.Contains(Reference.File))
{
if (Dependencies.Any(x => Reference.UniqueFragments.Contains(x)) ||
(Reference.File.InputFile.Flags & SourceFileFlags.Pinned) != 0 ||
Reference == ForcedHeaderFileReference ||
Reference.File == MonolithicHeader ||
ExplicitIncludes.Contains(Reference.File) ||
((InputFile.Flags & SourceFileFlags.Aggregate) != 0 && Reference.File == Include.TargetFile) || // Always include the original header for aggregates. They are written explicitly to include certain files.
Reference.UniqueFragments.Any(x => FragmentsWithReferencedSymbols.Contains(x)))
{
Include.FinalFiles.Insert(0, Reference.File);
Dependencies.ExceptWith(Reference.File.IncludedFragments);
Dependencies.UnionWith(Reference.File.Dependencies);
}
}
}
}
}
// Remove any includes that are already included by the matching header
if (HeaderFile != null)
{
HashSet <OutputFile> HeaderIncludedFiles = new HashSet <OutputFile>(HeaderFile.Includes.SelectMany(x => x.FinalFiles));
foreach (OutputFileInclude Include in Includes)
{
Include.FinalFiles.RemoveAll(x => HeaderIncludedFiles.Contains(x));
}
}
// Check that all the dependencies have been satisfied
if (Dependencies.Count > 0)
{
// Find those which are completely invalid
List <SourceFragment> InvalidDependencies = Dependencies.Where(x => !InputFileStack.Contains(x.File)).ToList();
if (InvalidDependencies.Count > 0)
{
Log.WriteLine("warning: {0} does not include {1}{2}; may have missing dependencies.", InputFile, String.Join(", ", InvalidDependencies.Select(x => x.Location.FullName).Take(3)), (InvalidDependencies.Count > 3)? String.Format(" and {0} others", InvalidDependencies.Count - 3) : "");
}
Dependencies.ExceptWith(InvalidDependencies);
// Otherwise warn about those which were not pinned
foreach (SourceFile DependencyFile in Dependencies.Select(x => x.File))
{
Log.WriteLine("warning: {0} is included by {1} ({2}), but depends on it and should be marked as pinned.", InputFile, DependencyFile, String.Join(" -> ", InputFileStack.SkipWhile(x => x != DependencyFile).Select(x => x.Location.GetFileName())));
}
// Mark it as non-standalone and pinned
InputFile.Flags = (InputFile.Flags | SourceFileFlags.Pinned) & ~SourceFileFlags.Standalone;
}
// Do one more forward pass through all the headers, and remove anything that's included more than once. That can happen if we have a referenced symbol as well as
// an explicit include, for example.
HashSet <OutputFile> FinalIncludes = new HashSet <OutputFile>();
foreach (OutputFileInclude Include in Includes)
{
for (int Idx = 0; Idx < Include.FinalFiles.Count; Idx++)
{
if (!FinalIncludes.Add(Include.FinalFiles[Idx]))
{
Include.FinalFiles.RemoveAt(Idx);
Idx--;
}
}
}
// Create the optimized file
OutputFile OptimizedFile = new OutputFile(InputFile, Includes, Dependencies, ForwardDeclarations);
// Write the verbose log
InputFile.LogVerbose("");
foreach (OutputFile IncludedFile in OptimizedFile.IncludedFiles)
{
InputFile.LogVerbose("Output: {0}", IncludedFile.InputFile.Location.FullName);
}
// Return the optimized file
return(OptimizedFile);
}
/// <summary>
/// Find or create an output file for a corresponding input file
/// </summary>
/// <param name="InputFile">The input file</param>
/// <param name="IncludeStack">The current include stack</param>
/// <param name="OutputFiles">List of output files</param>
/// <param name="OutputFileLookup">Mapping from source file to output file</param>
/// <returns>The new or existing output file</returns>
static OutputFile FindOrCreateOutputFile(List <SourceFile> InputFileStack, Dictionary <SourceFile, SourceFile> CppFileToHeaderFile, HashList <OutputFile> PreviousFiles, Dictionary <SourceFile, OutputFile> OutputFileLookup, Dictionary <Symbol, OutputFile> FwdSymbolToHeader, bool bMakeStandalone, bool bUseOriginalIncludes, LineBasedTextWriter Log)
{
// Get the file at the top of the stack
SourceFile InputFile = InputFileStack[InputFileStack.Count - 1];
// Try to find an existing file
OutputFile OutputFile;
if (OutputFileLookup.TryGetValue(InputFile, out OutputFile))
{
if (OutputFile == null)
{
throw new Exception("Circular include dependencies are not allowed.");
}
foreach (OutputFile IncludedFile in OutputFile.OriginalIncludedFiles.Where(x => !PreviousFiles.Contains(x)))
{
PreviousFiles.Add(IncludedFile);
}
}
else
{
// Add a placeholder entry in the output file lookup, so we can detect circular include dependencies
OutputFileLookup[InputFile] = null;
// Duplicate the list of previously included files, so we can construct the
List <OutputFile> PreviousFilesCopy = new List <OutputFile>(PreviousFiles);
// Build a list of includes for this file. First include is a placeholder for any missing includes that need to be inserted.
List <OutputFileInclude> Includes = new List <OutputFileInclude>();
if ((InputFile.Flags & SourceFileFlags.External) == 0)
{
for (int MarkupIdx = 0; MarkupIdx < InputFile.Markup.Length; MarkupIdx++)
{
PreprocessorMarkup Markup = InputFile.Markup[MarkupIdx];
if (Markup.IsActive && Markup.IncludedFile != null && (Markup.IncludedFile.Flags & SourceFileFlags.Inline) == 0 && Markup.IncludedFile.Counterpart == null)
{
InputFileStack.Add(Markup.IncludedFile);
OutputFile IncludeFile = FindOrCreateOutputFile(InputFileStack, CppFileToHeaderFile, PreviousFiles, OutputFileLookup, FwdSymbolToHeader, bMakeStandalone, bUseOriginalIncludes, Log);
InputFileStack.RemoveAt(InputFileStack.Count - 1);
Includes.Add(new OutputFileInclude(MarkupIdx, IncludeFile));
}
}
}
// Find the matching header file
OutputFile HeaderFile = null;
if ((InputFile.Flags & SourceFileFlags.TranslationUnit) != 0)
{
SourceFile CandidateHeaderFile;
if (CppFileToHeaderFile.TryGetValue(InputFile, out CandidateHeaderFile) && (CandidateHeaderFile.Flags & SourceFileFlags.Standalone) != 0)
{
OutputFileLookup.TryGetValue(CandidateHeaderFile, out HeaderFile);
}
}
// Create the output file.
if ((InputFile.Flags & SourceFileFlags.Output) != 0 && !bUseOriginalIncludes)
{
OutputFile = CreateOptimizedOutputFile(InputFile, HeaderFile, PreviousFilesCopy, Includes, InputFileStack, FwdSymbolToHeader, bMakeStandalone, Log);
}
else
{
OutputFile = CreatePassthroughOutputFile(InputFile, Includes, Log);
}
// Replace the null entry in the output file lookup that we added earlier
OutputFileLookup[InputFile] = OutputFile;
// Add this file to the list of included files
PreviousFiles.Add(OutputFile);
// If the output file dependends on something on the stack, make sure it's marked as pinned
if ((InputFile.Flags & SourceFileFlags.Pinned) == 0)
{
SourceFragment Dependency = OutputFile.Dependencies.FirstOrDefault(x => InputFileStack.Contains(x.File) && x.File != InputFile);
if (Dependency != null)
{
throw new Exception(String.Format("'{0}' is not marked as pinned, but depends on '{1}' which includes it", InputFile.Location.GetFileName(), Dependency.UniqueName));
}
}
}
return(OutputFile);
}
/// <summary>
/// Wait for this worker to complete
/// </summary>
/// <param name="Writer">Writer for log output</param>
/// <returns>Return code from the thread</returns>
public SequenceProbeResult Join(LineBasedTextWriter Writer)
{
// Finish the task instance
BufferedTextWriter BufferedWriter = new BufferedTextWriter();
int ExitCode = ActiveInstance.Join(BufferedWriter);
ActiveInstance.Dispose();
ActiveInstance = null;
// Read the new state
FileReference OutputFile = new FileReference(TaskStateFile.FullName + ".out");
SequenceWorker NewWorker = SequenceWorker.Deserialize(OutputFile.FullName);
OutputFile.Delete();
// Make sure the exit code reflects the failure state. XGE can sometimes fail transferring back.
if (ExitCode == 0 && !NewWorker.bResult)
{
ExitCode = -1;
}
// If it's a hard failure, print the compile log to the regular log
if (ExitCode == -1)
{
Writer.WriteLine("Failed to compile {0}, exit code {1}:", UniqueName, ExitCode);
foreach (string Line in NewWorker.CompileLog)
{
Writer.WriteLine(" > {0}", Line.Replace("error:", "err:"));
}
}
// Annotate the log data if this is from a failed attempt. It still may be useful for debugging purposes.
if (ExitCode != 0)
{
NewWorker.SummaryLog.Insert(0, String.Format("ExitCode={0}", ExitCode));
NewWorker.SummaryLog = NewWorker.SummaryLog.Select(x => "FAIL > " + x).ToList();
NewWorker.CompileLog.Insert(0, String.Format("ExitCode={0}", ExitCode));
NewWorker.CompileLog = NewWorker.CompileLog.Select(x => "FAIL > " + x).ToList();
}
// Append the log data back to the local output
File.AppendAllLines(SummaryLogFile.FullName, NewWorker.SummaryLog);
NewWorker.SummaryLog.Clear();
File.AppendAllLines(CompileLogFile.FullName, NewWorker.CompileLog);
NewWorker.CompileLog.Clear();
// If we failed, return the
if (ExitCode != 0)
{
if (ExitCode == -1)
{
Writer.WriteLine("Warning: Failed to compile {0}, exit code {1}. Aborting.", UniqueName, ExitCode);
return(SequenceProbeResult.Failed);
}
else
{
Writer.WriteLine("Warning: Failed to compile {0}; exit code {1}. Will retry.", UniqueName, ExitCode);
return(SequenceProbeResult.FailedAllowRetry);
}
}
// Update the task
Worker = NewWorker;
// Check if this is just an incremental update
if (Type == SequenceProbeType.Verify)
{
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Return that we're done
return(SequenceProbeResult.Completed);
}
else if (Type == SequenceProbeType.Optimize)
{
if (Worker.RemainingFragmentCount > 0)
{
// Get the top-most fragment - the one we've just established is a dependency for this leaf node - and add it to the list of known dependencies
SourceFragment NextFragment = Fragments[Worker.RemainingFragmentCount - 1];
AddDependency(Worker, Fragments, Worker.RemainingFragmentCount - 1);
Worker.SummaryLog.Add(String.Format(" [Added {0}: {1}]", Worker.RemainingFragmentCount - 1, Fragments[Worker.RemainingFragmentCount - 1].Location));
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Otherwise, return that we've just updated
return(SequenceProbeResult.Updated);
}
else
{
// Save the task
Worker.Serialize(TaskStateFile.FullName);
// Return that we're done
SetCompletedDependencies();
return(SequenceProbeResult.Completed);
}
}
else
{
throw new NotImplementedException();
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="Type">The type of probe to create</param>
/// <param name="Node">The node to optimize</param>
/// <param name="IntermediateDir">Directory for intermediate files</param>
/// <param name="UniqueName">Unique name prefix for all temporary files</param>
public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName)
{
this.Type = Type;
this.IntermediateDir = IntermediateDir;
this.Fragments = Fragments;
this.LastFragment = Fragments[Fragments.Length - 1];
this.UniqueName = UniqueName;
this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state"));
this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt");
this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt");
// Get the file to use for trying to compile different permutations
FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation");
// Create the response file
FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response");
CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment);
if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang)
{
WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/')));
}
else
{
WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI");
WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName));
WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null));
}
WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile);
string ResponseFileDigest = Utility.ComputeDigest(ResponseFile);
// Keep track of the include stack, so we can format the flat fragment list with context
int IncludeHistoryIdx = 0;
List <SourceFile> IncludeStack = new List <SourceFile>();
// Create the script for the probe
List <Tuple <int, string> > Lines = new List <Tuple <int, string> >();
for (int Idx = 0; Idx < Fragments.Length; Idx++)
{
SourceFragment Fragment = Fragments[Idx];
// Figure out which tag it's bound to
int Tag = (Fragment.File.Counterpart == null)? Idx : -1;
// Update the stack for new includes
while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx)
{
if (IncludeHistory[IncludeHistoryIdx].Item2 == null)
{
SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1];
IncludeStack.RemoveAt(IncludeStack.Count - 1);
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeHistoryIdx++;
}
else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null)
{
IncludeHistoryIdx += 2;
}
else
{
SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2;
Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName)));
IncludeStack.Add(IncludeFile);
IncludeHistoryIdx++;
}
}
// Get the indent at this point
string Indent = new string(' ', (IncludeStack.Count - 0) * 4);
// Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments
// if the heuristic for detecting when to use them doesn't work.
if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0)
{
foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
{
if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration))
{
Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration));
}
}
}
// Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work
DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x));
if (BaseSystemIncludePath != null)
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath))));
}
else
{
Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName)));
}
}
// Create the new task
string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray();
string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray();
Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName);
AddDependency(Worker, Fragments, Fragments.Length - 1);
// Log the referenced symbols
if (LastFragment.ReferencedSymbols.Count > 0)
{
List <string> ReferenceLog = new List <string>();
ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location));
foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name))
{
Symbol ReferencedSymbol = Pair.Key;
ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location));
}
ReferenceLog.Add("");
Worker.SummaryLog.InsertRange(0, ReferenceLog);
}
// Check to see if an existing version of the task exists which we can continue
if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists())
{
// Try to read the old task
SequenceWorker OldWorker;
try
{
OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName);
}
catch (Exception)
{
OldWorker = null;
}
// If it succeeded, compare it to the new task
if (OldWorker != null)
{
SequenceWorker NewWorker = Worker;
// Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task
HashSet <string> IgnoreFragments = new HashSet <string>();
for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++)
{
if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount)
{
IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]);
}
}
IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x]));
// Compute digests for the old and new tasks
string OldDigest = CreateDigest(OldWorker, IgnoreFragments);
string NewDigest = CreateDigest(NewWorker, IgnoreFragments);
if (OldDigest == NewDigest)
{
// Build a map of fragment file names to their new index
Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++)
{
string FragmentFileName = FragmentFileNames[Idx];
FragmentFileNameToNewIndex[FragmentFileName] = Idx;
}
// Add known dependencies to the new worker
foreach (int OldKnownDependency in OldWorker.KnownDependencies)
{
string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency];
int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName];
NewWorker.KnownDependencies.Add(NewKnownDependency);
}
// Update the remaining count. All these fragments must match, because they're not part of the ignore list.
NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount;
}
}
}
// If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare.
FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location;
if (MainFileLocation.HasExtension(".cpp"))
{
string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h");
for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++)
{
if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0)
{
AddDependency(Worker, Fragments, FragmentIdx);
}
}
}
// Update the finished fragment if we're done, otherwise clear out all the intermediate files
if (Worker.RemainingFragmentCount == 0)
{
SetCompletedDependencies();
}
else
{
Worker.Serialize(TaskStateFile.FullName);
PermutationFile.Delete();
SummaryLogFile.Delete();
CompileLogFile.Delete();
}
}
/// <summary>
/// Read all the exported symbol definitions from a file
/// </summary>
/// <param name="File">The file to parse</param>
/// <param name="MarkupIdx">The current index into the markup array of this file</param>
/// <param name="Scope">The scoping depth, ie. number of unmatched '{' tokens encountered in source code</param>
void ReadExportedSymbols(SourceFile File, ref int MarkupIdx, ref int Scope)
{
// Ignore files which only have a header guard; there are no fragments that we can link to
if (File.BodyMinIdx == File.BodyMaxIdx)
{
return;
}
// Read markup from the file
for (; MarkupIdx < File.Markup.Length; MarkupIdx++)
{
PreprocessorMarkup Markup = File.Markup[MarkupIdx];
if (Markup.Type == PreprocessorMarkupType.Define)
{
// If the macro is not already defined, add it
if (Markup.Tokens[0].Text != "LOCTEXT_NAMESPACE")
{
SourceFragment Fragment = FindFragment(File, MarkupIdx);
AddSymbol(Markup.Tokens[0].Text, SymbolType.Macro, null, Fragment, Markup.Location);
}
}
else if (Markup.Type == PreprocessorMarkupType.If && Markup.Tokens.Count == 2 && Markup.Tokens[0].Text == "!" && Markup.Tokens[1].Text == "CPP")
{
// Completely ignore these blocks; they sometimes differ in class/struct usage, and exist solely for noexport types to be parsed by UHT
int Depth = 1;
while (Depth > 0)
{
Markup = File.Markup[++MarkupIdx];
if (Markup.Type == PreprocessorMarkupType.If || Markup.Type == PreprocessorMarkupType.Ifdef || Markup.Type == PreprocessorMarkupType.Ifndef)
{
Depth++;
}
else if (Markup.Type == PreprocessorMarkupType.Endif)
{
Depth--;
}
}
}
else if (Markup.Type == PreprocessorMarkupType.If || Markup.Type == PreprocessorMarkupType.Ifdef || Markup.Type == PreprocessorMarkupType.Ifndef)
{
// Loop through all the branches of this conditional block, saving the resulting scope of each derivation
List <int> NewScopes = new List <int>();
while (Markup.Type != PreprocessorMarkupType.Endif)
{
// Skip over the conditional directive
MarkupIdx++;
// Read any exported symbols from this conditional block
int NewScope = Scope;
ReadExportedSymbols(File, ref MarkupIdx, ref NewScope);
NewScopes.Add(NewScope);
// Get the new preprocessor markup
Markup = File.Markup[MarkupIdx];
}
// Make sure they were all consistent
if (NewScopes.Any(x => x != NewScopes[0]))
{
throw new Exception(String.Format("Unbalanced scope depth between conditional block derivations while parsing {0}", File.Location));
}
// Update the scope
Scope = NewScopes[0];
}
else if (Markup.Type == PreprocessorMarkupType.Else || Markup.Type == PreprocessorMarkupType.Elif || Markup.Type == PreprocessorMarkupType.Endif)
{
// We've reached the end of this block; return success
break;
}
else if (Markup.Type == PreprocessorMarkupType.Text)
{
// Read the declarations in this block
SourceFragment Fragment = FindFragment(File, MarkupIdx);
ParseDeclarations(File, Fragment, File.Markup[MarkupIdx], ref Scope);
}
}
}
/// <summary>
/// Try to read a template class or struct header
/// </summary>
/// <param name="Reader">Tokens to parse</param>
/// <param name="Fragment">Fragment containing these tokens</param>
bool ReadTemplateClassOrStructHeader(TokenReader OriginalReader, SourceFragment Fragment)
{
TokenReader Reader = new TokenReader(OriginalReader);
// Check for the template keyword
if (Reader.Current.Text != "template")
{
return(false);
}
// Create a buffer to store the template prefix
List <Token> Tokens = new List <Token>();
Tokens.Add(Reader.Current);
// Check for the opening argument list
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "<")
{
return(false);
}
// Read the argument list, keeping track of any symbols referenced along the way
while (Tokens[Tokens.Count - 1].Text != ">")
{
Tokens.Add(Reader.Current);
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
}
// Get the symbol type
SymbolType Type;
if (Reader.Current.Text == "class")
{
Type = SymbolType.TemplateClass;
}
else if (Reader.Current.Text == "struct")
{
Type = SymbolType.TemplateStruct;
}
else
{
return(false);
}
// Add the class or struct keyword
Tokens.Add(Reader.Current);
// Move to the name
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Skip over an optional DLL export declaration
if (Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API") && !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Read the class name and check it's followed by a class body or inheritance list
string Name = Reader.Current.Text;
if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
if (Reader.Current.Text != ":" && Reader.Current.Text != "{")
{
return(false);
}
// Create the symbol.
if (Rules.AllowSymbol(Name))
{
// Only allow forward declarations of templates with class and typename arguments and no defaults (ie. no enums or class names which may have to be forward-declared separately).
string ForwardDeclaration = null;
if (!Tokens.Any(x => x.Text == "="))
{
ForwardDeclaration = String.Format("{0} {1};", Token.Format(Tokens), Name);
for (int Idx = 2; Idx < Tokens.Count - 2; Idx += 3)
{
if (Tokens[Idx].Text != "class" && Tokens[Idx].Text != "typename")
{
ForwardDeclaration = null;
break;
}
}
}
AddSymbol(Name, Type, ForwardDeclaration, Fragment, OriginalReader.TokenLocation);
}
// Move the original reader past the declaration
OriginalReader.Set(Reader);
return(true);
}
/// <summary>
/// Parse an "enum class" declaration, and add a symbol for it
/// </summary>
/// <param name="Reader">Tokens to be parsed. On success, this is assigned to a new </param>
/// <param name="Fragment">Fragment containing these tokens</param>
bool ReadEnumClassHeader(TokenReader OriginalReader, SourceFragment Fragment)
{
TokenReader Reader = new TokenReader(OriginalReader);
// Read the UENUM prefix if present. We don't want to forward-declare types that need to be parsed by UHT, because it needs the definition.
bool bIsUENUM = false;
if (Reader.Current.Text == "UENUM")
{
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "(")
{
return(false);
}
while (Reader.Current.Text != ")")
{
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
}
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
bIsUENUM = true;
}
// Read the 'enum class' tokens
if (Reader.Current.Text != "enum" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
if (Reader.Current.Text != "class" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
// Read the name, make sure we haven't read a definition for it already, and check it's an enum declaration
string Name = Reader.Current.Text;
if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
if (Reader.Current.Text != ";" && Reader.Current.Text != ":" && Reader.Current.Text != "{")
{
return(false);
}
// Build the forward declaration for it. Don't forward-declare UENUM types because UHT needs to parse their definition first.
string ForwardDeclaration = null;
if (!bIsUENUM)
{
StringBuilder ForwardDeclarationBuilder = new StringBuilder();
ForwardDeclarationBuilder.AppendFormat("enum class {0}", Name);
while (Reader.Current.Text != ";" && Reader.Current.Text != "{")
{
// Append the next token
if (Reader.Current.HasLeadingSpace)
{
ForwardDeclarationBuilder.Append(" ");
}
ForwardDeclarationBuilder.Append(Reader.Current.Text);
// Try to move to the next token
if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines))
{
return(false);
}
}
ForwardDeclarationBuilder.Append(";");
ForwardDeclaration = ForwardDeclarationBuilder.ToString();
}
// Create a symbol for it if it's an actual definition rather than a forward declaration
if (Reader.Current.Text == "{" && Rules.AllowSymbol(Name))
{
AddSymbol(Name, SymbolType.Enumeration, ForwardDeclaration, Fragment, OriginalReader.TokenLocation);
}
// Update the original reader to be the new location
OriginalReader.Set(Reader);
return(true);
}
