/// <summary> /// Try to skip over a template header /// </summary> /// <param name="OriginalReader">Tokens to parse</param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadTypedefHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Check for the typedef keyword Token PreviousToken = Reader.Current; if (Reader.Current.Text != "typedef" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Scan to the next semicolon while (Reader.MoveNext(TokenReaderContext.IgnoreNewlines) && Reader.Current.Text != ";" && Reader.Current.Text != "{") { PreviousToken = Reader.Current; } // Ignore 'typedef struct' and 'typedef union' declarations. if (Reader.Current.Text == "{") { return(false); } // Try to add a symbol for the previous token. If it already exists, replace it. if (PreviousToken.Type == TokenType.Identifier && Rules.AllowSymbol(PreviousToken.Text)) { AddSymbol(PreviousToken.Text, SymbolType.Typedef, null, Fragment, OriginalReader.TokenLocation); } // Move the original reader past the declaration OriginalReader.Set(Reader); return(true); }
/// <summary> /// Parse declarations from the reader /// </summary> /// <param name="File">The file containing the declarations</param> /// <param name="MarkupIdx">The markup to parse declarations from</param> /// <param name="Scope">The scope depth; the number of unmatched opening brace tokens</param> void ParseDeclarations(SourceFile File, SourceFragment Fragment, PreprocessorMarkup Markup, ref int Scope) { TokenReader Reader = new TokenReader(File.Text, Markup.Location, Markup.EndLocation); if (Reader.MoveNext()) { for (;;) { if (Scope > 0 || (!ReadEnumClassHeader(Reader, Fragment) && !ReadClassOrStructHeader(Reader, Fragment) && !ReadTypedefHeader(Reader, Fragment) && !ReadTemplateClassOrStructHeader(Reader, Fragment) && !SkipTemplateHeader(Reader))) { // Update the scope depth if (Reader.Current.Text == "{") { Scope++; } else if (Reader.Current.Text == "}" && --Scope < 0) { throw new Exception(String.Format("Unbalanced '}}' while parsing '{0}'", File.Location)); } // Move to the next token if (!Reader.MoveNext()) { break; } } } } }
/// <summary> /// Creates an output file which represents the same includes as the inpu tfile /// </summary> /// <param name="InputFile">The input file that this output file corresponds to</param> /// <param name="Includes">The active set of includes parsed for this file</param> /// <param name="Log">Writer for log messages</param> public static OutputFile CreatePassthroughOutputFile(SourceFile InputFile, List <OutputFileInclude> Includes, LineBasedTextWriter Log) { // Write the state InputFile.LogVerbose("InputFile={0}", InputFile.Location.FullName); InputFile.LogVerbose("Duplicate."); // Reduce the list of includes to those that are required. HashSet <SourceFragment> Dependencies = new HashSet <SourceFragment>(); for (int FragmentIdx = InputFile.Fragments.Length - 1, IncludeIdx = Includes.Count - 1; FragmentIdx >= 0; FragmentIdx--) { // Update the dependency lists for this fragment SourceFragment InputFragment = InputFile.Fragments[FragmentIdx]; if (InputFragment.Dependencies != null) { Dependencies.UnionWith(InputFragment.Dependencies); } Dependencies.Remove(InputFragment); // Scan backwards through the list of includes, expanding each include to those which are required int MarkupMin = (FragmentIdx == 0)? -1 : InputFragment.MarkupMin; for (; IncludeIdx >= 0 && Includes[IncludeIdx].MarkupIdx >= MarkupMin; IncludeIdx--) { OutputFileInclude Include = Includes[IncludeIdx]; Include.FinalFiles.Add(Include.TargetFile); Dependencies.ExceptWith(Include.TargetFile.IncludedFragments); Dependencies.UnionWith(Include.TargetFile.Dependencies); } } // Create the optimized file return(new OutputFile(InputFile, Includes, new HashSet <SourceFragment>(), new List <Symbol>())); }
/// <summary> /// Constructor /// </summary> /// <param name="Name">Symbol name</param> /// <param name="Type">Type of the symbol</param> /// <param name="ForwardDeclaration">Text for a forward declaration of this symbol</param> /// <param name="Fragment">File that this symbol is defined in</param> /// <param name="Location">Location within the file that this symbol is declared</param> public Symbol(string Name, SymbolType Type, string ForwardDeclaration, SourceFragment Fragment, TextLocation Location) { this.Name = Name; this.Type = Type; this.ForwardDeclaration = ForwardDeclaration; this.Fragment = Fragment; this.Location = Location; }
/// <summary> /// Try to read a class or struct header /// </summary> /// <param name="Reader">Tokens to parse</param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadClassOrStructHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Make sure it's the right type SymbolType Type; if (Reader.Current.Text == "class") { Type = SymbolType.Class; } else if (Reader.Current.Text == "struct") { Type = SymbolType.Struct; } else { return(false); } // Move to the name string ClassOrStructKeyword = Reader.Current.Text; if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Skip over an optional DLL export declaration if (Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API") && !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Read the class name string Name = Reader.Current.Text; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != ":" && Reader.Current.Text != "{") { return(false); } // Create the symbol if (Rules.AllowSymbol(Name)) { string ForwardDeclaration = String.Format("{0} {1};", ClassOrStructKeyword, Name); AddSymbol(Name, Type, ForwardDeclaration, Fragment, OriginalReader.TokenLocation); } // Move the original reader past the declaration OriginalReader.Set(Reader); return(true); }
/// <summary> /// Parse an "enum class" declaration, and add a symbol for it /// </summary> /// <param name="Reader">Tokens to be parsed. On success, this is assigned to a new </param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadEnumClassHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Read the 'enum class' tokens if (Reader.Current.Text != "enum" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != "class" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Read the name, make sure we haven't read a definition for it already, and check it's an enum declaration string Name = Reader.Current.Text; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != ";" && Reader.Current.Text != ":" && Reader.Current.Text != "{") { return(false); } // Build the forward declaration for it StringBuilder ForwardDeclaration = new StringBuilder(); ForwardDeclaration.AppendFormat("enum class {0}", Name); while (Reader.Current.Text != ";" && Reader.Current.Text != "{") { // Append the next token if (Reader.Current.HasLeadingSpace) { ForwardDeclaration.Append(" "); } ForwardDeclaration.Append(Reader.Current.Text); // Try to move to the next token if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } ForwardDeclaration.Append(";"); // Create a symbol for it if it's an actual definition rather than a forward declaration if (Reader.Current.Text == "{" && Rules.AllowSymbol(Name)) { AddSymbol(Name, SymbolType.Enumeration, ForwardDeclaration.ToString(), Fragment, OriginalReader.TokenLocation); } // Update the original reader to be the new location OriginalReader.Set(Reader); return(true); }
/// <summary> /// Determines whether a symbol reference should be ignored, because it's to an item defined above in the same file /// </summary> /// <param name="Symbol">The referenced symbol</param> /// <param name="Fragment">The current fragment being parsed</param> /// <param name="Location">Position within the current fragment</param> /// <returns>True if the symbol should be ignored</returns> static bool IgnoreSymbol(Symbol Symbol, SourceFragment Fragment, TextLocation Location) { if (Symbol.Fragment.File == Fragment.File && Location > Symbol.Location) { return(true); } else { return(false); } }
/// <summary> /// Update the list of known dependencies to include dependencies of our already known dependencies /// </summary> /// <returns>True if the dependencies were updated, false if a dependency was encountered which is not available to this fragment</returns> void UpdateDependencies(LineBasedTextWriter Log) { // Update the task to account for any new dependencies that may have been found. Loop back through all the fragments so we include // dependencies we find along the way. for (int FragmentIdx = Worker.FragmentCount - 1; FragmentIdx > 0; FragmentIdx--) { SourceFragment Fragment = Fragments[FragmentIdx]; if (Worker.KnownDependencies.Contains(FragmentIdx) && Fragment.Dependencies != null) { // Include the dependencies of this fragment in our dependencies list List <SourceFragment> MissingDependencies = new List <SourceFragment>(); foreach (SourceFragment Dependency in Fragment.Dependencies) { int DependencyIdx = Array.IndexOf(Fragments, Dependency); if (DependencyIdx == -1 || DependencyIdx > FragmentIdx) { MissingDependencies.Add(Dependency); } else { Worker.KnownDependencies.Add(DependencyIdx); } } // If any were missing (because they were calculated by a different derivation), provide a useful diagnostic. Otherwise re-queue the worker // to find the next dependency. if (MissingDependencies.Count > 0 && MissingDependencies.Any(x => !WarnedMissingDependencies.Contains(x))) { StringBuilder Message = new StringBuilder(); Log.WriteLine("warning: could not find dependencies of '{0}':", Fragment.Location.FullName); foreach (SourceFragment MissingDependency in MissingDependencies) { Log.WriteLine(" {0}", MissingDependency.Location); } Log.WriteLine("In fragments for '{0}':", LastFragment.Location); for (int Idx = 0; Idx < Worker.RemainingFragmentCount; Idx++) { Log.WriteLine(" {0,3}: {1}", Idx, Fragments[Idx].Location.FullName); } for (int Idx = Worker.RemainingFragmentCount; Idx < Fragments.Length; Idx++) { Log.WriteLine(" {0,3}: ({1})", Idx, Fragments[Idx].Location.FullName); } WarnedMissingDependencies.UnionWith(MissingDependencies); } } } }
/// <summary> /// Adds a symbol to the lookup. If the symbol already exists and is marked as a different type, mark it as ambiguous. /// </summary> /// <param name="Name">The symbol name</param> /// <param name="Type">Type of the symbol</param> /// <param name="ForwardDeclaration">Code used to declare a forward declaration</param> /// <param name="Fragment">The current fragment</param> /// <param name="Location">Location within the file that this symbol is declared</param> Symbol AddSymbol(string Name, SymbolType Type, string ForwardDeclaration, SourceFragment Fragment, TextLocation Location) { // Try to find an existing symbol which matches foreach (Symbol ExistingSymbol in Lookup.WithKey(Name)) { if (ExistingSymbol.Type == Type && ExistingSymbol.Fragment == Fragment) { return(ExistingSymbol); } } // Otherwise create a new one Symbol NewSymbol = new Symbol(Name, Type, ForwardDeclaration, Fragment, Location); Lookup.Add(Name, NewSymbol); return(NewSymbol); }
/// <summary> /// Adds a known dependency to the given task /// </summary> /// <param name="Worker">The task to add the dependency to</param> /// <param name="Fragments">The array of fragments</param> /// <param name="FragmentIdx">Index of the dependency</param> static void AddDependency(SequenceWorker Worker, SourceFragment[] Fragments, int FragmentIdx) { // Add the dependency Worker.KnownDependencies.Add(FragmentIdx); // Check for any macro definitions that the given fragment depends on SourceFragment Fragment = Fragments[FragmentIdx]; foreach (Symbol ReferencedSymbol in Fragment.ReferencedSymbols.Keys) { if (ReferencedSymbol.Type == SymbolType.Macro) { int ReferencedFragmentIdx = Array.IndexOf(Fragments, ReferencedSymbol.Fragment); if (ReferencedFragmentIdx != -1 && ReferencedFragmentIdx < FragmentIdx) { Worker.KnownDependencies.Add(ReferencedFragmentIdx); } } } // Force dependencies onto pinned fragments. This can make a difference when templated functions are implemented in pinned headers // but declared in another (eg. LegacyText.h). The file will compile fine without the implementations being present, expecting them to be linked // into another object file, but fail when they are included due to being pinned later because MSVC only parses the token stream then. for (int MarkupIdx = Fragment.MarkupMin; MarkupIdx < Fragment.MarkupMax; MarkupIdx++) { PreprocessorMarkup Markup = Fragment.File.Markup[MarkupIdx]; if (Markup.Type == PreprocessorMarkupType.Include && Markup.IncludedFile != null && (Markup.IncludedFile.Flags & SourceFileFlags.Pinned) != 0 && Markup.IncludedFile.Counterpart == null) { foreach (SourceFragment PinnedFragment in Markup.IncludedFile.Fragments) { int PinnedFragmentIdx = Array.IndexOf(Fragments, PinnedFragment); if (PinnedFragmentIdx != -1 && PinnedFragmentIdx < FragmentIdx) { AddDependency(Worker, Fragments, PinnedFragmentIdx); } } } } }
/// <summary> /// Finds all the imported symbols from the given fragment /// </summary> /// <param name="Fragment">Fragment to find symbols for</param> /// <returns>Array of unique symbols for this fragment, which are not already forward-declared</returns> public Dictionary <Symbol, SymbolReferenceType> FindReferences(SourceFragment Fragment) { Dictionary <Symbol, SymbolReferenceType> References = new Dictionary <Symbol, SymbolReferenceType>(); TextBuffer Text = Fragment.File.Text; if (Text != null && Fragment.MarkupMax > Fragment.MarkupMin) { TokenReader Reader = new TokenReader(Fragment.File.Text, Fragment.MinLocation, Fragment.MaxLocation); for (bool bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); bMoveNext;) { // Read the current token, and immediately move to the next so that we can lookahead if we need to Token Token = Reader.Current; bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); // Check if the previous token is a symbol if (Token.Type == TokenType.Identifier) { if (Token.Text == "class" || Token.Text == "struct") { // Skip over "class", "struct" and "enum class" declarations and forward declarations. They're not actually referencing another symbol. if (bMoveNext && Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API")) { bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); } if (bMoveNext && Reader.Current.Type == TokenType.Identifier) { bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); } } else if (IsSmartPointerClass(Token.Text) && bMoveNext && Reader.Current.Text == "<") { // Smart pointer types which do not require complete declarations bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); if (bMoveNext && Reader.Current.Type == TokenType.Identifier) { foreach (Symbol MatchingSymbol in Lookup.WithKey(Reader.Current.Text)) { if ((MatchingSymbol.Type == SymbolType.Class || MatchingSymbol.Type == SymbolType.Struct) && !References.ContainsKey(MatchingSymbol) && !IgnoreSymbol(MatchingSymbol, Fragment, Reader.TokenLocation)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); } } else { // Otherwise check what type of symbol it is (if any), and whether it's referenced in an opaque way (though a pointer/reference or so on). foreach (Symbol MatchingSymbol in Lookup.WithKey(Token.Text)) { if (IgnoreSymbol(MatchingSymbol, Fragment, Reader.TokenLocation)) { continue; } else if (MatchingSymbol.Type == SymbolType.Macro) { // Add the symbol if it doesn't already exist if (!References.ContainsKey(MatchingSymbol)) { References.Add(MatchingSymbol, SymbolReferenceType.RequiresDefinition); } } else if (MatchingSymbol.Type == SymbolType.Enumeration) { // Check whether we're using the type in an opaque way, or we actually require the definition if (!bMoveNext || Reader.Current.Text == "::") { References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition; } else if (!References.ContainsKey(MatchingSymbol)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } else if (MatchingSymbol.Type == SymbolType.Class || MatchingSymbol.Type == SymbolType.Struct) { // Check whether we're declaring a pointer to this type, and just need a forward declaration if (!bMoveNext || (Reader.Current.Text != "*" && Reader.Current.Text != "&")) { References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition; } else if (!References.ContainsKey(MatchingSymbol)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } else if (MatchingSymbol.Type == SymbolType.TemplateClass || MatchingSymbol.Type == SymbolType.TemplateStruct) { // Check whether we're declaring a pointer to this type, and just need a forward declaration TokenReader Lookahead = new TokenReader(Reader); bool bLookaheadMoveNext = bMoveNext; if (!bMoveNext || !SkipTemplateArguments(Lookahead, ref bLookaheadMoveNext) || (Lookahead.Current.Text != "*" && Lookahead.Current.Text != "&")) { References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition; } else if (!References.ContainsKey(MatchingSymbol)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } } } } } } return(References); }
/// <summary> /// Creates an optimized output file /// </summary> /// <param name="InputFile">The input file that this output file corresponds to</param> /// <param name="HeaderFile">The corresponding header file</param> /// <param name="PreviousFiles">List of files parsed before this one</param> /// <param name="Includes">The active set of includes parsed for this file</param> /// <param name="InputFileStack">The active include stack</param> /// <param name="FwdSymbolToHeader"></param> /// <param name="bMakeStandalone">Whether to make this output file standalone</param> /// <param name="Log">Writer for log messages</param> public static OutputFile CreateOptimizedOutputFile(SourceFile InputFile, OutputFile HeaderFile, List <OutputFile> PreviousFiles, List <OutputFileInclude> Includes, List <SourceFile> InputFileStack, Dictionary <Symbol, OutputFile> FwdSymbolToHeader, bool bMakeStandalone, LineBasedTextWriter Log) { Debug.Assert(HeaderFile == null || (InputFile.Flags & SourceFileFlags.TranslationUnit) != 0); // Write the state InputFile.LogVerbose("InputFile={0}", InputFile.Location.FullName); InputFile.LogVerbose("InputFile.Flags={0}", InputFile.Flags.ToString()); if (HeaderFile != null) { InputFile.LogVerbose("HeaderFile={0}", HeaderFile.InputFile.Location.FullName); InputFile.LogVerbose("HeaderFile.Flags={0}", HeaderFile.InputFile.Flags.ToString()); } InputFile.LogVerbose(""); for (int Idx = 0; Idx < InputFileStack.Count; Idx++) { InputFile.LogVerbose("InputFileStack[{0}]={1}", Idx, InputFileStack[Idx].Location.FullName); } InputFile.LogVerbose(""); for (int Idx = 0; Idx < PreviousFiles.Count; Idx++) { InputFile.LogVerbose("PreviousFiles[{0}]={1}", Idx, PreviousFiles[Idx].InputFile.Location.FullName); } InputFile.LogVerbose(""); for (int Idx = 0; Idx < Includes.Count; Idx++) { } InputFile.LogVerbose(""); for (int Idx = 0; Idx < Includes.Count; Idx++) { OutputFileInclude Include = Includes[Idx]; InputFile.LogVerbose("Includes[{0}]={1}", Idx, Includes[Idx].TargetFile.InputFile.Location.FullName); foreach (SourceFragment Fragment in Include.FinalFiles.SelectMany(x => x.IncludedFragments)) { InputFile.LogVerbose("Includes[{0}].FinalFiles.IncludedFragments={1}", Idx, Fragment); } } // Traverse through all the included headers, figuring out the first unique include for each file and fragment HashSet <OutputFile> VisitedFiles = new HashSet <OutputFile>(); HashSet <SourceFragment> VisitedFragments = new HashSet <SourceFragment>(); // Go through the standalone headers first OutputFile MonolithicHeader = null; if (HeaderFile == null && (InputFile.Flags & SourceFileFlags.Standalone) != 0 && (InputFile.Flags & SourceFileFlags.External) == 0 && (InputFile.Flags & SourceFileFlags.Aggregate) == 0) { // Insert a dummy include to receive all the inserted headers OutputFileInclude ImplicitInclude = new OutputFileInclude(-1, null); ImplicitInclude.ExpandedReferences = new List <OutputFileReference>(); Includes.Insert(0, ImplicitInclude); // Determine which monolithic header to use IEnumerable <OutputFile> PotentialMonolithicHeaders = PreviousFiles.Union(Includes.Select(x => x.TargetFile).Where(x => x != null).SelectMany(x => x.IncludedFiles)); if (InputFile.Module != null && InputFile.Module.PublicDependencyModules.Union(InputFile.Module.PrivateDependencyModules).Any(x => x.Name == "Core")) { MonolithicHeader = PotentialMonolithicHeaders.FirstOrDefault(x => (x.InputFile.Flags & SourceFileFlags.IsCoreMinimal) != 0); } else { MonolithicHeader = PotentialMonolithicHeaders.FirstOrDefault(x => (x.InputFile.Flags & SourceFileFlags.IsCoreTypes) != 0); } // Update the dependencies to treat all the contents of a monolithic header as pinned if (MonolithicHeader != null) { SourceFragment[] UniqueFragments = MonolithicHeader.IncludedFragments.Except(VisitedFragments).ToArray(); ImplicitInclude.ExpandedReferences.Add(new OutputFileReference(MonolithicHeader, UniqueFragments)); VisitedFragments.UnionWith(UniqueFragments); VisitedFiles.Add(MonolithicHeader); } // Insert all the forward declaration headers, but only treat them as supplying the forward declarations themselves. They may happen to include // some utility classes (eg. TSharedPtr), and we don't want to include an unrelated header to satisfy that dependency. foreach (OutputFile FwdHeader in FwdSymbolToHeader.Values) { FindExpandedReferences(FwdHeader, ImplicitInclude.ExpandedReferences, VisitedFiles, VisitedFragments, true); } // Add all the other files if (bMakeStandalone) { foreach (OutputFile PreviousFile in PreviousFiles) { if ((InputFile.Flags & SourceFileFlags.Standalone) != 0 && (PreviousFile.InputFile.Flags & SourceFileFlags.Inline) == 0 && (PreviousFile.InputFile.Flags & SourceFileFlags.Pinned) == 0 && VisitedFiles.Add(PreviousFile)) { SourceFragment[] UniqueFragments = PreviousFile.IncludedFragments.Except(VisitedFragments).ToArray(); ImplicitInclude.ExpandedReferences.Add(new OutputFileReference(PreviousFile, UniqueFragments)); VisitedFragments.UnionWith(UniqueFragments); } } } } // Figure out a list of files which are uniquely reachable through each include. Force an include of the matching header as the first thing. OutputFileReference ForcedHeaderFileReference = null; foreach (OutputFileInclude Include in Includes) { if (Include.ExpandedReferences == null) { Include.ExpandedReferences = new List <OutputFileReference>(); if (Include == Includes[0] && HeaderFile != null) { ForcedHeaderFileReference = new OutputFileReference(HeaderFile, HeaderFile.IncludedFragments); Include.ExpandedReferences.Add(ForcedHeaderFileReference); VisitedFragments.UnionWith(HeaderFile.IncludedFragments); } FindExpandedReferences(Include.TargetFile, Include.ExpandedReferences, VisitedFiles, VisitedFragments, true); } } // Find all the symbols which are referenced by this file HashSet <SourceFragment> FragmentsWithReferencedSymbols = new HashSet <SourceFragment>(); foreach (SourceFragment Fragment in InputFile.Fragments) { foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols) { if (ReferencedSymbol.Value == SymbolReferenceType.RequiresDefinition) { FragmentsWithReferencedSymbols.Add(ReferencedSymbol.Key.Fragment); } } } // Aggregate headers are designed to explicitly include headers from the current module. Expand out a list of them, so they can be included when encountered. HashSet <OutputFile> ExplicitIncludes = new HashSet <OutputFile>(); if ((InputFile.Flags & SourceFileFlags.Aggregate) != 0) { foreach (OutputFileInclude Include in Includes) { ExplicitIncludes.UnionWith(Include.ExpandedReferences.Where(x => x.File.InputFile.Location.IsUnderDirectory(InputFile.Location.Directory)).Select(x => x.File)); } foreach (OutputFileInclude Include in Includes) { ExplicitIncludes.Remove(Include.TargetFile); } } // Create the list of remaining dependencies for this file, and add any forward declarations HashSet <SourceFragment> Dependencies = new HashSet <SourceFragment>(); List <Symbol> ForwardDeclarations = new List <Symbol>(); AddForwardDeclarations(InputFile, ForwardDeclarations, Dependencies, FwdSymbolToHeader); // Reduce the list of includes to those that are required. for (int FragmentIdx = InputFile.Fragments.Length - 1, IncludeIdx = Includes.Count - 1; FragmentIdx >= 0; FragmentIdx--) { // Update the dependency lists for this fragment SourceFragment InputFragment = InputFile.Fragments[FragmentIdx]; if (InputFragment.Dependencies != null) { Dependencies.UnionWith(InputFragment.Dependencies); } Dependencies.Remove(InputFragment); // Scan backwards through the list of includes, expanding each include to those which are required int MarkupMin = (FragmentIdx == 0)? -1 : InputFragment.MarkupMin; for (; IncludeIdx >= 0 && Includes[IncludeIdx].MarkupIdx >= MarkupMin; IncludeIdx--) { OutputFileInclude Include = Includes[IncludeIdx]; // Always include the same header for aggregates if ((InputFile.Flags & SourceFileFlags.Aggregate) != 0) { Include.FinalFiles.Insert(0, Include.TargetFile); Dependencies.ExceptWith(Include.TargetFile.IncludedFragments); Dependencies.UnionWith(Include.TargetFile.Dependencies); } // Include any indirectly included files for (int Idx = Include.ExpandedReferences.Count - 1; Idx >= 0; Idx--) { // Make sure we haven't already added it above OutputFileReference Reference = Include.ExpandedReferences[Idx]; if (!Include.FinalFiles.Contains(Reference.File)) { if (Dependencies.Any(x => Reference.UniqueFragments.Contains(x)) || (Reference.File.InputFile.Flags & SourceFileFlags.Pinned) != 0 || Reference == ForcedHeaderFileReference || Reference.File == MonolithicHeader || ExplicitIncludes.Contains(Reference.File) || ((InputFile.Flags & SourceFileFlags.Aggregate) != 0 && Reference.File == Include.TargetFile) || // Always include the original header for aggregates. They are written explicitly to include certain files. Reference.UniqueFragments.Any(x => FragmentsWithReferencedSymbols.Contains(x))) { Include.FinalFiles.Insert(0, Reference.File); Dependencies.ExceptWith(Reference.File.IncludedFragments); Dependencies.UnionWith(Reference.File.Dependencies); } } } } } // Remove any includes that are already included by the matching header if (HeaderFile != null) { HashSet <OutputFile> HeaderIncludedFiles = new HashSet <OutputFile>(HeaderFile.Includes.SelectMany(x => x.FinalFiles)); foreach (OutputFileInclude Include in Includes) { Include.FinalFiles.RemoveAll(x => HeaderIncludedFiles.Contains(x)); } } // Check that all the dependencies have been satisfied if (Dependencies.Count > 0) { // Find those which are completely invalid List <SourceFragment> InvalidDependencies = Dependencies.Where(x => !InputFileStack.Contains(x.File)).ToList(); if (InvalidDependencies.Count > 0) { Log.WriteLine("warning: {0} does not include {1}{2}; may have missing dependencies.", InputFile, String.Join(", ", InvalidDependencies.Select(x => x.Location.FullName).Take(3)), (InvalidDependencies.Count > 3)? String.Format(" and {0} others", InvalidDependencies.Count - 3) : ""); } Dependencies.ExceptWith(InvalidDependencies); // Otherwise warn about those which were not pinned foreach (SourceFile DependencyFile in Dependencies.Select(x => x.File)) { Log.WriteLine("warning: {0} is included by {1} ({2}), but depends on it and should be marked as pinned.", InputFile, DependencyFile, String.Join(" -> ", InputFileStack.SkipWhile(x => x != DependencyFile).Select(x => x.Location.GetFileName()))); } // Mark it as non-standalone and pinned InputFile.Flags = (InputFile.Flags | SourceFileFlags.Pinned) & ~SourceFileFlags.Standalone; } // Do one more forward pass through all the headers, and remove anything that's included more than once. That can happen if we have a referenced symbol as well as // an explicit include, for example. HashSet <OutputFile> FinalIncludes = new HashSet <OutputFile>(); foreach (OutputFileInclude Include in Includes) { for (int Idx = 0; Idx < Include.FinalFiles.Count; Idx++) { if (!FinalIncludes.Add(Include.FinalFiles[Idx])) { Include.FinalFiles.RemoveAt(Idx); Idx--; } } } // Create the optimized file OutputFile OptimizedFile = new OutputFile(InputFile, Includes, Dependencies, ForwardDeclarations); // Write the verbose log InputFile.LogVerbose(""); foreach (OutputFile IncludedFile in OptimizedFile.IncludedFiles) { InputFile.LogVerbose("Output: {0}", IncludedFile.InputFile.Location.FullName); } // Return the optimized file return(OptimizedFile); }
/// <summary> /// Find or create an output file for a corresponding input file /// </summary> /// <param name="InputFile">The input file</param> /// <param name="IncludeStack">The current include stack</param> /// <param name="OutputFiles">List of output files</param> /// <param name="OutputFileLookup">Mapping from source file to output file</param> /// <returns>The new or existing output file</returns> static OutputFile FindOrCreateOutputFile(List <SourceFile> InputFileStack, Dictionary <SourceFile, SourceFile> CppFileToHeaderFile, HashList <OutputFile> PreviousFiles, Dictionary <SourceFile, OutputFile> OutputFileLookup, Dictionary <Symbol, OutputFile> FwdSymbolToHeader, bool bMakeStandalone, bool bUseOriginalIncludes, LineBasedTextWriter Log) { // Get the file at the top of the stack SourceFile InputFile = InputFileStack[InputFileStack.Count - 1]; // Try to find an existing file OutputFile OutputFile; if (OutputFileLookup.TryGetValue(InputFile, out OutputFile)) { if (OutputFile == null) { throw new Exception("Circular include dependencies are not allowed."); } foreach (OutputFile IncludedFile in OutputFile.OriginalIncludedFiles.Where(x => !PreviousFiles.Contains(x))) { PreviousFiles.Add(IncludedFile); } } else { // Add a placeholder entry in the output file lookup, so we can detect circular include dependencies OutputFileLookup[InputFile] = null; // Duplicate the list of previously included files, so we can construct the List <OutputFile> PreviousFilesCopy = new List <OutputFile>(PreviousFiles); // Build a list of includes for this file. First include is a placeholder for any missing includes that need to be inserted. List <OutputFileInclude> Includes = new List <OutputFileInclude>(); if ((InputFile.Flags & SourceFileFlags.External) == 0) { for (int MarkupIdx = 0; MarkupIdx < InputFile.Markup.Length; MarkupIdx++) { PreprocessorMarkup Markup = InputFile.Markup[MarkupIdx]; if (Markup.IsActive && Markup.IncludedFile != null && (Markup.IncludedFile.Flags & SourceFileFlags.Inline) == 0 && Markup.IncludedFile.Counterpart == null) { InputFileStack.Add(Markup.IncludedFile); OutputFile IncludeFile = FindOrCreateOutputFile(InputFileStack, CppFileToHeaderFile, PreviousFiles, OutputFileLookup, FwdSymbolToHeader, bMakeStandalone, bUseOriginalIncludes, Log); InputFileStack.RemoveAt(InputFileStack.Count - 1); Includes.Add(new OutputFileInclude(MarkupIdx, IncludeFile)); } } } // Find the matching header file OutputFile HeaderFile = null; if ((InputFile.Flags & SourceFileFlags.TranslationUnit) != 0) { SourceFile CandidateHeaderFile; if (CppFileToHeaderFile.TryGetValue(InputFile, out CandidateHeaderFile) && (CandidateHeaderFile.Flags & SourceFileFlags.Standalone) != 0) { OutputFileLookup.TryGetValue(CandidateHeaderFile, out HeaderFile); } } // Create the output file. if ((InputFile.Flags & SourceFileFlags.Output) != 0 && !bUseOriginalIncludes) { OutputFile = CreateOptimizedOutputFile(InputFile, HeaderFile, PreviousFilesCopy, Includes, InputFileStack, FwdSymbolToHeader, bMakeStandalone, Log); } else { OutputFile = CreatePassthroughOutputFile(InputFile, Includes, Log); } // Replace the null entry in the output file lookup that we added earlier OutputFileLookup[InputFile] = OutputFile; // Add this file to the list of included files PreviousFiles.Add(OutputFile); // If the output file dependends on something on the stack, make sure it's marked as pinned if ((InputFile.Flags & SourceFileFlags.Pinned) == 0) { SourceFragment Dependency = OutputFile.Dependencies.FirstOrDefault(x => InputFileStack.Contains(x.File) && x.File != InputFile); if (Dependency != null) { throw new Exception(String.Format("'{0}' is not marked as pinned, but depends on '{1}' which includes it", InputFile.Location.GetFileName(), Dependency.UniqueName)); } } } return(OutputFile); }
/// <summary> /// Wait for this worker to complete /// </summary> /// <param name="Writer">Writer for log output</param> /// <returns>Return code from the thread</returns> public SequenceProbeResult Join(LineBasedTextWriter Writer) { // Finish the task instance BufferedTextWriter BufferedWriter = new BufferedTextWriter(); int ExitCode = ActiveInstance.Join(BufferedWriter); ActiveInstance.Dispose(); ActiveInstance = null; // Read the new state FileReference OutputFile = new FileReference(TaskStateFile.FullName + ".out"); SequenceWorker NewWorker = SequenceWorker.Deserialize(OutputFile.FullName); OutputFile.Delete(); // Make sure the exit code reflects the failure state. XGE can sometimes fail transferring back. if (ExitCode == 0 && !NewWorker.bResult) { ExitCode = -1; } // If it's a hard failure, print the compile log to the regular log if (ExitCode == -1) { Writer.WriteLine("Failed to compile {0}, exit code {1}:", UniqueName, ExitCode); foreach (string Line in NewWorker.CompileLog) { Writer.WriteLine(" > {0}", Line.Replace("error:", "err:")); } } // Annotate the log data if this is from a failed attempt. It still may be useful for debugging purposes. if (ExitCode != 0) { NewWorker.SummaryLog.Insert(0, String.Format("ExitCode={0}", ExitCode)); NewWorker.SummaryLog = NewWorker.SummaryLog.Select(x => "FAIL > " + x).ToList(); NewWorker.CompileLog.Insert(0, String.Format("ExitCode={0}", ExitCode)); NewWorker.CompileLog = NewWorker.CompileLog.Select(x => "FAIL > " + x).ToList(); } // Append the log data back to the local output File.AppendAllLines(SummaryLogFile.FullName, NewWorker.SummaryLog); NewWorker.SummaryLog.Clear(); File.AppendAllLines(CompileLogFile.FullName, NewWorker.CompileLog); NewWorker.CompileLog.Clear(); // If we failed, return the if (ExitCode != 0) { if (ExitCode == -1) { Writer.WriteLine("Warning: Failed to compile {0}, exit code {1}. Aborting.", UniqueName, ExitCode); return(SequenceProbeResult.Failed); } else { Writer.WriteLine("Warning: Failed to compile {0}; exit code {1}. Will retry.", UniqueName, ExitCode); return(SequenceProbeResult.FailedAllowRetry); } } // Update the task Worker = NewWorker; // Check if this is just an incremental update if (Type == SequenceProbeType.Verify) { // Save the task Worker.Serialize(TaskStateFile.FullName); // Return that we're done return(SequenceProbeResult.Completed); } else if (Type == SequenceProbeType.Optimize) { if (Worker.RemainingFragmentCount > 0) { // Get the top-most fragment - the one we've just established is a dependency for this leaf node - and add it to the list of known dependencies SourceFragment NextFragment = Fragments[Worker.RemainingFragmentCount - 1]; AddDependency(Worker, Fragments, Worker.RemainingFragmentCount - 1); Worker.SummaryLog.Add(String.Format(" [Added {0}: {1}]", Worker.RemainingFragmentCount - 1, Fragments[Worker.RemainingFragmentCount - 1].Location)); // Save the task Worker.Serialize(TaskStateFile.FullName); // Otherwise, return that we've just updated return(SequenceProbeResult.Updated); } else { // Save the task Worker.Serialize(TaskStateFile.FullName); // Return that we're done SetCompletedDependencies(); return(SequenceProbeResult.Completed); } } else { throw new NotImplementedException(); } }
/// <summary> /// Constructor /// </summary> /// <param name="Type">The type of probe to create</param> /// <param name="Node">The node to optimize</param> /// <param name="IntermediateDir">Directory for intermediate files</param> /// <param name="UniqueName">Unique name prefix for all temporary files</param> public SequenceProbe(SequenceProbeType Type, SourceFragment[] Fragments, Tuple <int, SourceFile>[] IncludeHistory, CompileEnvironment CompileEnvironment, DirectoryReference IntermediateDir, IEnumerable <DirectoryReference> ExtraSystemIncludePaths, string UniqueName) { this.Type = Type; this.IntermediateDir = IntermediateDir; this.Fragments = Fragments; this.LastFragment = Fragments[Fragments.Length - 1]; this.UniqueName = UniqueName; this.TaskStateFile = FileReference.Combine(IntermediateDir, UniqueName + ((Type == SequenceProbeType.Verify)? ".verify.state" : ".state")); this.SummaryLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".summary.txt"); this.CompileLogFile = FileReference.Combine(IntermediateDir, UniqueName + ".compile.txt"); // Get the file to use for trying to compile different permutations FileReference PermutationFile = FileReference.Combine(IntermediateDir, UniqueName + ".permutation"); // Create the response file FileReference ResponseFile = FileReference.Combine(IntermediateDir, UniqueName + ".response"); CompileEnvironment WorkerCompileEnvironment = new CompileEnvironment(CompileEnvironment); if (WorkerCompileEnvironment.CompilerType == CompilerType.Clang) { WorkerCompileEnvironment.Options.Add(new CompileOption("-o", FileReference.Combine(IntermediateDir, UniqueName + ".o").FullName.Replace('\\', '/'))); } else { WorkerCompileEnvironment.Options.RemoveAll(x => x.Name == "/Z7" || x.Name == "/Zi" || x.Name == "/ZI"); WorkerCompileEnvironment.Options.Add(new CompileOption("/Fo", FileReference.Combine(IntermediateDir, UniqueName + ".obj").FullName)); WorkerCompileEnvironment.Options.Add(new CompileOption("/WX", null)); } WorkerCompileEnvironment.WriteResponseFile(ResponseFile, PermutationFile); string ResponseFileDigest = Utility.ComputeDigest(ResponseFile); // Keep track of the include stack, so we can format the flat fragment list with context int IncludeHistoryIdx = 0; List <SourceFile> IncludeStack = new List <SourceFile>(); // Create the script for the probe List <Tuple <int, string> > Lines = new List <Tuple <int, string> >(); for (int Idx = 0; Idx < Fragments.Length; Idx++) { SourceFragment Fragment = Fragments[Idx]; // Figure out which tag it's bound to int Tag = (Fragment.File.Counterpart == null)? Idx : -1; // Update the stack for new includes while (IncludeHistoryIdx < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx].Item1 == Idx) { if (IncludeHistory[IncludeHistoryIdx].Item2 == null) { SourceFile IncludeFile = IncludeStack[IncludeStack.Count - 1]; IncludeStack.RemoveAt(IncludeStack.Count - 1); Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// END INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName))); IncludeHistoryIdx++; } else if (IncludeHistoryIdx + 1 < IncludeHistory.Length && IncludeHistory[IncludeHistoryIdx + 1].Item2 == null) { IncludeHistoryIdx += 2; } else { SourceFile IncludeFile = IncludeHistory[IncludeHistoryIdx].Item2; Lines.Add(new Tuple <int, string>(Tag, String.Format("{0}// INCLUDE {1}", new string(' ', IncludeStack.Count * 4), IncludeFile.Location.FullName))); IncludeStack.Add(IncludeFile); IncludeHistoryIdx++; } } // Get the indent at this point string Indent = new string(' ', (IncludeStack.Count - 0) * 4); // Write out the forward declarations for every symbol referenced in this fragment. We don't want false dependencies caused by forward declarations in other fragments // if the heuristic for detecting when to use them doesn't work. if ((Fragment.File.Flags & SourceFileFlags.TranslationUnit) == 0) { foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols) { if (!String.IsNullOrEmpty(ReferencedSymbol.Key.ForwardDeclaration)) { Lines.Add(Tuple.Create(Tag, Indent + ReferencedSymbol.Key.ForwardDeclaration)); } } } // Some Clang/GCC system header wrappers require including as system includes in order to make the #include_next directive work DirectoryReference BaseSystemIncludePath = ExtraSystemIncludePaths.FirstOrDefault(x => Fragment.Location.IsUnderDirectory(x)); if (BaseSystemIncludePath != null) { Lines.Add(Tuple.Create(Tag, String.Format("{0}#include <{1}>", Indent, Fragment.Location.MakeRelativeTo(BaseSystemIncludePath)))); } else { Lines.Add(Tuple.Create(Tag, String.Format("{0}#include \"{1}\"", Indent, Fragment.Location.FullName))); } } // Create the new task string[] FragmentFileNames = Fragments.Select(Fragment => Fragment.Location.FullName).ToArray(); string[] FragmentDigests = Fragments.Select(Fragment => Fragment.Digest ?? "").ToArray(); Worker = new SequenceWorker(PermutationFile.FullName, ResponseFile.FullName, ResponseFileDigest, FragmentFileNames, FragmentDigests, Lines.ToArray(), CompileEnvironment.Compiler.FullName); AddDependency(Worker, Fragments, Fragments.Length - 1); // Log the referenced symbols if (LastFragment.ReferencedSymbols.Count > 0) { List <string> ReferenceLog = new List <string>(); ReferenceLog.Add(String.Format("Referenced symbols for {0}:", LastFragment.Location)); foreach (KeyValuePair <Symbol, SymbolReferenceType> Pair in LastFragment.ReferencedSymbols.OrderBy(x => x.Key.Type).ThenBy(x => x.Key.Name)) { Symbol ReferencedSymbol = Pair.Key; ReferenceLog.Add(String.Format(" {0}: {1} ({2}; {3})", ReferencedSymbol.Type.ToString(), ReferencedSymbol.Name, Pair.Value.ToString(), ReferencedSymbol.Fragment.Location)); } ReferenceLog.Add(""); Worker.SummaryLog.InsertRange(0, ReferenceLog); } // Check to see if an existing version of the task exists which we can continue if (Type == SequenceProbeType.Optimize && TaskStateFile.Exists()) { // Try to read the old task SequenceWorker OldWorker; try { OldWorker = SequenceWorker.Deserialize(TaskStateFile.FullName); } catch (Exception) { OldWorker = null; } // If it succeeded, compare it to the new task if (OldWorker != null) { SequenceWorker NewWorker = Worker; // Create a list of fragments which can be ignored, because they're already determined to be not part of the result for the old task HashSet <string> IgnoreFragments = new HashSet <string>(); for (int Idx = 0; Idx < OldWorker.FragmentCount; Idx++) { if (!OldWorker.KnownDependencies.Contains(Idx) && Idx >= OldWorker.RemainingFragmentCount) { IgnoreFragments.Add(OldWorker.FragmentFileNames[Idx]); } } IgnoreFragments.ExceptWith(NewWorker.KnownDependencies.Select(x => NewWorker.FragmentFileNames[x])); // Compute digests for the old and new tasks string OldDigest = CreateDigest(OldWorker, IgnoreFragments); string NewDigest = CreateDigest(NewWorker, IgnoreFragments); if (OldDigest == NewDigest) { // Build a map of fragment file names to their new index Dictionary <string, int> FragmentFileNameToNewIndex = new Dictionary <string, int>(); for (int Idx = 0; Idx < FragmentFileNames.Length; Idx++) { string FragmentFileName = FragmentFileNames[Idx]; FragmentFileNameToNewIndex[FragmentFileName] = Idx; } // Add known dependencies to the new worker foreach (int OldKnownDependency in OldWorker.KnownDependencies) { string OldFragmentFileName = OldWorker.FragmentFileNames[OldKnownDependency]; int NewKnownDependency = FragmentFileNameToNewIndex[OldFragmentFileName]; NewWorker.KnownDependencies.Add(NewKnownDependency); } // Update the remaining count. All these fragments must match, because they're not part of the ignore list. NewWorker.RemainingFragmentCount = OldWorker.RemainingFragmentCount; } } } // If this is a cpp file, make sure we have a dependency on the header file with the same name. It may specify linkage for the functions we declare. FileReference MainFileLocation = Fragments[Fragments.Length - 1].File.Location; if (MainFileLocation.HasExtension(".cpp")) { string HeaderFileName = Path.ChangeExtension(MainFileLocation.GetFileName(), ".h"); for (int FragmentIdx = 0; FragmentIdx < Fragments.Length; FragmentIdx++) { if (String.Compare(Fragments[FragmentIdx].File.Location.GetFileName(), HeaderFileName, true) == 0) { AddDependency(Worker, Fragments, FragmentIdx); } } } // Update the finished fragment if we're done, otherwise clear out all the intermediate files if (Worker.RemainingFragmentCount == 0) { SetCompletedDependencies(); } else { Worker.Serialize(TaskStateFile.FullName); PermutationFile.Delete(); SummaryLogFile.Delete(); CompileLogFile.Delete(); } }
/// <summary> /// Read all the exported symbol definitions from a file /// </summary> /// <param name="File">The file to parse</param> /// <param name="MarkupIdx">The current index into the markup array of this file</param> /// <param name="Scope">The scoping depth, ie. number of unmatched '{' tokens encountered in source code</param> void ReadExportedSymbols(SourceFile File, ref int MarkupIdx, ref int Scope) { // Ignore files which only have a header guard; there are no fragments that we can link to if (File.BodyMinIdx == File.BodyMaxIdx) { return; } // Read markup from the file for (; MarkupIdx < File.Markup.Length; MarkupIdx++) { PreprocessorMarkup Markup = File.Markup[MarkupIdx]; if (Markup.Type == PreprocessorMarkupType.Define) { // If the macro is not already defined, add it if (Markup.Tokens[0].Text != "LOCTEXT_NAMESPACE") { SourceFragment Fragment = FindFragment(File, MarkupIdx); AddSymbol(Markup.Tokens[0].Text, SymbolType.Macro, null, Fragment, Markup.Location); } } else if (Markup.Type == PreprocessorMarkupType.If && Markup.Tokens.Count == 2 && Markup.Tokens[0].Text == "!" && Markup.Tokens[1].Text == "CPP") { // Completely ignore these blocks; they sometimes differ in class/struct usage, and exist solely for noexport types to be parsed by UHT int Depth = 1; while (Depth > 0) { Markup = File.Markup[++MarkupIdx]; if (Markup.Type == PreprocessorMarkupType.If || Markup.Type == PreprocessorMarkupType.Ifdef || Markup.Type == PreprocessorMarkupType.Ifndef) { Depth++; } else if (Markup.Type == PreprocessorMarkupType.Endif) { Depth--; } } } else if (Markup.Type == PreprocessorMarkupType.If || Markup.Type == PreprocessorMarkupType.Ifdef || Markup.Type == PreprocessorMarkupType.Ifndef) { // Loop through all the branches of this conditional block, saving the resulting scope of each derivation List <int> NewScopes = new List <int>(); while (Markup.Type != PreprocessorMarkupType.Endif) { // Skip over the conditional directive MarkupIdx++; // Read any exported symbols from this conditional block int NewScope = Scope; ReadExportedSymbols(File, ref MarkupIdx, ref NewScope); NewScopes.Add(NewScope); // Get the new preprocessor markup Markup = File.Markup[MarkupIdx]; } // Make sure they were all consistent if (NewScopes.Any(x => x != NewScopes[0])) { throw new Exception(String.Format("Unbalanced scope depth between conditional block derivations while parsing {0}", File.Location)); } // Update the scope Scope = NewScopes[0]; } else if (Markup.Type == PreprocessorMarkupType.Else || Markup.Type == PreprocessorMarkupType.Elif || Markup.Type == PreprocessorMarkupType.Endif) { // We've reached the end of this block; return success break; } else if (Markup.Type == PreprocessorMarkupType.Text) { // Read the declarations in this block SourceFragment Fragment = FindFragment(File, MarkupIdx); ParseDeclarations(File, Fragment, File.Markup[MarkupIdx], ref Scope); } } }
/// <summary> /// Try to read a template class or struct header /// </summary> /// <param name="Reader">Tokens to parse</param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadTemplateClassOrStructHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Check for the template keyword if (Reader.Current.Text != "template") { return(false); } // Create a buffer to store the template prefix List <Token> Tokens = new List <Token>(); Tokens.Add(Reader.Current); // Check for the opening argument list if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "<") { return(false); } // Read the argument list, keeping track of any symbols referenced along the way while (Tokens[Tokens.Count - 1].Text != ">") { Tokens.Add(Reader.Current); if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } // Get the symbol type SymbolType Type; if (Reader.Current.Text == "class") { Type = SymbolType.TemplateClass; } else if (Reader.Current.Text == "struct") { Type = SymbolType.TemplateStruct; } else { return(false); } // Add the class or struct keyword Tokens.Add(Reader.Current); // Move to the name if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Skip over an optional DLL export declaration if (Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API") && !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Read the class name and check it's followed by a class body or inheritance list string Name = Reader.Current.Text; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != ":" && Reader.Current.Text != "{") { return(false); } // Create the symbol. if (Rules.AllowSymbol(Name)) { // Only allow forward declarations of templates with class and typename arguments and no defaults (ie. no enums or class names which may have to be forward-declared separately). string ForwardDeclaration = null; if (!Tokens.Any(x => x.Text == "=")) { ForwardDeclaration = String.Format("{0} {1};", Token.Format(Tokens), Name); for (int Idx = 2; Idx < Tokens.Count - 2; Idx += 3) { if (Tokens[Idx].Text != "class" && Tokens[Idx].Text != "typename") { ForwardDeclaration = null; break; } } } AddSymbol(Name, Type, ForwardDeclaration, Fragment, OriginalReader.TokenLocation); } // Move the original reader past the declaration OriginalReader.Set(Reader); return(true); }
/// <summary> /// Parse an "enum class" declaration, and add a symbol for it /// </summary> /// <param name="Reader">Tokens to be parsed. On success, this is assigned to a new </param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadEnumClassHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Read the UENUM prefix if present. We don't want to forward-declare types that need to be parsed by UHT, because it needs the definition. bool bIsUENUM = false; if (Reader.Current.Text == "UENUM") { if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "(") { return(false); } while (Reader.Current.Text != ")") { if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } bIsUENUM = true; } // Read the 'enum class' tokens if (Reader.Current.Text != "enum" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != "class" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Read the name, make sure we haven't read a definition for it already, and check it's an enum declaration string Name = Reader.Current.Text; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != ";" && Reader.Current.Text != ":" && Reader.Current.Text != "{") { return(false); } // Build the forward declaration for it. Don't forward-declare UENUM types because UHT needs to parse their definition first. string ForwardDeclaration = null; if (!bIsUENUM) { StringBuilder ForwardDeclarationBuilder = new StringBuilder(); ForwardDeclarationBuilder.AppendFormat("enum class {0}", Name); while (Reader.Current.Text != ";" && Reader.Current.Text != "{") { // Append the next token if (Reader.Current.HasLeadingSpace) { ForwardDeclarationBuilder.Append(" "); } ForwardDeclarationBuilder.Append(Reader.Current.Text); // Try to move to the next token if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } ForwardDeclarationBuilder.Append(";"); ForwardDeclaration = ForwardDeclarationBuilder.ToString(); } // Create a symbol for it if it's an actual definition rather than a forward declaration if (Reader.Current.Text == "{" && Rules.AllowSymbol(Name)) { AddSymbol(Name, SymbolType.Enumeration, ForwardDeclaration, Fragment, OriginalReader.TokenLocation); } // Update the original reader to be the new location OriginalReader.Set(Reader); return(true); }