/// <summary> /// Try to read a template class or struct header /// </summary> /// <param name="Reader">Tokens to parse</param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadTemplateClassOrStructHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Check for the template keyword if (Reader.Current.Text != "template") { return(false); } // Create a buffer to store the template prefix List <Token> Tokens = new List <Token>(); Tokens.Add(Reader.Current); // Check for the opening argument list if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "<") { return(false); } // Read the argument list, keeping track of any symbols referenced along the way while (Tokens[Tokens.Count - 1].Text != ">") { Tokens.Add(Reader.Current); if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } // Get the symbol type SymbolType Type; if (Reader.Current.Text == "class") { Type = SymbolType.TemplateClass; } else if (Reader.Current.Text == "struct") { Type = SymbolType.TemplateStruct; } else { return(false); } // Add the class or struct keyword Tokens.Add(Reader.Current); // Move to the name if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Skip over an optional DLL export declaration if (Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API") && !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Read the class name and check it's followed by a class body or inheritance list string Name = Reader.Current.Text; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != ":" && Reader.Current.Text != "{") { return(false); } // Create the symbol. if (Rules.AllowSymbol(Name)) { // Only allow forward declarations of templates with class and typename arguments and no defaults (ie. no enums or class names which may have to be forward-declared separately). string ForwardDeclaration = null; if (!Tokens.Any(x => x.Text == "=")) { ForwardDeclaration = String.Format("{0} {1};", Token.Format(Tokens), Name); for (int Idx = 2; Idx < Tokens.Count - 2; Idx += 3) { if (Tokens[Idx].Text != "class" && Tokens[Idx].Text != "typename") { ForwardDeclaration = null; break; } } } AddSymbol(Name, Type, ForwardDeclaration, Fragment, OriginalReader.TokenLocation); } // Move the original reader past the declaration OriginalReader.Set(Reader); return(true); }
/// <summary> /// Finds all the imported symbols from the given fragment /// </summary> /// <param name="Fragment">Fragment to find symbols for</param> /// <returns>Array of unique symbols for this fragment, which are not already forward-declared</returns> public Dictionary <Symbol, SymbolReferenceType> FindReferences(SourceFragment Fragment) { Dictionary <Symbol, SymbolReferenceType> References = new Dictionary <Symbol, SymbolReferenceType>(); TextBuffer Text = Fragment.File.Text; if (Text != null && Fragment.MarkupMax > Fragment.MarkupMin) { TokenReader Reader = new TokenReader(Fragment.File.Text, Fragment.MinLocation, Fragment.MaxLocation); for (bool bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); bMoveNext;) { // Read the current token, and immediately move to the next so that we can lookahead if we need to Token Token = Reader.Current; bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); // Check if the previous token is a symbol if (Token.Type == TokenType.Identifier) { if (Token.Text == "class" || Token.Text == "struct") { // Skip over "class", "struct" and "enum class" declarations and forward declarations. They're not actually referencing another symbol. if (bMoveNext && Reader.Current.Type == TokenType.Identifier && Reader.Current.Text.EndsWith("_API")) { bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); } if (bMoveNext && Reader.Current.Type == TokenType.Identifier) { bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); } } else if (IsSmartPointerClass(Token.Text) && bMoveNext && Reader.Current.Text == "<") { // Smart pointer types which do not require complete declarations bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); if (bMoveNext && Reader.Current.Type == TokenType.Identifier) { foreach (Symbol MatchingSymbol in Lookup.WithKey(Reader.Current.Text)) { if ((MatchingSymbol.Type == SymbolType.Class || MatchingSymbol.Type == SymbolType.Struct) && !References.ContainsKey(MatchingSymbol) && !IgnoreSymbol(MatchingSymbol, Fragment, Reader.TokenLocation)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } bMoveNext = Reader.MoveNext(TokenReaderContext.IgnoreNewlines); } } else { // Otherwise check what type of symbol it is (if any), and whether it's referenced in an opaque way (though a pointer/reference or so on). foreach (Symbol MatchingSymbol in Lookup.WithKey(Token.Text)) { if (IgnoreSymbol(MatchingSymbol, Fragment, Reader.TokenLocation)) { continue; } else if (MatchingSymbol.Type == SymbolType.Macro) { // Add the symbol if it doesn't already exist if (!References.ContainsKey(MatchingSymbol)) { References.Add(MatchingSymbol, SymbolReferenceType.RequiresDefinition); } } else if (MatchingSymbol.Type == SymbolType.Enumeration) { // Check whether we're using the type in an opaque way, or we actually require the definition if (!bMoveNext || Reader.Current.Text == "::") { References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition; } else if (!References.ContainsKey(MatchingSymbol)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } else if (MatchingSymbol.Type == SymbolType.Class || MatchingSymbol.Type == SymbolType.Struct) { // Check whether we're declaring a pointer to this type, and just need a forward declaration if (!bMoveNext || (Reader.Current.Text != "*" && Reader.Current.Text != "&")) { References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition; } else if (!References.ContainsKey(MatchingSymbol)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } else if (MatchingSymbol.Type == SymbolType.TemplateClass || MatchingSymbol.Type == SymbolType.TemplateStruct) { // Check whether we're declaring a pointer to this type, and just need a forward declaration TokenReader Lookahead = new TokenReader(Reader); bool bLookaheadMoveNext = bMoveNext; if (!bMoveNext || !SkipTemplateArguments(Lookahead, ref bLookaheadMoveNext) || (Lookahead.Current.Text != "*" && Lookahead.Current.Text != "&")) { References[MatchingSymbol] = SymbolReferenceType.RequiresDefinition; } else if (!References.ContainsKey(MatchingSymbol)) { References[MatchingSymbol] = SymbolReferenceType.Opaque; } } } } } } } return(References); }
/// <summary> /// Try to skip over a template header /// </summary> /// <param name="OriginalReader">Tokens to parse</param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadTypedefHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); if (Reader.Current.Text == "typedef") { // Check for the typedef keyword Token PreviousToken = Reader.Current; if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Scan to the next semicolon while (Reader.MoveNext(TokenReaderContext.IgnoreNewlines) && Reader.Current.Text != ";" && Reader.Current.Text != "{") { PreviousToken = Reader.Current; } // Ignore 'typedef struct' and 'typedef union' declarations. if (Reader.Current.Text == "{") { return(false); } // Try to add a symbol for the previous token. If it already exists, replace it. if (PreviousToken.Type == TokenType.Identifier && Rules.AllowSymbol(PreviousToken.Text)) { AddSymbol(PreviousToken.Text, SymbolType.Typedef, null, Fragment, OriginalReader.TokenLocation); } } else if (Reader.Current.Text == "using") { // Check for the using keyword if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Get the identifier Token Identifier = Reader.Current; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "=") { return(false); } // Scan to the next semicolon while (Reader.MoveNext(TokenReaderContext.IgnoreNewlines) && Reader.Current.Text != ";" && Reader.Current.Text != "{") { } // Get the identifier if (Rules.AllowSymbol(Identifier.Text)) { AddSymbol(Identifier.Text, SymbolType.Typedef, null, Fragment, OriginalReader.TokenLocation); } } else { return(false); } // Move the original reader past the declaration OriginalReader.Set(Reader); return(true); }
/// <summary> /// Parse an "enum class" declaration, and add a symbol for it /// </summary> /// <param name="Reader">Tokens to be parsed. On success, this is assigned to a new </param> /// <param name="Fragment">Fragment containing these tokens</param> bool ReadEnumClassHeader(TokenReader OriginalReader, SourceFragment Fragment) { TokenReader Reader = new TokenReader(OriginalReader); // Read the UENUM prefix if present. We don't want to forward-declare types that need to be parsed by UHT, because it needs the definition. bool bIsUENUM = false; if (Reader.Current.Text == "UENUM") { if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines) || Reader.Current.Text != "(") { return(false); } while (Reader.Current.Text != ")") { if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } bIsUENUM = true; } // Read the 'enum class' tokens if (Reader.Current.Text != "enum" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != "class" || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } // Read the name, make sure we haven't read a definition for it already, and check it's an enum declaration string Name = Reader.Current.Text; if (Reader.Current.Type != TokenType.Identifier || !Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } if (Reader.Current.Text != ";" && Reader.Current.Text != ":" && Reader.Current.Text != "{") { return(false); } // Build the forward declaration for it. Don't forward-declare UENUM types because UHT needs to parse their definition first. string ForwardDeclaration = null; if (!bIsUENUM) { StringBuilder ForwardDeclarationBuilder = new StringBuilder(); ForwardDeclarationBuilder.AppendFormat("enum class {0}", Name); while (Reader.Current.Text != ";" && Reader.Current.Text != "{") { // Append the next token if (Reader.Current.HasLeadingSpace) { ForwardDeclarationBuilder.Append(" "); } ForwardDeclarationBuilder.Append(Reader.Current.Text); // Try to move to the next token if (!Reader.MoveNext(TokenReaderContext.IgnoreNewlines)) { return(false); } } ForwardDeclarationBuilder.Append(";"); ForwardDeclaration = ForwardDeclarationBuilder.ToString(); } // Create a symbol for it if it's an actual definition rather than a forward declaration if (Reader.Current.Text == "{" && Rules.AllowSymbol(Name)) { AddSymbol(Name, SymbolType.Enumeration, ForwardDeclaration, Fragment, OriginalReader.TokenLocation); } // Update the original reader to be the new location OriginalReader.Set(Reader); return(true); }
/// <summary> /// Create markup for the given text buffer /// </summary> /// <param name="Reader">Reader for token objects</param> /// <returns>Array of markup objects which split up the given text buffer</returns> public static PreprocessorMarkup[] ParseArray(TextBuffer Text) { TokenReader Reader = new TokenReader(Text, TextLocation.Origin); List <PreprocessorMarkup> Markup = new List <PreprocessorMarkup>(); if (Reader.MoveNext()) { bool bMoveNext = true; while (bMoveNext) { TextLocation StartLocation = Reader.TokenWhitespaceLocation; if (Reader.Current.Text == "#") { // Create the appropriate markup object for the directive PreprocessorMarkupType Type = PreprocessorMarkupType.OtherDirective; if (Reader.MoveNext()) { switch (Reader.Current.Text) { case "include": Type = PreprocessorMarkupType.Include; break; case "define": Type = PreprocessorMarkupType.Define; break; case "undef": Type = PreprocessorMarkupType.Undef; break; case "if": Type = PreprocessorMarkupType.If; break; case "ifdef": Type = PreprocessorMarkupType.Ifdef; break; case "ifndef": Type = PreprocessorMarkupType.Ifndef; break; case "elif": Type = PreprocessorMarkupType.Elif; break; case "else": Type = PreprocessorMarkupType.Else; break; case "endif": Type = PreprocessorMarkupType.Endif; break; case "pragma": Type = PreprocessorMarkupType.Pragma; break; case "error": Type = PreprocessorMarkupType.Error; break; case "\n": Type = PreprocessorMarkupType.Empty; break; } } // Create the token list List <Token> Tokens = new List <Token>(); if (Type == PreprocessorMarkupType.OtherDirective) { Tokens.Add(Reader.Current); } // Read the first token if (Type == PreprocessorMarkupType.Empty) { bMoveNext = true; } else if (Type == PreprocessorMarkupType.Include) { bMoveNext = Reader.MoveNext(TokenReaderContext.IncludeDirective); } else if (Type == PreprocessorMarkupType.Error) { bMoveNext = Reader.MoveNext(TokenReaderContext.TokenString); } else { bMoveNext = Reader.MoveNext(); } // Read the rest of the tokens while (bMoveNext && Reader.Current.Text != "\n") { Tokens.Add(Reader.Current); bMoveNext = Reader.MoveNext(); } // Create the markup Markup.Add(new PreprocessorMarkup(Type, StartLocation, Reader.TokenEndLocation, Tokens)); // Move to the next token bMoveNext = Reader.MoveNext(); } else if (Reader.Current.Text != "\n") { // Skip over as many parser tokens as possible before the next directive (or EOF) bMoveNext = Reader.MoveToNextLine(); while (bMoveNext && Reader.Current.Text != "#") { bMoveNext = Reader.MoveToNextLine(); } // Create the new fragment PreprocessorMarkupType Type = IsIncludeMarkup(Text, StartLocation, Reader.TokenLocation)? PreprocessorMarkupType.IncludeMarkup : PreprocessorMarkupType.Text; Markup.Add(new PreprocessorMarkup(Type, StartLocation, Reader.TokenLocation, null)); } else { // Skip the empty line bMoveNext = Reader.MoveNext(); } } } return(Markup.ToArray()); }
/// <summary> /// Write out an optimized file to the given location /// </summary> /// <param name="IncludePaths">Base directories for relative include paths</param> /// <param name="SystemIncludePaths">Base directories for system include paths</param> /// <param name="Writer">Writer for the output text</param> public void Write(IEnumerable <DirectoryReference> IncludePaths, IEnumerable <DirectoryReference> SystemIncludePaths, TextWriter Writer, bool bRemoveForwardDeclarations, LineBasedTextWriter Log) { // Write the file header TextLocation LastLocation = Text.Start; // Write the standalone includes if (MissingIncludes.Count > 0) { TextLocation BoilerplateLocation = (BodyMinIdx == Markup.Length || (Markup[BodyMinIdx].Type != PreprocessorMarkupType.Include && BodyMinIdx > 0))? Markup[BodyMinIdx - 1].EndLocation : Markup[BodyMinIdx].Location; WriteLines(Text, LastLocation, BoilerplateLocation, Writer); LastLocation = BoilerplateLocation; if (LastLocation.LineIdx > 0 && Text.Lines[LastLocation.LineIdx - 1].TrimEnd().Length > 0) { if (LastLocation.LineIdx + 1 < Text.Lines.Length && Text.Lines[LastLocation.LineIdx].TrimEnd().Length == 0 && Text.Lines[LastLocation.LineIdx + 1].TrimEnd().Length == 0) { LastLocation.LineIdx++; } Writer.WriteLine(); } foreach (SourceFile MissingInclude in MissingIncludes) { string IncludeText = FormatInclude(Location.Directory, MissingInclude.Location, IncludePaths, SystemIncludePaths, Log); Writer.WriteLine("#include {0}", IncludeText); } } // Figure out before which markup object to write forward declarations, skipping over all the includes at the start of the file int ForwardDeclarationsBeforeMarkupIdx = -1; if ((Flags & SourceFileFlags.TranslationUnit) == 0) { int ConditionDepth = 0; for (int MarkupIdx = BodyMinIdx; MarkupIdx < Markup.Length; MarkupIdx++) { if (ConditionDepth == 0) { ForwardDeclarationsBeforeMarkupIdx = MarkupIdx; } if (Markup[MarkupIdx].Type == PreprocessorMarkupType.Text) { break; } ConditionDepth += Markup[MarkupIdx].GetConditionDepthDelta(); } } // Write all the other markup for (int MarkupIdx = BodyMinIdx; MarkupIdx < Markup.Length; MarkupIdx++) { PreprocessorMarkup ThisMarkup = Markup[MarkupIdx]; // Write the forward declarations if (MarkupIdx == ForwardDeclarationsBeforeMarkupIdx) { // Write out at least up to the end of the last markup if (MarkupIdx > 0 && LastLocation <= Markup[MarkupIdx - 1].EndLocation) { WriteLines(Text, LastLocation, Markup[MarkupIdx - 1].EndLocation, Writer); LastLocation = Markup[MarkupIdx - 1].EndLocation; } // Skip a blank line in the existing text. TextLocation NewLastLocation = LastLocation; if (LastLocation.LineIdx < Text.Lines.Length && String.IsNullOrWhiteSpace(Text.Lines[LastLocation.LineIdx])) { NewLastLocation = new TextLocation(LastLocation.LineIdx + 1, 0); } // Merge all the existing forward declarations with the new set. HashSet <string> PreviousForwardDeclarations = new HashSet <string>(); while (NewLastLocation.LineIdx < Text.Lines.Length) { string TrimLine = Text.Lines[NewLastLocation.LineIdx].Trim(); if (TrimLine.Length > 0 && !TrimLine.Equals("// Forward declarations", StringComparison.OrdinalIgnoreCase) && !TrimLine.Equals("// Forward declarations.", StringComparison.OrdinalIgnoreCase)) { // Create a token reader for the current line TokenReader Reader = new TokenReader(Text, new TextLocation(NewLastLocation.LineIdx, 0), new TextLocation(NewLastLocation.LineIdx, Text.Lines[NewLastLocation.LineIdx].Length)); // Read it into a buffer List <Token> Tokens = new List <Token>(); while (Reader.MoveNext()) { Tokens.Add(Reader.Current); } // Check it matches the syntax for a forward declaration, and add it to the list if it does if (Tokens.Count == 3 && (Tokens[0].Text == "struct" || Tokens[0].Text == "class") && Tokens[1].Type == TokenType.Identifier && Tokens[2].Text == ";") { PreviousForwardDeclarations.Add(String.Format("{0} {1};", Tokens[0].Text, Tokens[1].Text)); } else if (Tokens.Count == 4 && Tokens[0].Text == "enum" && Tokens[1].Text == "class" && Tokens[2].Type == TokenType.Identifier && Tokens[3].Text == ";") { PreviousForwardDeclarations.Add(String.Format("enum class {0};", Tokens[2].Text)); } else if (Tokens.Count == 6 && Tokens[0].Text == "enum" && Tokens[1].Text == "class" && Tokens[2].Type == TokenType.Identifier && Tokens[3].Text == ":" && Tokens[4].Type == TokenType.Identifier && Tokens[5].Text == ";") { PreviousForwardDeclarations.Add(String.Format("enum class {0} : {1};", Tokens[2].Text, Tokens[4].Text)); } else if (ForwardDeclarations.Contains(Text.Lines[NewLastLocation.LineIdx])) { PreviousForwardDeclarations.Add(Text.Lines[NewLastLocation.LineIdx]); } else { break; } } NewLastLocation = new TextLocation(NewLastLocation.LineIdx + 1, 0); } // Create a full list of new forward declarations, combining with the ones that are already there. Normally we optimize with the forward declarations present, // so we shouldn't remove any unless running a specific pass designed to do so. HashSet <string> MergedForwardDeclarations = new HashSet <string>(ForwardDeclarations); if (!bRemoveForwardDeclarations) { MergedForwardDeclarations.UnionWith(PreviousForwardDeclarations); } // Write them out if (MergedForwardDeclarations.Count > 0) { Writer.WriteLine(); foreach (string ForwardDeclaration in MergedForwardDeclarations.Distinct().OrderBy(x => GetForwardDeclarationSortKey(x)).ThenBy(x => x)) { Writer.WriteLine("{0}{1}", GetIndent(MarkupIdx), ForwardDeclaration); } Writer.WriteLine(); LastLocation = NewLastLocation; } else if (PreviousForwardDeclarations.Count > 0) { Writer.WriteLine(); LastLocation = NewLastLocation; } } // Write the includes if (ThisMarkup.Type == PreprocessorMarkupType.Include && ThisMarkup.IsActive && !ThisMarkup.IsInlineInclude()) { // Write up to the start of this include WriteLines(Text, LastLocation, ThisMarkup.Location, Writer); // Get the original include text. Some modules - particularly editor modules - include headers from other modules based from Engine/Source which are not listed as dependencies. If // the original include is based from a shallower directory than the one we would include otherwise, we'll use that instead. string OriginalIncludeText = null; if (ThisMarkup.Tokens.Count == 1) { OriginalIncludeText = ThisMarkup.Tokens[0].Text.Replace('\\', '/'); } // Write the replacement includes foreach (SourceFile OutputIncludedFile in ThisMarkup.OutputIncludedFiles) { string IncludeText = FormatInclude(Location.Directory, OutputIncludedFile.Location, IncludePaths, SystemIncludePaths, Log); if (OutputIncludedFile == ThisMarkup.IncludedFile && Rules.IsExternalIncludeMacro(ThisMarkup.Tokens)) { IncludeText = Token.Format(ThisMarkup.Tokens); } else if (OutputIncludedFile == ThisMarkup.IncludedFile && (OutputIncludedFile.Flags & SourceFileFlags.External) != 0) { IncludeText = OriginalIncludeText; } else if (OriginalIncludeText != null && (Flags & SourceFileFlags.TranslationUnit) == 0 && OriginalIncludeText.EndsWith(IncludeText.TrimStart('\"'), StringComparison.OrdinalIgnoreCase) && (OriginalIncludeText.StartsWith("\"Runtime/", StringComparison.InvariantCultureIgnoreCase) || OriginalIncludeText.StartsWith("\"Developer/", StringComparison.InvariantCultureIgnoreCase) || OriginalIncludeText.StartsWith("\"Editor/", StringComparison.InvariantCultureIgnoreCase))) { IncludeText = OriginalIncludeText; } Writer.WriteLine("{0}#include {1}", GetIndent(MarkupIdx), IncludeText); } // Copy any formatting if (ThisMarkup.EndLocation.LineIdx > ThisMarkup.Location.LineIdx + 1) { WriteLines(Text, new TextLocation(ThisMarkup.Location.LineIdx + 1, 0), ThisMarkup.EndLocation, Writer); } // Update the location to the start of the next line LastLocation = new TextLocation(ThisMarkup.Location.LineIdx + 1, 0); } } // Write to the end of the file WriteLines(Text, LastLocation, Text.End, Writer); }
/// <summary> /// Copy constructor /// </summary> /// <param name="Other">Token reader to copy from</param> public TokenReader(TokenReader Other) { Set(Other); }