private bool ParseReservedKeyword(LinkedListStream <IBaseToken> stream) { CppToken keywordToken = stream.Peek <CppToken>(); Debug.Assert(keywordToken.Kind == CppTokenKind.ReservedKeyword); string keyword = keywordToken.Value; switch (keyword) { case "typedef": ParseTypedef(stream); return(true); case "struct": case "union": case "class": ParseStruct(stream); return(true); case "enum": ParseEnum(stream, keywordToken.Position); return(true); default: return(false); } }
public override bool ParseToken(LinkedListStream <IBaseToken> stream) { var token = stream.Peek <CppToken>(); if (token == null) { return(false); } switch (token.Kind) { case CppTokenKind.PreprocessorStart: return(ParsePreprocessor(stream)); case CppTokenKind.ReservedKeyword: return(ParseReservedKeyword(stream)); case CppTokenKind.IdentLiteral: { if (IsToken(stream, SearchMode.Next, CppTokenKind.LeftParen)) { return(ParseFunction(stream)); } return(false); } } return(false); }
private bool ParseSingleBlock(LinkedListStream <IBaseToken> stream) { // @NOTE(final) Single block = auto-brief IBaseToken blockToken = stream.Peek(); DoxygenEntity blockEntity = new DoxygenEntity(DoxygenEntityKind.BlockSingle, blockToken); PushEntity(blockEntity); stream.Next(); IBaseToken endToken = null; IBaseToken briefToken = DoxygenTokenPool.Make(DoxygenTokenKind.Command, blockToken.Range, true); DoxygenEntity briefEntity = new DoxygenEntity(DoxygenEntityKind.Brief, briefToken); DoxygenNode briefNode = PushEntity(briefEntity); while (!stream.IsEOF) { IBaseToken token = stream.Peek(); DoxygenToken doxyToken = token as DoxygenToken; if (doxyToken != null && doxyToken.Kind == DoxygenTokenKind.DoxyBlockEnd) { endToken = token; stream.Next(); break; } if (!ParseBlockContent(stream, briefNode)) { break; } Debug.Assert(stream.CurrentValue != token); } Pop(); // Pop brief Pop(); // Pop block if (endToken != null) { blockEntity.EndRange = endToken.Range; } return(true); }
private void ParseStruct(LinkedListStream <IBaseToken> stream) { CppToken structKeywordToken = stream.Peek <CppToken>(); Debug.Assert(structKeywordToken.Kind == CppTokenKind.ReservedKeyword && ("struct".Equals(structKeywordToken.Value) || "union".Equals(structKeywordToken.Value))); var typedefResult = Search(stream.CurrentNode, SearchMode.Prev, (t) => t.Kind == CppTokenKind.ReservedKeyword && "typedef".Equals(t.Value)); bool isTypedef = typedefResult != null; stream.Next(); // There are multiple ways to define a struct in C // A = typedef struct { int a; } foo; // B = typedef struct foo { int a; } sFoo; (prefered style) // C = struct { int a; }; (Anonymous struct without a name) // D = struct { int a; } memberName; (Anonymous struct named as a member) // E = struct foo { int a; }; (C++ style) // B/E var identTokenResult = Search(stream, SearchMode.Current, CppTokenKind.IdentLiteral); if (identTokenResult != null) { var identToken = identTokenResult.Token; string structIdent = identToken.Value; stream.Next(); CppEntityKind kind = CppEntityKind.Struct; var terminatorTokenResult = Search(stream, SearchMode.Current, CppTokenKind.Semicolon, CppTokenKind.LeftBrace); if (terminatorTokenResult != null) { var terminatorToken = terminatorTokenResult.Token; if (terminatorToken.Kind == CppTokenKind.Semicolon) { kind = CppEntityKind.ForwardStruct; } stream.Next(); } CppEntity structEntity = new CppEntity(kind, identToken, structIdent) { DocumentationNode = FindDocumentationNode(identTokenResult.Node, 1), }; CppNode structNode = new CppNode(Top, structEntity); Add(structNode); SymbolCache.AddSource(Tag, structIdent, new SourceSymbol(SourceSymbolKind.CppStruct, identToken.Range, structNode)); } // @TODO(final): Parse struct members }
private void ParseText(LinkedListStream <IBaseToken> stream, IBaseNode contentNode) { var nextToken = stream.Peek <DoxygenToken>(); Debug.Assert(nextToken.Kind == DoxygenTokenKind.TextStart); TextPosition textStart = nextToken.Position; TextPosition textEnd = textStart; stream.Next(); while (!stream.IsEOF) { var t = stream.Peek <DoxygenToken>(); if (t == null) { break; } if (t.Kind == DoxygenTokenKind.TextEnd) { textEnd = t.Position; stream.Next(); break; } stream.Next(); } if (contentNode != null) { int textLen = textEnd.Index - textStart.Index; string text = Source.Substring(textStart.Index, textLen).Trim(); if (text.Length > 0) { var textNode = new DoxygenNode(contentNode, new DoxygenEntity(DoxygenEntityKind.Text, new TextRange(textStart, textLen))); textNode.Entity.Value = text; textNode.Entity.EndRange = new TextRange(textEnd, 0); contentNode.AddChild(textNode); } } }
private bool ParsePreprocessor(LinkedListStream <IBaseToken> stream) { CppToken token = stream.Peek <CppToken>(); Debug.Assert(token.Kind == CppTokenKind.PreprocessorStart); stream.Next(); var keywordResult = Search(stream, SearchMode.Current, CppTokenKind.PreprocessorKeyword); if (keywordResult != null) { var keywordToken = keywordResult.Token; stream.Next(); if ("define".Equals(keywordResult.Token.Value)) { var defineResult = Search(stream, SearchMode.Current, CppTokenKind.PreprocessorDefineSource); if (defineResult != null) { var defineNameToken = defineResult.Token; string defineName = defineNameToken.Value; stream.Next(); var defineValueResult = Search(stream, SearchMode.Current, CppTokenKind.IdentLiteral); if (defineValueResult != null) { stream.Next(); } CppEntity defineKeyEntity = new CppEntity(CppEntityKind.Define, defineNameToken, defineName) { DocumentationNode = FindDocumentationNode(defineResult.Node, 1), Value = defineValueResult?.Token.Value }; CppNode defineNode = new CppNode(Top, defineKeyEntity); Add(defineNode); SymbolCache.AddSource(Tag, defineName, new SourceSymbol(SourceSymbolKind.CppDefine, defineNameToken.Range, defineNode)); } else { AddError(keywordToken.Position, $"Processor define has no identifier!", "Define"); } } } return(true); }
public override bool ParseToken(LinkedListStream <IBaseToken> stream) { IBaseToken token = stream.Peek(); if (typeof(DoxygenToken).Equals(token.GetType())) { DoxygenToken doxyToken = (DoxygenToken)token; switch (doxyToken.Kind) { case DoxygenTokenKind.DoxyBlockStartSingle: return(ParseSingleBlock(stream)); case DoxygenTokenKind.DoxyBlockStartMulti: { DoxygenEntity blockEntity = new DoxygenEntity(DoxygenEntityKind.BlockMulti, doxyToken); PushEntity(blockEntity); stream.Next(); return(true); } case DoxygenTokenKind.DoxyBlockEnd: { CloseEverythingUntil(DoxygenEntityKind.BlockMulti); Debug.Assert(Top != null); DoxygenEntity rootEntity = (DoxygenEntity)Top.Entity; Debug.Assert(rootEntity.Kind == DoxygenEntityKind.BlockMulti); Pop(); rootEntity.EndRange = doxyToken.Range; stream.Next(); return(true); } default: { ParseBlockContent(stream, Top); return(true); } } } else { return(false); } }
private bool ParseBlockContent(LinkedListStream <IBaseToken> stream, IBaseNode contentRoot) { IBaseToken token = stream.Peek(); if (typeof(DoxygenToken).Equals(token.GetType())) { DoxygenToken doxyToken = (DoxygenToken)token; switch (doxyToken.Kind) { case DoxygenTokenKind.EmptyLine: CloseParagraphOrSection(contentRoot); return(true); case DoxygenTokenKind.Command: return(ParseCommand(stream, contentRoot)); case DoxygenTokenKind.InvalidCommand: string commandName = doxyToken.Value.Substring(1); AddError(doxyToken.Position, $"Unknown doxygen command '{commandName}'", "Command", commandName); stream.Next(); return(true); case DoxygenTokenKind.TextStart: ParseText(stream, contentRoot); return(true); default: stream.Next(); return(true); } } else { return(false); } }
private bool ParseFunction(LinkedListStream <IBaseToken> stream) { var functionIdentNode = stream.CurrentNode; CppToken functionIdentToken = functionIdentNode.Value as CppToken; Debug.Assert(functionIdentToken.Kind == CppTokenKind.IdentLiteral); var functionName = functionIdentToken.Value; CppTokenKind[] funcCallKinds = { CppTokenKind.EqOp, CppTokenKind.Comma, CppTokenKind.LeftParen, CppTokenKind.AddOp, CppTokenKind.AddAssign, CppTokenKind.SubOp, CppTokenKind.SubAssign, CppTokenKind.DivOp, CppTokenKind.DivAssign, CppTokenKind.MulAssign, CppTokenKind.XorOp, CppTokenKind.XorAssign, CppTokenKind.LeftShiftOp, CppTokenKind.LeftShiftAssign, CppTokenKind.RightShiftOp, CppTokenKind.RightShiftAssign, }; var funcCallResult = Search(stream, SearchMode.Prev, funcCallKinds); CppEntityKind kind = CppEntityKind.FunctionCall; // Skip ident stream.Next(); // Skip parameters CppToken openParenToken = stream.Peek <CppToken>(); Debug.Assert(openParenToken.Kind == CppTokenKind.LeftParen); stream.Next(); var closeParenResult = Search(stream, SearchMode.Forward, CppTokenKind.RightParen); if (closeParenResult == null) { AddError(openParenToken.Position, $"Unterminated function '{functionName}'", "Function", functionName); return(true); } stream.Seek(closeParenResult.Node); stream.Next(); if (funcCallResult == null) { var endingTokenResult = Search(stream, SearchMode.Current, CppTokenKind.Semicolon, CppTokenKind.LeftBrace); if (endingTokenResult != null) { stream.Next(); if (endingTokenResult.Token.Kind == CppTokenKind.LeftBrace) { kind = CppEntityKind.FunctionBody; if (Configuration.SkipFunctionBlocks) { return(true); } } else { kind = CppEntityKind.FunctionDefinition; } } } CppEntity functionEntity = new CppEntity(kind, functionIdentToken, functionName) { DocumentationNode = FindDocumentationNode(functionIdentNode, 1), }; CppNode functionNode = new CppNode(Top, functionEntity); Add(functionNode); if (kind == CppEntityKind.FunctionDefinition) { SymbolCache.AddSource(Tag, functionName, new SourceSymbol(SourceSymbolKind.CppFunctionDefinition, functionIdentToken.Range, functionNode)); } else if (kind == CppEntityKind.FunctionBody) { SymbolCache.AddSource(Tag, functionName, new SourceSymbol(SourceSymbolKind.CppFunctionBody, functionIdentToken.Range, functionNode)); } else if (Configuration.FunctionCallSymbolsEnabled) { SymbolCache.AddReference(Tag, functionName, new ReferenceSymbol(ReferenceSymbolKind.CppFunction, functionIdentToken.Range, functionNode)); } return(true); }
private void ParseTypedef(LinkedListStream <IBaseToken> stream) { CppToken typedefToken = stream.Peek <CppToken>(); Debug.Assert(typedefToken.Kind == CppTokenKind.ReservedKeyword && "typedef".Equals(typedefToken.Value)); stream.Next(); var reservedKeywordResult = Search(stream, SearchMode.Current, CppTokenKind.ReservedKeyword); if (reservedKeywordResult != null) { var reservedKeywordToken = reservedKeywordResult.Token; string keyword = reservedKeywordToken.Value; switch (keyword) { case "union": case "struct": ParseStruct(stream); return; case "enum": ParseEnum(stream, reservedKeywordToken.Position); return; } } // Normal typedef CppEntityKind kind = CppEntityKind.Typedef; var semicolonResult = Search(stream, SearchMode.Forward, CppTokenKind.Semicolon); if (semicolonResult != null) { stream.Seek(semicolonResult.Node); stream.Next(); SearchResult <CppToken> identResult = null; var prevResult = Search(semicolonResult, SearchMode.Prev, CppTokenKind.RightParen, CppTokenKind.IdentLiteral); if (prevResult != null) { if (prevResult.Token.Kind == CppTokenKind.RightParen) { // Function typedef var leftParenResult = Search(prevResult, SearchMode.Backward, CppTokenKind.LeftParen); if (leftParenResult != null) { var rightParentResult = Search(leftParenResult, SearchMode.Prev, CppTokenKind.RightParen); if (rightParentResult != null) { leftParenResult = Search(rightParentResult, SearchMode.Backward, CppTokenKind.LeftParen); if (leftParenResult != null) { identResult = Search(leftParenResult, SearchMode.Next, CppTokenKind.IdentLiteral); kind = CppEntityKind.FunctionTypedef; } } else { // Typedef on define // #define MY_TYPEDEF(name) // typedef MY_TYPEDEF(my_typedef); //Debug.WriteLine("Right paren not found!"); } } } else { // Type typedef Debug.Assert(prevResult.Token.Kind == CppTokenKind.IdentLiteral); identResult = prevResult; } } if (identResult != null) { var identToken = identResult.Token; string typedefIdent = identToken.Value; CppEntity typedefEntity = new CppEntity(kind, identToken, typedefIdent) { DocumentationNode = FindDocumentationNode(identResult.Node, 1), }; var typedefNode = new CppNode(Top, typedefEntity); Add(typedefNode); SymbolCache.AddSource(Tag, typedefIdent, new SourceSymbol(SourceSymbolKind.CppType, identToken.Range)); } } }
private void ParseEnum(LinkedListStream <IBaseToken> stream, TextPosition pos) { CppToken enumBaseToken = stream.Peek <CppToken>(); Debug.Assert(enumBaseToken.Kind == CppTokenKind.ReservedKeyword && "enum".Equals(enumBaseToken.Value)); stream.Next(); SearchResult <CppToken> enumIdentResult = null; var classKeywordResult = Search(stream, SearchMode.Current, CppTokenKind.ReservedKeyword); if (classKeywordResult != null) { string keyword = classKeywordResult.Token.Value; if ("class".Equals(keyword)) { // enum class stream.Next(); enumIdentResult = Search(stream, SearchMode.Current, CppTokenKind.IdentLiteral); if (enumIdentResult == null) { var token = stream.Peek <CppToken>(); AddError(pos, $"Expect identifier token, but got token kind {token?.Kind} for enum", "enum"); return; } } } if (enumIdentResult == null) { // Ident after enum var identAfterEnumResult = Search(stream, SearchMode.Current, CppTokenKind.IdentLiteral); if (identAfterEnumResult != null) { stream.Next(); enumIdentResult = identAfterEnumResult; } } if (enumIdentResult != null) { var braceOrSemiResult = Search(stream, SearchMode.Current, CppTokenKind.LeftBrace, CppTokenKind.Semicolon); if (braceOrSemiResult == null) { return; } var enumIdentToken = enumIdentResult.Token; string enumIdent = enumIdentToken.Value; CppEntityKind kind = braceOrSemiResult.Token.Kind == CppTokenKind.Semicolon ? CppEntityKind.ForwardEnum : CppEntityKind.Enum; CppEntity enumRootEntity = new CppEntity(kind, enumIdentToken, enumIdent) { DocumentationNode = FindDocumentationNode(enumIdentResult.Node, 1), }; CppNode enumRootNode = new CppNode(Top, enumRootEntity); Add(enumRootNode); SymbolCache.AddSource(Tag, enumIdent, new SourceSymbol(SourceSymbolKind.CppEnum, enumIdentToken.Range, enumRootNode)); if (braceOrSemiResult.Token.Kind == CppTokenKind.LeftBrace) { ParseEnumValues(stream, enumRootNode); } else { stream.Next(); } } else { var token = stream.Peek <CppToken>(); AddError(pos, $"Expect identifier token, but got token kind {token?.Kind} for enum", "enum"); return; } }
private void ParseEnumValues(LinkedListStream <IBaseToken> stream, CppNode rootNode) { CppToken leftBraceToken = stream.Peek <CppToken>(); Debug.Assert(leftBraceToken.Kind == CppTokenKind.LeftBrace); stream.Next(); while (!stream.IsEOF) { var rightBraceResult = Search(stream, SearchMode.Current, CppTokenKind.RightBrace); if (rightBraceResult != null) { // Enum complete stream.Next(); break; } var identResult = Search(stream, SearchMode.Current, CppTokenKind.IdentLiteral); if (identResult != null) { stream.Seek(identResult.Node); // Enum value var enumValueToken = identResult.Token; string enumValueName = enumValueToken.Value; stream.Next(); CppEntity enumValueEntity = new CppEntity(CppEntityKind.EnumValue, enumValueToken, enumValueName) { DocumentationNode = FindDocumentationNode(identResult.Node, 1), }; CppNode enumValueNode = new CppNode(rootNode, enumValueEntity); rootNode.AddChild(enumValueNode); SymbolCache.AddSource(Tag, enumValueName, new SourceSymbol(SourceSymbolKind.CppMember, enumValueToken.Range, enumValueNode)); var equalsResult = Search(stream, SearchMode.Current, CppTokenKind.EqOp); if (equalsResult != null) { stream.Next(); // Skip until comma or right brace var tmpResult = Search(stream, SearchMode.Forward, CppTokenKind.Comma, CppTokenKind.RightBrace); if (tmpResult != null) { stream.Seek(tmpResult.Node); } else { AddError(equalsResult.Token.Position, $"Expect assignment token, but got token '{stream.Peek()}' for enum member '{enumValueName}'", "Enum", enumValueName); break; } } var commaOrRightBraceResult = Search(stream, SearchMode.Current, CppTokenKind.Comma, CppTokenKind.RightBrace); if (commaOrRightBraceResult != null) { stream.Seek(commaOrRightBraceResult.Node); if (commaOrRightBraceResult.Token.Kind == CppTokenKind.Comma) { stream.Next(); } continue; } else { var tok = stream.Peek <CppToken>(); if (tok != null) { AddError(tok.Position, $"Unexpected token '{tok.Kind}'", "EnumValue"); } break; } } stream.Next(); } }
private bool ParseCommand(LinkedListStream <IBaseToken> stream, IBaseNode contentRoot) { // @NOTE(final): This must always return true, due to the fact that the stream is advanced at least once DoxygenToken commandToken = stream.Peek <DoxygenToken>(); Debug.Assert(commandToken != null && commandToken.Kind == DoxygenTokenKind.Command); string commandName = commandToken.Value.Substring(1); stream.Next(); string typeName = "Command"; var rule = DoxygenSyntax.GetCommandRule(commandName); if (rule != null) { if (rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock) { var t = Top; if (t == null) { AddError(commandToken.Position, $"Unterminated starting command block in command '{commandName}'", typeName, commandName); return(false); } if (t.Entity.Kind != DoxygenEntityKind.BlockCommand) { AddError(commandToken.Position, $"Expect starting command block, but found '{t.Entity.Kind}' in command '{commandName}'", typeName, commandName); return(false); } Pop(); } // Paragraph or section command starts or command block starts -> Close previous paragraph or sectioning command if (rule.Kind == DoxygenSyntax.CommandKind.Paragraph || rule.Kind == DoxygenSyntax.CommandKind.Section || rule.Kind == DoxygenSyntax.CommandKind.StartCommandBlock) { var t = Top; if (t != null) { if (t.Entity.Kind == DoxygenEntityKind.Paragraph || t.Entity.Kind == DoxygenEntityKind.Section || t.Entity.Kind == DoxygenEntityKind.SubSection || t.Entity.Kind == DoxygenEntityKind.SubSubSection) { Pop(); } } } DoxygenEntity commandEntity = null; IEntityBaseNode <DoxygenEntity> commandNode = null; if (rule.EntityKind != DoxygenEntityKind.None) { commandEntity = new DoxygenEntity(rule.EntityKind, commandToken.Range); commandEntity.Id = commandName; commandNode = new DoxygenNode(Top, commandEntity); if (rule.IsPush) { Push(commandNode); } else { Add(commandNode); } } foreach (var arg in rule.Args) { DoxygenToken argToken = stream.Peek <DoxygenToken>(); if (argToken == null) { break; } DoxygenTokenKind expectedTokenKind = DoxygenSyntax.ArgumentToTokenKindMap.ContainsKey(arg.Kind) ? DoxygenSyntax.ArgumentToTokenKindMap[arg.Kind] : DoxygenTokenKind.Invalid; if (expectedTokenKind == DoxygenTokenKind.Invalid) { break; } if (expectedTokenKind != argToken.Kind) { AddError(argToken.Position, $"Expect argument token '{expectedTokenKind}', but got '{argToken.Kind}'", typeName, commandName); break; } if (commandNode != null) { string paramName = arg.Name; string paramValue = argToken.Value; commandNode.Entity.AddParameter(argToken, paramName, paramValue); } stream.Next(); } if (commandEntity != null) { // Get name and text parameter (Some commands, have different names and text parameters, so there is a variable list of strings) var nameParam = commandEntity.FindParameterByName("name", "id"); var textParam = commandEntity.FindParameterByName("text", "title", "caption"); if (nameParam == null || string.IsNullOrWhiteSpace(nameParam.Value)) { if (rule.Kind == DoxygenSyntax.CommandKind.Section) { if (!"mainpage".Equals(commandName)) { AddError(commandToken.Position, $"Missing identifier mapping for command '{commandName}'", typeName, commandName); } } } if (nameParam != null && !string.IsNullOrWhiteSpace(nameParam.Value)) { string symbolName = nameParam.Value; Debug.Assert(commandNode != null); if (rule.Kind == DoxygenSyntax.CommandKind.Section) { SourceSymbolKind kind = SourceSymbolKind.DoxygenSection; if ("page".Equals(commandName) || "mainpage".Equals(commandName)) { kind = SourceSymbolKind.DoxygenPage; } SymbolCache.AddSource(Tag, symbolName, new SourceSymbol(kind, nameParam.Token.Range, commandNode)); } else if ("ref".Equals(commandName) || "refitem".Equals(commandName)) { string referenceValue = nameParam.Value; TextPosition startPos = new TextPosition(0, nameParam.Token.Position.Line, nameParam.Token.Position.Column); using (TextStream referenceTextStream = new BasicTextStream(referenceValue, startPos, referenceValue.Length)) { ReferenceSymbolKind referenceTarget = ReferenceSymbolKind.Any; while (!referenceTextStream.IsEOF) { char first = referenceTextStream.Peek(); char second = referenceTextStream.Peek(1); char third = referenceTextStream.Peek(2); if (SyntaxUtils.IsIdentStart(first)) { referenceTextStream.StartLexeme(); while (!referenceTextStream.IsEOF) { if (!SyntaxUtils.IsIdentPart(referenceTextStream.Peek())) { break; } referenceTextStream.AdvanceColumn(); } var refRange = referenceTextStream.LexemeRange; string singleRereference = referenceTextStream.GetSourceText(refRange.Index, refRange.Length); if (referenceTextStream.Peek() == '(') { referenceTarget = ReferenceSymbolKind.CppFunction; referenceTextStream.AdvanceColumn(); while (!referenceTextStream.IsEOF) { if (referenceTextStream.Peek() == ')') { break; } referenceTextStream.AdvanceColumn(); } } var symbolRange = new TextRange(new TextPosition(nameParam.Token.Position.Index + refRange.Position.Index, refRange.Position.Line, refRange.Position.Column), refRange.Length); SymbolCache.AddReference(Tag, singleRereference, new ReferenceSymbol(referenceTarget, symbolRange, commandNode)); } else if (first == '#' || first == '.') { referenceTarget = ReferenceSymbolKind.CppMember; referenceTextStream.AdvanceColumn(); } else if (first == ':' || second == ':') { referenceTarget = ReferenceSymbolKind.CppMember; referenceTextStream.AdvanceColumns(2); } else { break; } } } } else if ("subpage".Equals(commandName)) { SymbolCache.AddReference(Tag, symbolName, new ReferenceSymbol(ReferenceSymbolKind.DoxygenPage, nameParam.Token.Range, commandNode)); } } } ParseBlockContent(stream, commandNode); } else { AddError(commandToken.Position, $"No parse rule for command '{commandName}' found", "Command", commandName); } return(true); }