private CommandResult LexCommandTokens() { Debug.Assert(DoxygenSyntax.IsCommandBegin(Buffer.Peek())); // Command Buffer.StartLexeme(); Buffer.AdvanceColumn(); DoxygenTokenKind kind = DoxygenTokenKind.Command; { char first = Buffer.Peek(); switch (first) { case '{': case '}': kind = (first == '{') ? DoxygenTokenKind.GroupStart : DoxygenTokenKind.GroupEnd; Buffer.AdvanceColumn(); break; case '$': case '@': case '\\': case '~': case '<': case '=': case '>': case '#': case '"': Buffer.AdvanceColumn(); break; case ':': case '|': case '-': Buffer.AdvanceColumnsWhile(d => d.Equals(first)); break; default: if (DoxygenSyntax.IsCommandIdentStart(first)) { while (!Buffer.IsEOF) { if (!DoxygenSyntax.IsCommandIdentPart(Buffer.Peek())) { break; } Buffer.AdvanceColumn(); } } break; } } TextPosition commandStart = Buffer.LexemeStart; int commandLen = Buffer.LexemeWidth; string commandName = Buffer.GetSourceText(Buffer.LexemeStart.Index + 1, commandLen - 1); var rule = DoxygenSyntax.GetCommandRule(commandName); if (rule != null) { if (rule.Kind == DoxygenSyntax.CommandKind.StartCommandBlock) { kind = DoxygenTokenKind.CommandStart; } else if (rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock) { kind = DoxygenTokenKind.CommandEnd; } } else { // @NOTE(final): Group start/end are not a "known" command if (kind != DoxygenTokenKind.GroupStart && kind != DoxygenTokenKind.GroupEnd) { kind = DoxygenTokenKind.InvalidCommand; } } DoxygenToken commandToken = DoxygenTokenPool.Make(kind, Buffer.LexemeRange, true); PushToken(commandToken); CommandResult result = new CommandResult(commandStart, rule, commandName); string typeName = "Command"; if (rule != null) { int argNumber = 0; int argCount = rule.Args.Count(); bool noMoreArgs = false; foreach (var arg in rule.Args) { // @TODO(final): Handle rule repeat type for arguments on same type char first = Buffer.Peek(); if (!arg.Flags.HasFlag(DoxygenSyntax.ArgumentFlags.DirectlyAfterCommand)) { if (SyntaxUtils.IsSpacing(first) || first == '\t') { Buffer.SkipSpacings(TextStream.SkipType.All); } else { // No more arguments are following noMoreArgs = true; } } Buffer.StartLexeme(); // Prefix string prefix = arg.Prefix; string postfix = arg.Postfix; bool hadPrefix = false; if (prefix != null && !noMoreArgs) { if (!string.IsNullOrEmpty(prefix)) { if (Buffer.CompareText(0, prefix) == 0) { Buffer.AdvanceColumns(prefix.Length); hadPrefix = true; } } else if ((prefix.Length == 0) && (!string.IsNullOrEmpty(postfix))) { hadPrefix = true; } } switch (arg.Kind) { case DoxygenSyntax.ArgumentKind.PrefixToPostfix: { if (hadPrefix && !noMoreArgs) { Debug.Assert(!string.IsNullOrEmpty(postfix)); bool foundPrefixToPostfix = false; while (!Buffer.IsEOF) { if (Buffer.CompareText(0, postfix) == 0) { Buffer.AdvanceColumns(postfix.Length); foundPrefixToPostfix = true; break; } else if (SyntaxUtils.IsLineBreak(Buffer.Peek())) { break; } else { Buffer.AdvanceColumn(); } } if (arg.IsOptional || foundPrefixToPostfix) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, foundPrefixToPostfix); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Expected postfix '{postfix}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } else if (arg.IsOptional) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, false); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Expected prefix '{prefix}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.MultipleObjectReference: case DoxygenSyntax.ArgumentKind.SingleObjectReference: { // @TODO(final): ReferencedObject is not always a identifier // Here are some examples of valid referenced objects: // simple_identifier // a_function() // my::awesome::namespace::object // my::awesome::namespace::function() // my#awesome#namespace#function() // method1,method2(),class#field bool foundRef = false; if (!noMoreArgs) { bool allowMultiple = arg.Kind == DoxygenSyntax.ArgumentKind.MultipleObjectReference; bool requireIdent = true; int referenceCount = 0; while (!Buffer.IsEOF) { int oldPos = Buffer.StreamPosition; char c0 = Buffer.Peek(); char c1 = Buffer.Peek(1); if (!requireIdent) { if (c0 == ':' && c1 == ':') { Buffer.AdvanceColumns(2); requireIdent = true; continue; } else if (c0 == '#') { Buffer.AdvanceColumn(); requireIdent = true; continue; } else if (c0 == ',' && referenceCount > 0 && allowMultiple) { Buffer.AdvanceColumn(); requireIdent = true; continue; } else { // Correct termination of object-reference foundRef = true; break; } } else { if (SyntaxUtils.IsIdentStart(c0)) { requireIdent = false; while (!Buffer.IsEOF) { if (!SyntaxUtils.IsIdentPart(Buffer.Peek())) { break; } Buffer.AdvanceColumn(); } if (Buffer.Peek() == '(') { // Parse until right parent Buffer.AdvanceColumn(); bool terminatedFunc = false; while (!Buffer.IsEOF) { if (Buffer.Peek() == ')') { Buffer.AdvanceColumn(); terminatedFunc = true; break; } Buffer.AdvanceAuto(); } if (!terminatedFunc) { AddError(Buffer.TextPosition, $"Unterminated function reference for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } ++referenceCount; continue; } else { AddError(Buffer.TextPosition, $"Requires identifier, but found '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } } if (Buffer.IsEOF) { // Correct termination of object-reference when stream ends (Single-line) foundRef = true; } } if (arg.IsOptional || foundRef) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentIdent, Buffer.LexemeRange, foundRef); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.Identifier: { bool foundIdent = false; // Special handling for @param command and ... parameter if (!noMoreArgs && "param".Equals(commandName) && (arg.Kind == DoxygenSyntax.ArgumentKind.Identifier)) { if (Buffer.Peek() == '.') { char c1 = Buffer.Peek(1); char c2 = Buffer.Peek(2); if (c1 == '.' && c2 == '.') { Buffer.AdvanceColumns(3); foundIdent = true; } } } // We dont allow parsing a ident, when any special handling was matched if (!noMoreArgs && !foundIdent && SyntaxUtils.IsIdentStart(Buffer.Peek())) { foundIdent = true; while (!Buffer.IsEOF) { if (!SyntaxUtils.IsIdentPart(Buffer.Peek())) { break; } Buffer.AdvanceColumn(); } } if (arg.IsOptional || foundIdent) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentIdent, Buffer.LexemeRange, foundIdent); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.HeaderFile: case DoxygenSyntax.ArgumentKind.HeaderName: { bool foundFilename = false; if (!noMoreArgs) { bool requiredQuotes = arg.Kind == DoxygenSyntax.ArgumentKind.HeaderName; char curChar = Buffer.Peek(); if (curChar == '<' || curChar == '\"') { char quoteChar = curChar == '<' ? '>' : '\"'; Buffer.AdvanceColumn(); while (!Buffer.IsEOF) { curChar = Buffer.Peek(); if (curChar == quoteChar) { Buffer.AdvanceColumn(); foundFilename = true; break; } else if (SyntaxUtils.IsLineBreak(curChar)) { break; } Buffer.AdvanceColumn(); } if (!foundFilename) { AddError(Buffer.TextPosition, $"Unterminated filename, expect quote char '{quoteChar}' but got '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } else if (!requiredQuotes) { if (SyntaxUtils.IsFilename(Buffer.Peek())) { foundFilename = true; while (!Buffer.IsEOF) { if (!SyntaxUtils.IsFilename(Buffer.Peek())) { break; } Buffer.AdvanceColumn(); } } } } if (arg.IsOptional || foundFilename) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentFile, Buffer.LexemeRange, foundFilename); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.SingleWord: { // @TODO(final): IsWordStart() bool foundWord = false; if (!noMoreArgs && char.IsLetterOrDigit(Buffer.Peek())) { foundWord = true; while (!Buffer.IsEOF) { // @TODO(final): IsWordPart() if (char.IsWhiteSpace(Buffer.Peek())) { break; } Buffer.AdvanceColumn(); } } if (arg.IsOptional || foundWord) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, foundWord); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.QuotedString: { bool isComplete = false; // @TODO(final): Make quotes configurable in the argument rule bool hasQuote = Buffer.Peek() == '"' || Buffer.Peek() == '<'; char endQuote = char.MaxValue; if (hasQuote && !noMoreArgs) { endQuote = Buffer.Peek() == '<' ? '>' : '"'; Buffer.AdvanceColumn(); while (!Buffer.IsEOF) { if (!hasQuote) { if (char.IsWhiteSpace(Buffer.Peek())) { break; } } else { if (Buffer.Peek() == endQuote) { Buffer.AdvanceColumn(); isComplete = true; break; } else if (SyntaxUtils.IsLineBreak(Buffer.Peek()) || Buffer.Peek() == TextStream.InvalidCharacter) { break; } } Buffer.AdvanceColumn(); } if (!isComplete) { AddError(Buffer.TextPosition, $"Unterminated quote string for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } if (arg.IsOptional || isComplete) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentText, Buffer.LexemeRange, isComplete); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.UntilEndOfLine: { bool eolFound = false; if (!noMoreArgs) { while (!Buffer.IsEOF) { if (SyntaxUtils.IsLineBreak(Buffer.Peek())) { eolFound = true; break; } Buffer.AdvanceColumn(); } if (Buffer.IsEOF) { eolFound = true; } } if (arg.IsOptional || eolFound) { DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentText, Buffer.LexemeRange, true); PushToken(argToken); } else if (arg.IsRequired) { AddError(Buffer.TextPosition, $"Unterminated end-of-line for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } break; case DoxygenSyntax.ArgumentKind.ComplexLine: case DoxygenSyntax.ArgumentKind.ComplexBlock: // @TODO(final): Implement complex line/block properly goto CommandDone; default: AddError(Buffer.TextPosition, $"Unsupported argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } // Postfix if (!noMoreArgs && (hadPrefix && !string.IsNullOrWhiteSpace(postfix) && arg.Kind != DoxygenSyntax.ArgumentKind.PrefixToPostfix)) { if (Buffer.CompareText(0, postfix) == 0) { Buffer.AdvanceColumns(prefix.Length); } else { AddError(Buffer.TextPosition, $"Expected postfix '{postfix}' for pp-argument({argNumber}:{arg}) in command '{commandName}'", typeName, commandName); return(result); } } ++argNumber; } } CommandDone: result.IsValid = true; return(result); }