public static DoxygenToken Make(DoxygenTokenKind kind, TextRange range, bool isComplete)
{
if (_pool == null)
{
_pool = new ObjectPool <DoxygenToken>(() => new DoxygenToken());
}
DoxygenToken result = _pool.Aquire();
result.Set(kind, range, isComplete);
return(result);
}
private CommandResult LexCommandTokens()
{
Debug.Assert(DoxygenSyntax.IsCommandBegin(Buffer.Peek()));
// Command
Buffer.StartLexeme();
Buffer.AdvanceColumn();
DoxygenTokenKind kind = DoxygenTokenKind.Command;
{
char first = Buffer.Peek();
switch (first)
{
case '{':
case '}':
kind = (first == '{') ? DoxygenTokenKind.GroupStart : DoxygenTokenKind.GroupEnd;
Buffer.AdvanceColumn();
break;
case '$':
case '@':
case '\\':
case '~':
case '<':
case '=':
case '>':
case '#':
case '"':
Buffer.AdvanceColumn();
break;
case ':':
case '|':
case '-':
Buffer.AdvanceColumnsWhile(d => d.Equals(first));
break;
default:
if (DoxygenSyntax.IsCommandIdentStart(first))
{
while (!Buffer.IsEOF)
{
if (!DoxygenSyntax.IsCommandIdentPart(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
break;
}
}
TextPosition commandStart = Buffer.LexemeStart;
int commandLen = Buffer.LexemeWidth;
string commandName = Buffer.GetSourceText(Buffer.LexemeStart.Index + 1, commandLen - 1);
var rule = DoxygenSyntax.GetCommandRule(commandName);
if (rule != null)
{
if (rule.Kind == DoxygenSyntax.CommandKind.StartCommandBlock)
{
kind = DoxygenTokenKind.CommandStart;
}
else if (rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock)
{
kind = DoxygenTokenKind.CommandEnd;
}
}
else
{
// @NOTE(final): Group start/end are not a "known" command
if (kind != DoxygenTokenKind.GroupStart && kind != DoxygenTokenKind.GroupEnd)
{
kind = DoxygenTokenKind.InvalidCommand;
}
}
DoxygenToken commandToken = DoxygenTokenPool.Make(kind, Buffer.LexemeRange, true);
PushToken(commandToken);
CommandResult result = new CommandResult(commandStart, rule, commandName);
string typeName = "Command";
if (rule != null)
{
int argNumber = 0;
int argCount = rule.Args.Count();
bool noMoreArgs = false;
foreach (var arg in rule.Args)
{
// @TODO(final): Handle rule repeat type for arguments on same type
char first = Buffer.Peek();
if (!arg.Flags.HasFlag(DoxygenSyntax.ArgumentFlags.DirectlyAfterCommand))
{
if (SyntaxUtils.IsSpacing(first) || first == '\t')
{
Buffer.SkipSpacings(TextStream.SkipType.All);
}
else
{
// No more arguments are following
noMoreArgs = true;
}
}
Buffer.StartLexeme();
// Prefix
string prefix = arg.Prefix;
string postfix = arg.Postfix;
bool hadPrefix = false;
if (prefix != null && !noMoreArgs)
{
if (!string.IsNullOrEmpty(prefix))
{
if (Buffer.CompareText(0, prefix) == 0)
{
Buffer.AdvanceColumns(prefix.Length);
hadPrefix = true;
}
}
else if ((prefix.Length == 0) && (!string.IsNullOrEmpty(postfix)))
{
hadPrefix = true;
}
}
switch (arg.Kind)
{
case DoxygenSyntax.ArgumentKind.PrefixToPostfix:
{
if (hadPrefix && !noMoreArgs)
{
Debug.Assert(!string.IsNullOrEmpty(postfix));
bool foundPrefixToPostfix = false;
while (!Buffer.IsEOF)
{
if (Buffer.CompareText(0, postfix) == 0)
{
Buffer.AdvanceColumns(postfix.Length);
foundPrefixToPostfix = true;
break;
}
else if (SyntaxUtils.IsLineBreak(Buffer.Peek()))
{
break;
}
else
{
Buffer.AdvanceColumn();
}
}
if (arg.IsOptional || foundPrefixToPostfix)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, foundPrefixToPostfix);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Expected postfix '{postfix}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
else if (arg.IsOptional)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, false);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Expected prefix '{prefix}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.MultipleObjectReference:
case DoxygenSyntax.ArgumentKind.SingleObjectReference:
{
// @TODO(final): ReferencedObject is not always a identifier
// Here are some examples of valid referenced objects:
// simple_identifier
// a_function()
// my::awesome::namespace::object
// my::awesome::namespace::function()
// my#awesome#namespace#function()
// method1,method2(),class#field
bool foundRef = false;
if (!noMoreArgs)
{
bool allowMultiple = arg.Kind == DoxygenSyntax.ArgumentKind.MultipleObjectReference;
bool requireIdent = true;
int referenceCount = 0;
while (!Buffer.IsEOF)
{
int oldPos = Buffer.StreamPosition;
char c0 = Buffer.Peek();
char c1 = Buffer.Peek(1);
if (!requireIdent)
{
if (c0 == ':' && c1 == ':')
{
Buffer.AdvanceColumns(2);
requireIdent = true;
continue;
}
else if (c0 == '#')
{
Buffer.AdvanceColumn();
requireIdent = true;
continue;
}
else if (c0 == ',' && referenceCount > 0 && allowMultiple)
{
Buffer.AdvanceColumn();
requireIdent = true;
continue;
}
else
{
// Correct termination of object-reference
foundRef = true;
break;
}
}
else
{
if (SyntaxUtils.IsIdentStart(c0))
{
requireIdent = false;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsIdentPart(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
if (Buffer.Peek() == '(')
{
// Parse until right parent
Buffer.AdvanceColumn();
bool terminatedFunc = false;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() == ')')
{
Buffer.AdvanceColumn();
terminatedFunc = true;
break;
}
Buffer.AdvanceAuto();
}
if (!terminatedFunc)
{
AddError(Buffer.TextPosition, $"Unterminated function reference for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
++referenceCount;
continue;
}
else
{
AddError(Buffer.TextPosition, $"Requires identifier, but found '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
}
if (Buffer.IsEOF)
{
// Correct termination of object-reference when stream ends (Single-line)
foundRef = true;
}
}
if (arg.IsOptional || foundRef)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentIdent, Buffer.LexemeRange, foundRef);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.Identifier:
{
bool foundIdent = false;
// Special handling for @param command and ... parameter
if (!noMoreArgs && "param".Equals(commandName) && (arg.Kind == DoxygenSyntax.ArgumentKind.Identifier))
{
if (Buffer.Peek() == '.')
{
char c1 = Buffer.Peek(1);
char c2 = Buffer.Peek(2);
if (c1 == '.' && c2 == '.')
{
Buffer.AdvanceColumns(3);
foundIdent = true;
}
}
}
// We dont allow parsing a ident, when any special handling was matched
if (!noMoreArgs && !foundIdent && SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
foundIdent = true;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsIdentPart(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
if (arg.IsOptional || foundIdent)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentIdent, Buffer.LexemeRange, foundIdent);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.HeaderFile:
case DoxygenSyntax.ArgumentKind.HeaderName:
{
bool foundFilename = false;
if (!noMoreArgs)
{
bool requiredQuotes = arg.Kind == DoxygenSyntax.ArgumentKind.HeaderName;
char curChar = Buffer.Peek();
if (curChar == '<' || curChar == '\"')
{
char quoteChar = curChar == '<' ? '>' : '\"';
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
curChar = Buffer.Peek();
if (curChar == quoteChar)
{
Buffer.AdvanceColumn();
foundFilename = true;
break;
}
else if (SyntaxUtils.IsLineBreak(curChar))
{
break;
}
Buffer.AdvanceColumn();
}
if (!foundFilename)
{
AddError(Buffer.TextPosition, $"Unterminated filename, expect quote char '{quoteChar}' but got '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
else if (!requiredQuotes)
{
if (SyntaxUtils.IsFilename(Buffer.Peek()))
{
foundFilename = true;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsFilename(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
}
}
if (arg.IsOptional || foundFilename)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentFile, Buffer.LexemeRange, foundFilename);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.SingleWord:
{
// @TODO(final): IsWordStart()
bool foundWord = false;
if (!noMoreArgs && char.IsLetterOrDigit(Buffer.Peek()))
{
foundWord = true;
while (!Buffer.IsEOF)
{
// @TODO(final): IsWordPart()
if (char.IsWhiteSpace(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
if (arg.IsOptional || foundWord)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, foundWord);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.QuotedString:
{
bool isComplete = false;
// @TODO(final): Make quotes configurable in the argument rule
bool hasQuote = Buffer.Peek() == '"' || Buffer.Peek() == '<';
char endQuote = char.MaxValue;
if (hasQuote && !noMoreArgs)
{
endQuote = Buffer.Peek() == '<' ? '>' : '"';
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
if (!hasQuote)
{
if (char.IsWhiteSpace(Buffer.Peek()))
{
break;
}
}
else
{
if (Buffer.Peek() == endQuote)
{
Buffer.AdvanceColumn();
isComplete = true;
break;
}
else if (SyntaxUtils.IsLineBreak(Buffer.Peek()) || Buffer.Peek() == TextStream.InvalidCharacter)
{
break;
}
}
Buffer.AdvanceColumn();
}
if (!isComplete)
{
AddError(Buffer.TextPosition, $"Unterminated quote string for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
if (arg.IsOptional || isComplete)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentText, Buffer.LexemeRange, isComplete);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.UntilEndOfLine:
{
bool eolFound = false;
if (!noMoreArgs)
{
while (!Buffer.IsEOF)
{
if (SyntaxUtils.IsLineBreak(Buffer.Peek()))
{
eolFound = true;
break;
}
Buffer.AdvanceColumn();
}
if (Buffer.IsEOF)
{
eolFound = true;
}
}
if (arg.IsOptional || eolFound)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentText, Buffer.LexemeRange, true);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unterminated end-of-line for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.ComplexLine:
case DoxygenSyntax.ArgumentKind.ComplexBlock:
// @TODO(final): Implement complex line/block properly
goto CommandDone;
default:
AddError(Buffer.TextPosition, $"Unsupported argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
// Postfix
if (!noMoreArgs && (hadPrefix && !string.IsNullOrWhiteSpace(postfix) && arg.Kind != DoxygenSyntax.ArgumentKind.PrefixToPostfix))
{
if (Buffer.CompareText(0, postfix) == 0)
{
Buffer.AdvanceColumns(prefix.Length);
}
else
{
AddError(Buffer.TextPosition, $"Expected postfix '{postfix}' for pp-argument({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
++argNumber;
}
}
CommandDone:
result.IsValid = true;
return(result);
}
public void Set(DoxygenTokenKind kind, TextRange range, bool isComplete)
{
Set(range, isComplete);
Kind = kind;
}
private DoxygenToken(DoxygenTokenKind kind, TextRange range, bool isComplete) : base(range, isComplete)
{
Kind = kind;
}
private bool ParseCommand(LinkedListStream <IBaseToken> stream, IBaseNode contentRoot)
{
// @NOTE(final): This must always return true, due to the fact that the stream is advanced at least once
DoxygenToken commandToken = stream.Peek <DoxygenToken>();
Debug.Assert(commandToken != null && commandToken.Kind == DoxygenTokenKind.Command);
string commandName = commandToken.Value.Substring(1);
stream.Next();
string typeName = "Command";
var rule = DoxygenSyntax.GetCommandRule(commandName);
if (rule != null)
{
if (rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock)
{
var t = Top;
if (t == null)
{
AddError(commandToken.Position, $"Unterminated starting command block in command '{commandName}'", typeName, commandName);
return(false);
}
if (t.Entity.Kind != DoxygenEntityKind.BlockCommand)
{
AddError(commandToken.Position, $"Expect starting command block, but found '{t.Entity.Kind}' in command '{commandName}'", typeName, commandName);
return(false);
}
Pop();
}
// Paragraph or section command starts or command block starts -> Close previous paragraph or sectioning command
if (rule.Kind == DoxygenSyntax.CommandKind.Paragraph ||
rule.Kind == DoxygenSyntax.CommandKind.Section ||
rule.Kind == DoxygenSyntax.CommandKind.StartCommandBlock)
{
var t = Top;
if (t != null)
{
if (t.Entity.Kind == DoxygenEntityKind.Paragraph ||
t.Entity.Kind == DoxygenEntityKind.Section ||
t.Entity.Kind == DoxygenEntityKind.SubSection ||
t.Entity.Kind == DoxygenEntityKind.SubSubSection)
{
Pop();
}
}
}
DoxygenEntity commandEntity = null;
IEntityBaseNode <DoxygenEntity> commandNode = null;
if (rule.EntityKind != DoxygenEntityKind.None)
{
commandEntity = new DoxygenEntity(rule.EntityKind, commandToken.Range);
commandEntity.Id = commandName;
commandNode = new DoxygenNode(Top, commandEntity);
if (rule.IsPush)
{
Push(commandNode);
}
else
{
Add(commandNode);
}
}
foreach (var arg in rule.Args)
{
DoxygenToken argToken = stream.Peek <DoxygenToken>();
if (argToken == null)
{
break;
}
DoxygenTokenKind expectedTokenKind = DoxygenSyntax.ArgumentToTokenKindMap.ContainsKey(arg.Kind) ? DoxygenSyntax.ArgumentToTokenKindMap[arg.Kind] : DoxygenTokenKind.Invalid;
if (expectedTokenKind == DoxygenTokenKind.Invalid)
{
break;
}
if (expectedTokenKind != argToken.Kind)
{
AddError(argToken.Position, $"Expect argument token '{expectedTokenKind}', but got '{argToken.Kind}'", typeName, commandName);
break;
}
if (commandNode != null)
{
string paramName = arg.Name;
string paramValue = argToken.Value;
commandNode.Entity.AddParameter(argToken, paramName, paramValue);
}
stream.Next();
}
if (commandEntity != null)
{
// Get name and text parameter (Some commands, have different names and text parameters, so there is a variable list of strings)
var nameParam = commandEntity.FindParameterByName("name", "id");
var textParam = commandEntity.FindParameterByName("text", "title", "caption");
if (nameParam == null || string.IsNullOrWhiteSpace(nameParam.Value))
{
if (rule.Kind == DoxygenSyntax.CommandKind.Section)
{
if (!"mainpage".Equals(commandName))
{
AddError(commandToken.Position, $"Missing identifier mapping for command '{commandName}'", typeName, commandName);
}
}
}
if (nameParam != null && !string.IsNullOrWhiteSpace(nameParam.Value))
{
string symbolName = nameParam.Value;
Debug.Assert(commandNode != null);
if (rule.Kind == DoxygenSyntax.CommandKind.Section)
{
SourceSymbolKind kind = SourceSymbolKind.DoxygenSection;
if ("page".Equals(commandName) || "mainpage".Equals(commandName))
{
kind = SourceSymbolKind.DoxygenPage;
}
SymbolCache.AddSource(Tag, symbolName, new SourceSymbol(kind, nameParam.Token.Range, commandNode));
}
else if ("ref".Equals(commandName) || "refitem".Equals(commandName))
{
string referenceValue = nameParam.Value;
TextPosition startPos = new TextPosition(0, nameParam.Token.Position.Line, nameParam.Token.Position.Column);
using (TextStream referenceTextStream = new BasicTextStream(referenceValue, startPos, referenceValue.Length))
{
ReferenceSymbolKind referenceTarget = ReferenceSymbolKind.Any;
while (!referenceTextStream.IsEOF)
{
char first = referenceTextStream.Peek();
char second = referenceTextStream.Peek(1);
char third = referenceTextStream.Peek(2);
if (SyntaxUtils.IsIdentStart(first))
{
referenceTextStream.StartLexeme();
while (!referenceTextStream.IsEOF)
{
if (!SyntaxUtils.IsIdentPart(referenceTextStream.Peek()))
{
break;
}
referenceTextStream.AdvanceColumn();
}
var refRange = referenceTextStream.LexemeRange;
string singleRereference = referenceTextStream.GetSourceText(refRange.Index, refRange.Length);
if (referenceTextStream.Peek() == '(')
{
referenceTarget = ReferenceSymbolKind.CppFunction;
referenceTextStream.AdvanceColumn();
while (!referenceTextStream.IsEOF)
{
if (referenceTextStream.Peek() == ')')
{
break;
}
referenceTextStream.AdvanceColumn();
}
}
var symbolRange = new TextRange(new TextPosition(nameParam.Token.Position.Index + refRange.Position.Index, refRange.Position.Line, refRange.Position.Column), refRange.Length);
SymbolCache.AddReference(Tag, singleRereference, new ReferenceSymbol(referenceTarget, symbolRange, commandNode));
}
else if (first == '#' || first == '.')
{
referenceTarget = ReferenceSymbolKind.CppMember;
referenceTextStream.AdvanceColumn();
}
else if (first == ':' || second == ':')
{
referenceTarget = ReferenceSymbolKind.CppMember;
referenceTextStream.AdvanceColumns(2);
}
else
{
break;
}
}
}
}
else if ("subpage".Equals(commandName))
{
SymbolCache.AddReference(Tag, symbolName, new ReferenceSymbol(ReferenceSymbolKind.DoxygenPage, nameParam.Token.Range, commandNode));
}
}
}
ParseBlockContent(stream, commandNode);
}
else
{
AddError(commandToken.Position, $"No parse rule for command '{commandName}' found", "Command", commandName);
}
return(true);
}
public ExpectToken(DoxygenTokenKind kind, string value) : this(kind, value.Length, value.Length, value)
{
}
public ExpectToken(DoxygenTokenKind kind, int length, string value = null) : this(kind, length, length, value)
{
}
public ExpectToken(DoxygenTokenKind kind, int minLength, int maxLength, string value = null)
{
Kind = kind;
MinLength = MaxLength = minLength;
Value = value;
}