private bool LexUntilCodeEnd(CommandResult commandResult)
{
// Special case, we dont want to parse doxygen stuff inside a code section
// so we wait until a @endcode follows
bool isComplete = false;
while (!Buffer.IsEOF)
{
char c0 = Buffer.Peek();
char c1 = Buffer.Peek(1);
if ((c0 == '@' || c0 == '\\') && SyntaxUtils.IsIdentStart(c1))
{
Buffer.StartLexeme();
Buffer.AdvanceColumn();
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsIdentPart);
string ident = Buffer.GetSourceText(Buffer.LexemeStart.Index + 1, Buffer.LexemeWidth - 1);
if ("endcode".Equals(ident))
{
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.CommandEnd, Buffer.LexemeRange, true));
isComplete = true;
break;
}
}
else if (SyntaxUtils.IsLineBreak(c0))
{
int lb = SyntaxUtils.GetLineBreakChars(c0, c1);
Buffer.AdvanceLine(lb);
}
else if ('\t'.Equals(c0))
{
Buffer.AdvanceTab();
}
else
{
Buffer.AdvanceColumn();
}
}
if (!isComplete)
{
AddError(commandResult.StartPos, $"Unterminated code-block, expect '@endcode' or '\\endcode'", "Code", commandResult.CommandName);
return(false);
}
return(true);
}
private LexResult LexIdent(bool isPreprocessor)
{
Debug.Assert(SyntaxUtils.IsIdentStart(Buffer.Peek()));
StringBuilder identBuffer = new StringBuilder();
while (!Buffer.IsEOF)
{
char c = Buffer.Peek();
if (SyntaxUtils.IsIdentPart(c))
{
identBuffer.Append(c);
Buffer.AdvanceColumn();
}
else
{
break;
}
}
CppTokenKind kind = CppTokenKind.IdentLiteral;
TextPosition identStart = Buffer.LexemeStart;
int identLength = Buffer.LexemeWidth;
string identString = identBuffer.ToString();
if (isPreprocessor && PreProcessorKeywords.Contains(identString))
{
kind = CppTokenKind.PreprocessorKeyword;
}
else if (ReservedKeywords.Contains(identString))
{
kind = CppTokenKind.ReservedKeyword;
}
else if (TypeKeywords.Contains(identString) || GlobalClassKeywords.Contains(identString))
{
kind = CppTokenKind.TypeKeyword;
}
else
{
kind = CppTokenKind.IdentLiteral;
}
return(new LexResult(kind, true));
}
private CommandResult LexCommandTokens()
{
Debug.Assert(DoxygenSyntax.IsCommandBegin(Buffer.Peek()));
// Command
Buffer.StartLexeme();
Buffer.AdvanceColumn();
DoxygenTokenKind kind = DoxygenTokenKind.Command;
{
char first = Buffer.Peek();
switch (first)
{
case '{':
case '}':
kind = (first == '{') ? DoxygenTokenKind.GroupStart : DoxygenTokenKind.GroupEnd;
Buffer.AdvanceColumn();
break;
case '$':
case '@':
case '\\':
case '~':
case '<':
case '=':
case '>':
case '#':
case '"':
Buffer.AdvanceColumn();
break;
case ':':
case '|':
case '-':
Buffer.AdvanceColumnsWhile(d => d.Equals(first));
break;
default:
if (DoxygenSyntax.IsCommandIdentStart(first))
{
while (!Buffer.IsEOF)
{
if (!DoxygenSyntax.IsCommandIdentPart(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
break;
}
}
TextPosition commandStart = Buffer.LexemeStart;
int commandLen = Buffer.LexemeWidth;
string commandName = Buffer.GetSourceText(Buffer.LexemeStart.Index + 1, commandLen - 1);
var rule = DoxygenSyntax.GetCommandRule(commandName);
if (rule != null)
{
if (rule.Kind == DoxygenSyntax.CommandKind.StartCommandBlock)
{
kind = DoxygenTokenKind.CommandStart;
}
else if (rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock)
{
kind = DoxygenTokenKind.CommandEnd;
}
}
else
{
// @NOTE(final): Group start/end are not a "known" command
if (kind != DoxygenTokenKind.GroupStart && kind != DoxygenTokenKind.GroupEnd)
{
kind = DoxygenTokenKind.InvalidCommand;
}
}
DoxygenToken commandToken = DoxygenTokenPool.Make(kind, Buffer.LexemeRange, true);
PushToken(commandToken);
CommandResult result = new CommandResult(commandStart, rule, commandName);
string typeName = "Command";
if (rule != null)
{
int argNumber = 0;
int argCount = rule.Args.Count();
bool noMoreArgs = false;
foreach (var arg in rule.Args)
{
// @TODO(final): Handle rule repeat type for arguments on same type
char first = Buffer.Peek();
if (!arg.Flags.HasFlag(DoxygenSyntax.ArgumentFlags.DirectlyAfterCommand))
{
if (SyntaxUtils.IsSpacing(first) || first == '\t')
{
Buffer.SkipSpacings(TextStream.SkipType.All);
}
else
{
// No more arguments are following
noMoreArgs = true;
}
}
Buffer.StartLexeme();
// Prefix
string prefix = arg.Prefix;
string postfix = arg.Postfix;
bool hadPrefix = false;
if (prefix != null && !noMoreArgs)
{
if (!string.IsNullOrEmpty(prefix))
{
if (Buffer.CompareText(0, prefix) == 0)
{
Buffer.AdvanceColumns(prefix.Length);
hadPrefix = true;
}
}
else if ((prefix.Length == 0) && (!string.IsNullOrEmpty(postfix)))
{
hadPrefix = true;
}
}
switch (arg.Kind)
{
case DoxygenSyntax.ArgumentKind.PrefixToPostfix:
{
if (hadPrefix && !noMoreArgs)
{
Debug.Assert(!string.IsNullOrEmpty(postfix));
bool foundPrefixToPostfix = false;
while (!Buffer.IsEOF)
{
if (Buffer.CompareText(0, postfix) == 0)
{
Buffer.AdvanceColumns(postfix.Length);
foundPrefixToPostfix = true;
break;
}
else if (SyntaxUtils.IsLineBreak(Buffer.Peek()))
{
break;
}
else
{
Buffer.AdvanceColumn();
}
}
if (arg.IsOptional || foundPrefixToPostfix)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, foundPrefixToPostfix);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Expected postfix '{postfix}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
else if (arg.IsOptional)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, false);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Expected prefix '{prefix}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.MultipleObjectReference:
case DoxygenSyntax.ArgumentKind.SingleObjectReference:
{
// @TODO(final): ReferencedObject is not always a identifier
// Here are some examples of valid referenced objects:
// simple_identifier
// a_function()
// my::awesome::namespace::object
// my::awesome::namespace::function()
// my#awesome#namespace#function()
// method1,method2(),class#field
bool foundRef = false;
if (!noMoreArgs)
{
bool allowMultiple = arg.Kind == DoxygenSyntax.ArgumentKind.MultipleObjectReference;
bool requireIdent = true;
int referenceCount = 0;
while (!Buffer.IsEOF)
{
int oldPos = Buffer.StreamPosition;
char c0 = Buffer.Peek();
char c1 = Buffer.Peek(1);
if (!requireIdent)
{
if (c0 == ':' && c1 == ':')
{
Buffer.AdvanceColumns(2);
requireIdent = true;
continue;
}
else if (c0 == '#')
{
Buffer.AdvanceColumn();
requireIdent = true;
continue;
}
else if (c0 == ',' && referenceCount > 0 && allowMultiple)
{
Buffer.AdvanceColumn();
requireIdent = true;
continue;
}
else
{
// Correct termination of object-reference
foundRef = true;
break;
}
}
else
{
if (SyntaxUtils.IsIdentStart(c0))
{
requireIdent = false;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsIdentPart(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
if (Buffer.Peek() == '(')
{
// Parse until right parent
Buffer.AdvanceColumn();
bool terminatedFunc = false;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() == ')')
{
Buffer.AdvanceColumn();
terminatedFunc = true;
break;
}
Buffer.AdvanceAuto();
}
if (!terminatedFunc)
{
AddError(Buffer.TextPosition, $"Unterminated function reference for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
++referenceCount;
continue;
}
else
{
AddError(Buffer.TextPosition, $"Requires identifier, but found '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
}
if (Buffer.IsEOF)
{
// Correct termination of object-reference when stream ends (Single-line)
foundRef = true;
}
}
if (arg.IsOptional || foundRef)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentIdent, Buffer.LexemeRange, foundRef);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.Identifier:
{
bool foundIdent = false;
// Special handling for @param command and ... parameter
if (!noMoreArgs && "param".Equals(commandName) && (arg.Kind == DoxygenSyntax.ArgumentKind.Identifier))
{
if (Buffer.Peek() == '.')
{
char c1 = Buffer.Peek(1);
char c2 = Buffer.Peek(2);
if (c1 == '.' && c2 == '.')
{
Buffer.AdvanceColumns(3);
foundIdent = true;
}
}
}
// We dont allow parsing a ident, when any special handling was matched
if (!noMoreArgs && !foundIdent && SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
foundIdent = true;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsIdentPart(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
if (arg.IsOptional || foundIdent)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentIdent, Buffer.LexemeRange, foundIdent);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.HeaderFile:
case DoxygenSyntax.ArgumentKind.HeaderName:
{
bool foundFilename = false;
if (!noMoreArgs)
{
bool requiredQuotes = arg.Kind == DoxygenSyntax.ArgumentKind.HeaderName;
char curChar = Buffer.Peek();
if (curChar == '<' || curChar == '\"')
{
char quoteChar = curChar == '<' ? '>' : '\"';
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
curChar = Buffer.Peek();
if (curChar == quoteChar)
{
Buffer.AdvanceColumn();
foundFilename = true;
break;
}
else if (SyntaxUtils.IsLineBreak(curChar))
{
break;
}
Buffer.AdvanceColumn();
}
if (!foundFilename)
{
AddError(Buffer.TextPosition, $"Unterminated filename, expect quote char '{quoteChar}' but got '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
else if (!requiredQuotes)
{
if (SyntaxUtils.IsFilename(Buffer.Peek()))
{
foundFilename = true;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsFilename(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
}
}
if (arg.IsOptional || foundFilename)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentFile, Buffer.LexemeRange, foundFilename);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.SingleWord:
{
// @TODO(final): IsWordStart()
bool foundWord = false;
if (!noMoreArgs && char.IsLetterOrDigit(Buffer.Peek()))
{
foundWord = true;
while (!Buffer.IsEOF)
{
// @TODO(final): IsWordPart()
if (char.IsWhiteSpace(Buffer.Peek()))
{
break;
}
Buffer.AdvanceColumn();
}
}
if (arg.IsOptional || foundWord)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentCaption, Buffer.LexemeRange, foundWord);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.QuotedString:
{
bool isComplete = false;
// @TODO(final): Make quotes configurable in the argument rule
bool hasQuote = Buffer.Peek() == '"' || Buffer.Peek() == '<';
char endQuote = char.MaxValue;
if (hasQuote && !noMoreArgs)
{
endQuote = Buffer.Peek() == '<' ? '>' : '"';
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
if (!hasQuote)
{
if (char.IsWhiteSpace(Buffer.Peek()))
{
break;
}
}
else
{
if (Buffer.Peek() == endQuote)
{
Buffer.AdvanceColumn();
isComplete = true;
break;
}
else if (SyntaxUtils.IsLineBreak(Buffer.Peek()) || Buffer.Peek() == TextStream.InvalidCharacter)
{
break;
}
}
Buffer.AdvanceColumn();
}
if (!isComplete)
{
AddError(Buffer.TextPosition, $"Unterminated quote string for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
if (arg.IsOptional || isComplete)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentText, Buffer.LexemeRange, isComplete);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unexpected character '{Buffer.Peek()}' for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.UntilEndOfLine:
{
bool eolFound = false;
if (!noMoreArgs)
{
while (!Buffer.IsEOF)
{
if (SyntaxUtils.IsLineBreak(Buffer.Peek()))
{
eolFound = true;
break;
}
Buffer.AdvanceColumn();
}
if (Buffer.IsEOF)
{
eolFound = true;
}
}
if (arg.IsOptional || eolFound)
{
DoxygenToken argToken = DoxygenTokenPool.Make(DoxygenTokenKind.ArgumentText, Buffer.LexemeRange, true);
PushToken(argToken);
}
else if (arg.IsRequired)
{
AddError(Buffer.TextPosition, $"Unterminated end-of-line for argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
break;
case DoxygenSyntax.ArgumentKind.ComplexLine:
case DoxygenSyntax.ArgumentKind.ComplexBlock:
// @TODO(final): Implement complex line/block properly
goto CommandDone;
default:
AddError(Buffer.TextPosition, $"Unsupported argument ({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
// Postfix
if (!noMoreArgs && (hadPrefix && !string.IsNullOrWhiteSpace(postfix) && arg.Kind != DoxygenSyntax.ArgumentKind.PrefixToPostfix))
{
if (Buffer.CompareText(0, postfix) == 0)
{
Buffer.AdvanceColumns(prefix.Length);
}
else
{
AddError(Buffer.TextPosition, $"Expected postfix '{postfix}' for pp-argument({argNumber}:{arg}) in command '{commandName}'", typeName, commandName);
return(result);
}
}
++argNumber;
}
}
CommandDone:
result.IsValid = true;
return(result);
}
private bool ParseCommand(LinkedListStream <IBaseToken> stream, IBaseNode contentRoot)
{
// @NOTE(final): This must always return true, due to the fact that the stream is advanced at least once
DoxygenToken commandToken = stream.Peek <DoxygenToken>();
Debug.Assert(commandToken != null && commandToken.Kind == DoxygenTokenKind.Command);
string commandName = commandToken.Value.Substring(1);
stream.Next();
string typeName = "Command";
var rule = DoxygenSyntax.GetCommandRule(commandName);
if (rule != null)
{
if (rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock)
{
var t = Top;
if (t == null)
{
AddError(commandToken.Position, $"Unterminated starting command block in command '{commandName}'", typeName, commandName);
return(false);
}
if (t.Entity.Kind != DoxygenEntityKind.BlockCommand)
{
AddError(commandToken.Position, $"Expect starting command block, but found '{t.Entity.Kind}' in command '{commandName}'", typeName, commandName);
return(false);
}
Pop();
}
// Paragraph or section command starts or command block starts -> Close previous paragraph or sectioning command
if (rule.Kind == DoxygenSyntax.CommandKind.Paragraph ||
rule.Kind == DoxygenSyntax.CommandKind.Section ||
rule.Kind == DoxygenSyntax.CommandKind.StartCommandBlock)
{
var t = Top;
if (t != null)
{
if (t.Entity.Kind == DoxygenEntityKind.Paragraph ||
t.Entity.Kind == DoxygenEntityKind.Section ||
t.Entity.Kind == DoxygenEntityKind.SubSection ||
t.Entity.Kind == DoxygenEntityKind.SubSubSection)
{
Pop();
}
}
}
DoxygenEntity commandEntity = null;
IEntityBaseNode <DoxygenEntity> commandNode = null;
if (rule.EntityKind != DoxygenEntityKind.None)
{
commandEntity = new DoxygenEntity(rule.EntityKind, commandToken.Range);
commandEntity.Id = commandName;
commandNode = new DoxygenNode(Top, commandEntity);
if (rule.IsPush)
{
Push(commandNode);
}
else
{
Add(commandNode);
}
}
foreach (var arg in rule.Args)
{
DoxygenToken argToken = stream.Peek <DoxygenToken>();
if (argToken == null)
{
break;
}
DoxygenTokenKind expectedTokenKind = DoxygenSyntax.ArgumentToTokenKindMap.ContainsKey(arg.Kind) ? DoxygenSyntax.ArgumentToTokenKindMap[arg.Kind] : DoxygenTokenKind.Invalid;
if (expectedTokenKind == DoxygenTokenKind.Invalid)
{
break;
}
if (expectedTokenKind != argToken.Kind)
{
AddError(argToken.Position, $"Expect argument token '{expectedTokenKind}', but got '{argToken.Kind}'", typeName, commandName);
break;
}
if (commandNode != null)
{
string paramName = arg.Name;
string paramValue = argToken.Value;
commandNode.Entity.AddParameter(argToken, paramName, paramValue);
}
stream.Next();
}
if (commandEntity != null)
{
// Get name and text parameter (Some commands, have different names and text parameters, so there is a variable list of strings)
var nameParam = commandEntity.FindParameterByName("name", "id");
var textParam = commandEntity.FindParameterByName("text", "title", "caption");
if (nameParam == null || string.IsNullOrWhiteSpace(nameParam.Value))
{
if (rule.Kind == DoxygenSyntax.CommandKind.Section)
{
if (!"mainpage".Equals(commandName))
{
AddError(commandToken.Position, $"Missing identifier mapping for command '{commandName}'", typeName, commandName);
}
}
}
if (nameParam != null && !string.IsNullOrWhiteSpace(nameParam.Value))
{
string symbolName = nameParam.Value;
Debug.Assert(commandNode != null);
if (rule.Kind == DoxygenSyntax.CommandKind.Section)
{
SourceSymbolKind kind = SourceSymbolKind.DoxygenSection;
if ("page".Equals(commandName) || "mainpage".Equals(commandName))
{
kind = SourceSymbolKind.DoxygenPage;
}
SymbolCache.AddSource(Tag, symbolName, new SourceSymbol(kind, nameParam.Token.Range, commandNode));
}
else if ("ref".Equals(commandName) || "refitem".Equals(commandName))
{
string referenceValue = nameParam.Value;
TextPosition startPos = new TextPosition(0, nameParam.Token.Position.Line, nameParam.Token.Position.Column);
using (TextStream referenceTextStream = new BasicTextStream(referenceValue, startPos, referenceValue.Length))
{
ReferenceSymbolKind referenceTarget = ReferenceSymbolKind.Any;
while (!referenceTextStream.IsEOF)
{
char first = referenceTextStream.Peek();
char second = referenceTextStream.Peek(1);
char third = referenceTextStream.Peek(2);
if (SyntaxUtils.IsIdentStart(first))
{
referenceTextStream.StartLexeme();
while (!referenceTextStream.IsEOF)
{
if (!SyntaxUtils.IsIdentPart(referenceTextStream.Peek()))
{
break;
}
referenceTextStream.AdvanceColumn();
}
var refRange = referenceTextStream.LexemeRange;
string singleRereference = referenceTextStream.GetSourceText(refRange.Index, refRange.Length);
if (referenceTextStream.Peek() == '(')
{
referenceTarget = ReferenceSymbolKind.CppFunction;
referenceTextStream.AdvanceColumn();
while (!referenceTextStream.IsEOF)
{
if (referenceTextStream.Peek() == ')')
{
break;
}
referenceTextStream.AdvanceColumn();
}
}
var symbolRange = new TextRange(new TextPosition(nameParam.Token.Position.Index + refRange.Position.Index, refRange.Position.Line, refRange.Position.Column), refRange.Length);
SymbolCache.AddReference(Tag, singleRereference, new ReferenceSymbol(referenceTarget, symbolRange, commandNode));
}
else if (first == '#' || first == '.')
{
referenceTarget = ReferenceSymbolKind.CppMember;
referenceTextStream.AdvanceColumn();
}
else if (first == ':' || second == ':')
{
referenceTarget = ReferenceSymbolKind.CppMember;
referenceTextStream.AdvanceColumns(2);
}
else
{
break;
}
}
}
}
else if ("subpage".Equals(commandName))
{
SymbolCache.AddReference(Tag, symbolName, new ReferenceSymbol(ReferenceSymbolKind.DoxygenPage, nameParam.Token.Range, commandNode));
}
}
}
ParseBlockContent(stream, commandNode);
}
else
{
AddError(commandToken.Position, $"No parse rule for command '{commandName}' found", "Command", commandName);
}
return(true);
}
public static bool IsCommandIdentStart(char c)
{
bool result = SyntaxUtils.IsIdentStart(c);
return(result);
}
protected override bool LexNext(State hiddenState)
{
CppLexerState state = (CppLexerState)hiddenState;
bool allowWhitespaces = !state.IsInsidePreprocessor;
if (allowWhitespaces)
{
Buffer.SkipAllWhitespaces();
}
if (Buffer.IsEOF)
{
return(false);
}
int line = Buffer.TextPosition.Line;
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
Buffer.StartLexeme();
LexResult lexRes = new LexResult(CppTokenKind.Unknown, true);
switch (first)
{
case '&':
{
if (second == '&')
{
lexRes.Kind = CppTokenKind.LogicalAndOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AndAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AndOp;
Buffer.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
lexRes.Kind = CppTokenKind.LogicalOrOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.OrAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.OrOp;
Buffer.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.EqOp;
Buffer.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.LogicalNotEqualsOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ExclationMark;
Buffer.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.LeftShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.LeftShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.LessOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.LessThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
lexRes.Kind = CppTokenKind.RightShiftAssign;
Buffer.AdvanceColumns(3);
}
else
{
lexRes.Kind = CppTokenKind.RightShiftOp;
Buffer.AdvanceColumns(2);
}
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.GreaterOrEqualOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.GreaterThanOp;
Buffer.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
lexRes.Kind = CppTokenKind.IncOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.AddAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.AddOp;
Buffer.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
lexRes.Kind = CppTokenKind.DecOp;
Buffer.AdvanceColumns(2);
}
else if (second == '=')
{
lexRes.Kind = CppTokenKind.SubAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '>')
{
lexRes.Kind = CppTokenKind.PtrOp;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.SubOp;
Buffer.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.DivAssign;
Buffer.AdvanceColumns(2);
}
else if (second == '/')
{
lexRes = LexSingleLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '\n' or '\r' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else if (second == '*')
{
lexRes = LexMultiLineComment(Buffer, true);
if (!lexRes.IsComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '*/' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
}
}
else
{
Buffer.AdvanceColumn();
lexRes.Kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.MulAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.MulOp;
Buffer.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.ModAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.ModOp;
Buffer.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
lexRes.Kind = CppTokenKind.Ellipsis;
Buffer.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
lexRes = LexNumber();
}
else
{
lexRes.Kind = CppTokenKind.Dot;
Buffer.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
lexRes.Kind = CppTokenKind.XorAssign;
Buffer.AdvanceColumns(2);
}
else
{
lexRes.Kind = CppTokenKind.XorOp;
Buffer.AdvanceColumn();
}
}
break;
case '#':
return(LexPreprocessor(state));
case '"':
lexRes = LexString("string");
break;
case '\'':
lexRes = LexString("char");
break;
case '~':
lexRes.Kind = CppTokenKind.Tilde;
Buffer.AdvanceColumn();
break;
case '\\':
lexRes.Kind = CppTokenKind.Backslash;
Buffer.AdvanceColumn();
break;
case ',':
lexRes.Kind = CppTokenKind.Comma;
Buffer.AdvanceColumn();
break;
case ';':
lexRes.Kind = CppTokenKind.Semicolon;
Buffer.AdvanceColumn();
break;
case ':':
lexRes.Kind = CppTokenKind.Colon;
Buffer.AdvanceColumn();
break;
case '?':
lexRes.Kind = CppTokenKind.QuestionMark;
Buffer.AdvanceColumn();
break;
case '{':
lexRes.Kind = CppTokenKind.LeftBrace;
Buffer.AdvanceColumn();
break;
case '}':
lexRes.Kind = CppTokenKind.RightBrace;
Buffer.AdvanceColumn();
break;
case '[':
lexRes.Kind = CppTokenKind.LeftBracket;
Buffer.AdvanceColumn();
break;
case ']':
lexRes.Kind = CppTokenKind.RightBracket;
Buffer.AdvanceColumn();
break;
case '(':
lexRes.Kind = CppTokenKind.LeftParen;
Buffer.AdvanceColumn();
break;
case ')':
lexRes.Kind = CppTokenKind.RightParen;
Buffer.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(nb);
}
else if (first == '\t' && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() != '\t')
{
break;
}
Buffer.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first) && allowWhitespaces)
{
lexRes.Kind = CppTokenKind.Spacings;
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
Debug.Assert(!state.IsInsidePreprocessor);
lexRes = LexIdent(false);
}
else if (SyntaxUtils.IsNumeric(first))
{
lexRes = LexNumber();
}
else
{
AddError(Buffer.TextPosition, $"Unexpected character '{first}'", "Character");
return(false);
}
}
break;
}
return(PushToken(CppTokenPool.Make(lexRes.Kind, Buffer.LexemeRange, lexRes.IsComplete)));
}
private bool LexPreprocessor(CppLexerState state)
{
Debug.Assert(Buffer.Peek() == '#');
state.StartPreprocessor();
// Preprocessor start
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorStart, Buffer.LexemeRange, true));
do
{
Buffer.SkipSpacings(TextStream.SkipType.All);
Buffer.StartLexeme();
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
if (first == '\\')
{
if (SyntaxUtils.IsLineBreak(second))
{
Buffer.AdvanceColumn();
int lb = SyntaxUtils.GetLineBreakChars(second, third);
Buffer.AdvanceLine(lb);
continue;
}
else
{
AddError(Buffer.TextPosition, $"Unterminated preprocessor next-line, expect linebreak after '\' but got '{second}'", "Preprocessor");
return(false);
}
}
else if (first == '#')
{
Buffer.AdvanceColumn();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorOperator, Buffer.LexemeRange, true));
}
else if (SyntaxUtils.IsLineBreak(first))
{
int lb = SyntaxUtils.GetLineBreakChars(first, second);
Buffer.AdvanceLine(lb);
PushToken(CppTokenPool.Make(CppTokenKind.EndOfLine, Buffer.LexemeRange, true));
break;
}
else if (SyntaxUtils.IsIdentStart(first))
{
LexResult identResult = LexIdent(true);
CppToken identToken = CppTokenPool.Make(identResult.Kind, Buffer.LexemeRange, identResult.IsComplete);
PushToken(identToken);
Buffer.SkipSpacings(TextStream.SkipType.All);
Buffer.StartLexeme();
if (identToken.Kind == CppTokenKind.PreprocessorKeyword)
{
switch (identToken.Value)
{
case "define":
{
if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
AddError(Buffer.TextPosition, $"Expect identifier for define, but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
LexResult defineValueResult = LexIdent(false);
CppToken defineValueToken = CppTokenPool.Make(CppTokenKind.PreprocessorDefineSource, Buffer.LexemeRange, defineValueResult.IsComplete);
PushToken(defineValueToken);
}
break;
case "defined":
{
if (Buffer.Peek() == '(')
{
Buffer.AdvanceColumn();
if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
AddError(Buffer.TextPosition, $"Expect identifier for defined, but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
LexResult definedValueResult = LexIdent(false);
CppToken definedValueToken = CppTokenPool.Make(CppTokenKind.PreprocessorDefineTarget, Buffer.LexemeRange, definedValueResult.IsComplete);
PushToken(definedValueToken);
Buffer.SkipSpacings(TextStream.SkipType.All);
if (Buffer.Peek() != ')')
{
AddError(Buffer.TextPosition, $"Unterminated defined token, expect ')' but got '{Buffer.Peek()}'", "Preprocessor");
return(false);
}
}
}
break;
case "include":
{
char n = Buffer.Peek();
if (n == '<' || n == '"')
{
bool isComplete = false;
Buffer.AdvanceColumn();
char quote = (n == '<') ? '>' : n;
while (!Buffer.IsEOF)
{
if (Buffer.Peek() == quote)
{
isComplete = true;
Buffer.AdvanceColumn();
break;
}
Buffer.AdvanceColumn();
}
CppToken includeToken = CppTokenPool.Make(CppTokenKind.PreprocessorInclude, Buffer.LexemeRange, isComplete);
PushToken(includeToken);
}
else
{
return(false);
}
}
break;
case "pragma":
{
// @NOTE(final): Just skip until end-of-line
while (!Buffer.IsEOF)
{
char c = Buffer.Peek();
if (SyntaxUtils.IsLineBreak(c))
{
break;
}
if (c == '\t')
{
Buffer.AdvanceTab();
}
else
{
Buffer.AdvanceColumn();
}
}
}
break;
}
}
}
else
{
if (!LexNext(state))
{
break;
}
}
} while (!Buffer.IsEOF);
state.EndPreprocessor();
PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorEnd, new TextRange(Buffer.TextPosition, 0), true));
return(true);
}
public static CToken PeekTokenRaw(TextStream stream)
{
TextPosition startPos = stream.TextPosition;
if (stream.IsEOF)
{
return(new CToken(CppTokenKind.Eof, startPos, stream.TextPosition));
}
char first = stream.Peek(0);
char second = stream.Peek(1);
char third = stream.Peek(2);
CppTokenKind kind;
switch (first)
{
case '&':
{
if (second == '&')
{
kind = CppTokenKind.LogicalAndOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AndAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AndOp;
stream.AdvanceColumn();
}
}
break;
case '|':
{
if (second == '|')
{
kind = CppTokenKind.LogicalOrOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.OrAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.OrOp;
stream.AdvanceColumn();
}
}
break;
case '=':
{
if (second == '=')
{
kind = CppTokenKind.LogicalEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.EqOp;
stream.AdvanceColumn();
}
}
break;
case '!':
{
if (second == '=')
{
kind = CppTokenKind.LogicalNotEqualsOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ExclationMark;
stream.AdvanceColumn();
}
}
break;
case '<':
{
if (second == '<')
{
if (third == '=')
{
kind = CppTokenKind.LeftShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.LeftShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.LessOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.LessThanOp;
stream.AdvanceColumn();
}
}
break;
case '>':
{
if (second == '>')
{
if (third == '=')
{
kind = CppTokenKind.RightShiftAssign;
stream.AdvanceColumns(3);
}
else
{
kind = CppTokenKind.RightShiftOp;
stream.AdvanceColumns(2);
}
}
else if (second == '=')
{
kind = CppTokenKind.GreaterOrEqualOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.GreaterThanOp;
stream.AdvanceColumn();
}
}
break;
case '+':
{
if (second == '+')
{
kind = CppTokenKind.IncOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.AddAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.AddOp;
stream.AdvanceColumn();
}
}
break;
case '-':
{
if (second == '-')
{
kind = CppTokenKind.DecOp;
stream.AdvanceColumns(2);
}
else if (second == '=')
{
kind = CppTokenKind.SubAssign;
stream.AdvanceColumns(2);
}
else if (second == '>')
{
kind = CppTokenKind.PtrOp;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.SubOp;
stream.AdvanceColumn();
}
}
break;
case '/':
{
if (second == '=')
{
kind = CppTokenKind.DivAssign;
stream.AdvanceColumns(2);
}
else if (second == '/')
{
kind = CppTokenKind.SingleLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.SingleLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.SingleLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
if (SyntaxUtils.IsLineBreak(stream.Peek()))
{
break;
}
stream.AdvanceColumn();
}
}
else if (second == '*')
{
kind = CppTokenKind.MultiLineComment;
stream.AdvanceColumns(2);
char specialChar = stream.Peek();
if (DoxygenSyntax.MultiLineDocChars.Contains(specialChar))
{
kind = CppTokenKind.MultiLineCommentDoc;
stream.AdvanceColumn();
}
while (!stream.IsEOF)
{
char n0 = stream.Peek();
char n1 = stream.Peek(1);
if (n0 == '*' && n1 == '/')
{
stream.AdvanceColumns(2);
break;
}
else
{
stream.AdvanceManual(n0, n1);
}
}
}
else
{
stream.AdvanceColumn();
kind = CppTokenKind.DivOp;
}
}
break;
case '*':
{
if (second == '=')
{
kind = CppTokenKind.MulAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.MulOp;
stream.AdvanceColumn();
}
}
break;
case '%':
{
if (second == '=')
{
kind = CppTokenKind.ModAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.ModOp;
stream.AdvanceColumn();
}
}
break;
case '.':
{
if (second == '.' && third == '.')
{
kind = CppTokenKind.Ellipsis;
stream.AdvanceColumns(3);
}
else if (SyntaxUtils.IsNumeric(second))
{
stream.AdvanceColumn();
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Dot;
stream.AdvanceColumn();
}
}
break;
case '^':
{
if (second == '=')
{
kind = CppTokenKind.XorAssign;
stream.AdvanceColumns(2);
}
else
{
kind = CppTokenKind.XorOp;
stream.AdvanceColumn();
}
}
break;
case '"':
case '\'':
{
stream.AdvanceColumn();
if (first == '"')
{
kind = ReadStringLiteral(stream);
}
else
{
kind = ReadCharacterLiteral(stream);
}
}
break;
case '~':
kind = CppTokenKind.Tilde;
stream.AdvanceColumn();
break;
case '\\':
kind = CppTokenKind.Backslash;
stream.AdvanceColumn();
break;
case '#':
kind = CppTokenKind.PreprocessorStart;
stream.AdvanceColumn();
break;
case ',':
kind = CppTokenKind.Comma;
stream.AdvanceColumn();
break;
case ';':
kind = CppTokenKind.Semicolon;
stream.AdvanceColumn();
break;
case ':':
kind = CppTokenKind.Colon;
stream.AdvanceColumn();
break;
case '?':
kind = CppTokenKind.QuestionMark;
stream.AdvanceColumn();
break;
case '{':
kind = CppTokenKind.LeftBrace;
stream.AdvanceColumn();
break;
case '}':
kind = CppTokenKind.RightBrace;
stream.AdvanceColumn();
break;
case '[':
kind = CppTokenKind.LeftBracket;
stream.AdvanceColumn();
break;
case ']':
kind = CppTokenKind.RightBracket;
stream.AdvanceColumn();
break;
case '(':
kind = CppTokenKind.LeftParen;
stream.AdvanceColumn();
break;
case ')':
kind = CppTokenKind.RightParen;
stream.AdvanceColumn();
break;
default:
{
if (SyntaxUtils.IsLineBreak(first))
{
kind = CppTokenKind.EndOfLine;
int nb = SyntaxUtils.GetLineBreakChars(first, second);
stream.AdvanceLine(nb);
}
else if (first == '\t')
{
kind = CppTokenKind.Spacings;
while (!stream.IsEOF)
{
if (stream.Peek() != '\t')
{
break;
}
stream.AdvanceTab();
}
}
else if (SyntaxUtils.IsSpacing(first))
{
kind = CppTokenKind.Spacings;
stream.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
}
else if (SyntaxUtils.IsIdentStart(first))
{
kind = CppTokenKind.IdentLiteral;
stream.AdvanceColumnsWhile(SyntaxUtils.IsIdentPart);
}
else if (SyntaxUtils.IsNumeric(first))
{
kind = ReadNumberLiteral(stream);
}
else
{
kind = CppTokenKind.Unknown;
}
}
break;
}
CToken result = MakeToken(kind, stream, startPos, stream.TextPosition);
return(result);
}
private void LexTag()
{
Debug.Assert(Buffer.Peek() == '<');
Buffer.StartLexeme();
HtmlToken startTagToken = HtmlTokenPool.Make(HtmlTokenKind.MetaTagStart, Buffer.LexemeRange, false);
PushToken(startTagToken);
bool allowAttributes = true;
Buffer.AdvanceColumn();
if (Buffer.Peek() == '/')
{
startTagToken.ChangeKind(HtmlTokenKind.MetaTagClose);
Buffer.AdvanceColumn();
allowAttributes = false;
}
PushToken(HtmlTokenPool.Make(HtmlTokenKind.TagChars, Buffer.LexemeRange, true));
if (SyntaxUtils.IsIdentStart(Buffer.Peek()))
{
Buffer.StartLexeme();
while (!Buffer.IsEOF)
{
if (SyntaxUtils.IsIdentPart(Buffer.Peek()))
{
Buffer.AdvanceColumn();
}
else
{
break;
}
}
PushToken(HtmlTokenPool.Make(HtmlTokenKind.TagName, Buffer.LexemeRange, true));
}
if (allowAttributes)
{
while (!Buffer.IsEOF)
{
Buffer.SkipAllWhitespaces();
char c = Buffer.Peek();
if (!SyntaxUtils.IsIdentStart(c))
{
break;
}
else
{
Buffer.StartLexeme();
while (!Buffer.IsEOF)
{
if (SyntaxUtils.IsIdentPart(Buffer.Peek()))
{
Buffer.AdvanceColumn();
}
else
{
break;
}
}
PushToken(HtmlTokenPool.Make(HtmlTokenKind.AttrName, Buffer.LexemeRange, true));
Buffer.SkipAllWhitespaces(); // Allow whitespaces before =
if (Buffer.Peek() == '=')
{
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(HtmlTokenPool.Make(HtmlTokenKind.AttrChars, Buffer.LexemeRange, true));
Buffer.SkipAllWhitespaces(); // Allow whitespaces after =
if (Buffer.Peek() == '"' || Buffer.Peek() == '\'')
{
char quote = Buffer.Peek();
Buffer.StartLexeme();
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
char attrC = Buffer.Peek();
if (attrC != quote && attrC != '\n')
{
Buffer.AdvanceColumn();
}
else
{
break;
}
}
if (Buffer.Peek() == quote)
{
Buffer.AdvanceColumn();
}
PushToken(HtmlTokenPool.Make(HtmlTokenKind.AttrValue, Buffer.LexemeRange, true));
}
}
else
{
break;
}
}
}
}
Buffer.SkipAllWhitespaces(); // Allow whitespaces before /
if (Buffer.Peek() == '/')
{
startTagToken.ChangeKind(HtmlTokenKind.MetaTagStartAndClose);
Buffer.AdvanceColumn();
Buffer.SkipAllWhitespaces(); // Allow whitespaces after /
}
Buffer.SkipUntil('>');
if (Buffer.Peek() == '>')
{
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(HtmlTokenPool.Make(HtmlTokenKind.TagChars, Buffer.LexemeRange, true));
}
int tagLength = Buffer.StreamPosition - startTagToken.Index;
startTagToken.ChangeLength(tagLength);
}
