private void Done(DoxygenState state)
{
EndText(state);
if (state.Flags.HasFlag(StateFlags.InsideBlock))
{
// Block was not closed, so we close it now
state.Flags = StateFlags.None;
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockEnd, new TextRange(Buffer.TextPosition, 0), false));
}
}
private void EndText(DoxygenState state)
{
var lastTextStartOrEnd = Tokens.LastOrDefault(t => t.Kind == DoxygenTokenKind.TextStart || t.Kind == DoxygenTokenKind.TextEnd);
if (lastTextStartOrEnd != null && lastTextStartOrEnd.Kind == DoxygenTokenKind.TextStart)
{
int textLength = Buffer.TextPosition.Index - lastTextStartOrEnd.Index;
if (textLength > 0)
{
DoxygenToken token = DoxygenTokenPool.Make(DoxygenTokenKind.TextEnd, new TextRange(Buffer.TextPosition, 0), false);
PushToken(token);
}
else
{
RemoveToken(lastTextStartOrEnd);
}
}
}
protected override bool LexNext(State hiddenState)
{
DoxygenState state = (DoxygenState)hiddenState;
state.Flags = StateFlags.None;
state.CurrentLineStartIndex = Buffer.StreamPosition;
do
{
char first = Buffer.Peek();
char second = Buffer.Peek(1);
char third = Buffer.Peek(2);
switch (first)
{
case ' ':
case '\v':
case '\f':
case '\t':
Buffer.SkipSpacings(TextStream.SkipType.All);
break;
case '\r':
case '\n':
{
if (state.Flags.HasFlag(StateFlags.InsideBlock) && state.Flags.HasFlag(StateFlags.SingleLine))
{
Done(state);
return(true);
}
// @NOTE(final): Detect if our line content until the line break was empty
Debug.Assert(Buffer.StreamPosition >= state.CurrentLineStartIndex);
int len = Buffer.StreamPosition - state.CurrentLineStartIndex;
bool wasEmptyLine = Buffer.MatchCharacters(state.CurrentLineStartIndex, len, char.IsWhiteSpace) || (len == 0);
Buffer.StartLexeme();
Buffer.SkipLineBreaks(TextStream.SkipType.Single);
state.CurrentLineStartIndex = Buffer.StreamPosition;
PushToken(DoxygenTokenPool.Make(wasEmptyLine ? DoxygenTokenKind.EmptyLine : DoxygenTokenKind.EndOfLine, Buffer.LexemeRange, true));
}
break;
case '/':
{
if (second == '*')
{
// Multi line
if (DoxygenSyntax.MultiLineDocChars.Contains(third))
{
Debug.Assert(!state.Flags.HasFlag(StateFlags.InsideBlock));
Buffer.StartLexeme();
Buffer.AdvanceColumns(3);
state.Flags = StateFlags.InsideBlock;
if (third == '*')
{
char n3 = Buffer.Peek();
if (n3 == '/')
{
Buffer.AdvanceColumn();
return(PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartMulti, Buffer.LexemeRange, true)));
}
state.Flags |= StateFlags.JavaDoc;
}
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartMulti, Buffer.LexemeRange, true));
StartText(state);
continue;
}
else
{
// Just skip until normal multi-line comment ends
var r = CppLexer.LexMultiLineComment(Buffer, true);
if (!r.IsComplete)
{
AddError(Buffer.TextPosition, $"Unterminated multi-line comment, expect '*/' but got EOF", r.Kind.ToString());
return(false);
}
continue;
}
}
else if (second == '/')
{
// Single line
char n2 = Buffer.Peek(2);
if (DoxygenSyntax.SingleLineDocChars.Contains(n2))
{
Debug.Assert(!state.Flags.HasFlag(StateFlags.InsideBlock));
Buffer.StartLexeme();
Buffer.AdvanceColumns(3);
state.Flags = StateFlags.InsideBlock | StateFlags.SingleLine;
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockStartSingle, Buffer.LexemeRange, true));
StartText(state);
continue;
}
else
{
// Just skip until normal single-line comment ends
var r = CppLexer.LexSingleLineComment(Buffer, true);
if (!r.IsComplete)
{
AddError(Buffer.TextPosition, $"Unterminated single-line comment, expect linebreak but got EOF", r.Kind.ToString());
return(false);
}
continue;
}
}
else
{
Buffer.AdvanceColumn();
}
}
break;
case '*':
{
if (state.Flags.HasFlag(StateFlags.InsideBlock))
{
if (second == '/')
{
EndText(state);
Buffer.StartLexeme();
Buffer.AdvanceColumns(2);
state.Flags = StateFlags.None;
return(PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockEnd, Buffer.LexemeRange, true)));
}
else if (state.Flags.HasFlag(StateFlags.JavaDoc))
{
// Push single star token (java doc style)
EndText(state);
Buffer.StartLexeme();
Buffer.AdvanceColumn();
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.DoxyBlockChars, Buffer.LexemeRange, true));
StartText(state);
state.CurrentLineStartIndex = Buffer.StreamPosition;
continue;
}
}
Buffer.AdvanceColumn();
}
break;
case '@':
case '\\':
{
if (state.Flags.HasFlag(StateFlags.InsideBlock))
{
EndText(state);
var commandResult = LexCommandTokens();
if (commandResult.IsValid)
{
if ("code".Equals(commandResult.CommandName))
{
if (!LexUntilCodeEnd(commandResult))
{
return(false);
}
}
}
StartText(state);
}
else
{
Buffer.AdvanceColumn();
}
}
break;
case TextStream.InvalidCharacter:
{
if (Buffer.IsEOF)
{
Done(state);
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.EOF, new TextRange(Buffer.TextPosition, 0), false));
return(false);
}
else
{
Buffer.AdvanceColumn();
}
}
break;
default:
{
Buffer.AdvanceColumn();
break;
}
}
} while (!Buffer.IsEOF);
Done(state);
PushToken(DoxygenTokenPool.Make(DoxygenTokenKind.EOF, new TextRange(Buffer.TextPosition, 0), false));
return(false);
}
private void StartText(DoxygenState state)
{
DoxygenToken token = DoxygenTokenPool.Make(DoxygenTokenKind.TextStart, new TextRange(Buffer.TextPosition, 0), false);
PushToken(token);
}
