private void Lex(string source, params ExpectToken[] expectedTokens)
{
using (DoxygenLexer lexer = new DoxygenLexer(source, new TextPosition(0), source.Length))
{
var tokens = lexer.Tokenize();
if (expectedTokens.Length > 0)
{
Assert.AreEqual(expectedTokens.Length, tokens.Count());
int index = 0;
foreach (var token in tokens)
{
var et = expectedTokens[index];
Assert.AreEqual(et.Kind, token.Kind);
Assert.IsTrue(token.Length >= et.MinLength, $"Expect min token length of {et.MinLength} but got {token.Length}");
Assert.IsTrue(token.Length <= et.MaxLength, $"Expect max token length of {et.MaxLength} but got {token.Length}");
if (et.Value != null)
{
Assert.AreEqual(et.Value, token.Value);
}
++index;
}
}
}
}
private TokenizeResult TokenizeDoxy(string text, TextPosition pos, int length)
{
TokenizeResult result = new TokenizeResult();
Stopwatch timer = new Stopwatch();
timer.Restart();
List <DoxygenToken> doxyTokens = new List <DoxygenToken>();
using (DoxygenLexer doxyLexer = new DoxygenLexer(text, pos, length))
{
doxyTokens.AddRange(doxyLexer.Tokenize());
result.AddErrors(doxyLexer.LexErrors);
}
timer.Stop();
result.Stats.DoxyDuration += timer.Elapsed;
Stack <CommandStartState> startStates = new Stack <CommandStartState>();
Stack <DoxygenToken> textStartTokens = new Stack <DoxygenToken>();
var doxyTokenList = new LinkedList <DoxygenToken>(doxyTokens);
var curLink = doxyTokenList.First;
while (curLink != null)
{
var doxyToken = curLink.Value;
if (doxyToken.Kind == DoxygenTokenKind.CommandStart)
{
result.AddToken(doxyToken);
string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1);
List <DoxygenToken> argTokens = new List <DoxygenToken>();
if (curLink.Next != null)
{
LinkedListNode <DoxygenToken> nextLink = curLink.Next;
while (nextLink != null)
{
if (nextLink.Value.IsArgument)
{
argTokens.Add(nextLink.Value);
}
else
{
break;
}
nextLink = nextLink.Next;
}
curLink = nextLink;
}
else
{
curLink = null;
}
result.AddTokens(argTokens);
CommandStartState startState = new CommandStartState(doxyToken, commandName);
startState.ArgTokens.AddRange(argTokens);
startStates.Push(startState);
if (argTokens.Count > 0)
{
var last = argTokens.Last();
startState.StartPosition = new TextPosition(last.End, last.Position.Line, last.Position.Column);
}
else
{
startState.StartPosition = new TextPosition(doxyToken.End, doxyToken.Position.Line, doxyToken.Position.Column);
}
continue;
}
if (doxyToken.Kind == DoxygenTokenKind.CommandEnd)
{
string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1);
CommandStartState topStartState = startStates.Count > 0 ? startStates.Peek() : null;
if (topStartState != null)
{
var rule = DoxygenSyntax.GetCommandRule(commandName);
Debug.Assert(rule != null && rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock);
DoxygenSyntax.EndBlockCommandRule endRule = rule as DoxygenSyntax.EndBlockCommandRule;
Debug.Assert(endRule != null);
if (endRule.StartCommandNames.Contains(topStartState.CommandName))
{
TextPosition commandContentStart = topStartState.StartPosition;
TextPosition commandContentEnd = doxyToken.Position;
Debug.Assert(commandContentEnd.Index >= commandContentStart.Index);
int commandContentLength = commandContentEnd.Index - commandContentStart.Index;
// Special handling for code block
if ("code".Equals(topStartState.CommandName))
{
string codeType = null;
DoxygenToken firstArgToken = topStartState.ArgTokens.FirstOrDefault();
if (firstArgToken != null && firstArgToken.Kind == DoxygenTokenKind.ArgumentCaption)
{
codeType = text.Substring(firstArgToken.Index, firstArgToken.Length);
}
if ("{.c}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase) || "{.cpp}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase))
{
using (TokenizeResult cppRes = TokenizeCpp(text, commandContentStart, commandContentLength, false))
{
result.AddTokens(cppRes.Tokens);
result.AddErrors(cppRes.Errors);
}
}
}
startStates.Pop();
}
}
else
{
// @TODO(final): Print error (Command end without command start)
}
result.AddToken(doxyToken);
}
else if (doxyToken.Kind == DoxygenTokenKind.TextStart)
{
textStartTokens.Push(doxyToken);
result.AddToken(doxyToken);
}
else if (doxyToken.Kind == DoxygenTokenKind.TextEnd)
{
if (textStartTokens.Count > 0)
{
DoxygenToken textStartToken = textStartTokens.Pop();
Debug.Assert(doxyToken.Index >= textStartToken.Index);
int textContentLen = doxyToken.Index - textStartToken.Index;
using (TokenizeResult htmlRes = TokenizeHtml(text, textStartToken.Position, textContentLen))
{
result.AddTokens(htmlRes.Tokens);
result.AddErrors(htmlRes.Errors);
result.Stats.HtmlDuration += htmlRes.Stats.HtmlDuration;
}
}
result.AddToken(doxyToken);
}
else
{
result.AddToken(doxyToken);
}
curLink = curLink.Next;
}
return(result);
}
