private void Lex(string source, params ExpectToken[] expectedTokens) { using (DoxygenLexer lexer = new DoxygenLexer(source, new TextPosition(0), source.Length)) { var tokens = lexer.Tokenize(); if (expectedTokens.Length > 0) { Assert.AreEqual(expectedTokens.Length, tokens.Count()); int index = 0; foreach (var token in tokens) { var et = expectedTokens[index]; Assert.AreEqual(et.Kind, token.Kind); Assert.IsTrue(token.Length >= et.MinLength, $"Expect min token length of {et.MinLength} but got {token.Length}"); Assert.IsTrue(token.Length <= et.MaxLength, $"Expect max token length of {et.MaxLength} but got {token.Length}"); if (et.Value != null) { Assert.AreEqual(et.Value, token.Value); } ++index; } } } }
private TokenizeResult TokenizeDoxy(string text, TextPosition pos, int length) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = new Stopwatch(); timer.Restart(); List <DoxygenToken> doxyTokens = new List <DoxygenToken>(); using (DoxygenLexer doxyLexer = new DoxygenLexer(text, pos, length)) { doxyTokens.AddRange(doxyLexer.Tokenize()); result.AddErrors(doxyLexer.LexErrors); } timer.Stop(); result.Stats.DoxyDuration += timer.Elapsed; Stack <CommandStartState> startStates = new Stack <CommandStartState>(); Stack <DoxygenToken> textStartTokens = new Stack <DoxygenToken>(); var doxyTokenList = new LinkedList <DoxygenToken>(doxyTokens); var curLink = doxyTokenList.First; while (curLink != null) { var doxyToken = curLink.Value; if (doxyToken.Kind == DoxygenTokenKind.CommandStart) { result.AddToken(doxyToken); string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1); List <DoxygenToken> argTokens = new List <DoxygenToken>(); if (curLink.Next != null) { LinkedListNode <DoxygenToken> nextLink = curLink.Next; while (nextLink != null) { if (nextLink.Value.IsArgument) { argTokens.Add(nextLink.Value); } else { break; } nextLink = nextLink.Next; } curLink = nextLink; } else { curLink = null; } result.AddTokens(argTokens); CommandStartState startState = new CommandStartState(doxyToken, commandName); startState.ArgTokens.AddRange(argTokens); startStates.Push(startState); if (argTokens.Count > 0) { var last = argTokens.Last(); startState.StartPosition = new TextPosition(last.End, last.Position.Line, last.Position.Column); } else { startState.StartPosition = new TextPosition(doxyToken.End, doxyToken.Position.Line, doxyToken.Position.Column); } continue; } if (doxyToken.Kind == DoxygenTokenKind.CommandEnd) { string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1); CommandStartState topStartState = startStates.Count > 0 ? startStates.Peek() : null; if (topStartState != null) { var rule = DoxygenSyntax.GetCommandRule(commandName); Debug.Assert(rule != null && rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock); DoxygenSyntax.EndBlockCommandRule endRule = rule as DoxygenSyntax.EndBlockCommandRule; Debug.Assert(endRule != null); if (endRule.StartCommandNames.Contains(topStartState.CommandName)) { TextPosition commandContentStart = topStartState.StartPosition; TextPosition commandContentEnd = doxyToken.Position; Debug.Assert(commandContentEnd.Index >= commandContentStart.Index); int commandContentLength = commandContentEnd.Index - commandContentStart.Index; // Special handling for code block if ("code".Equals(topStartState.CommandName)) { string codeType = null; DoxygenToken firstArgToken = topStartState.ArgTokens.FirstOrDefault(); if (firstArgToken != null && firstArgToken.Kind == DoxygenTokenKind.ArgumentCaption) { codeType = text.Substring(firstArgToken.Index, firstArgToken.Length); } if ("{.c}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase) || "{.cpp}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase)) { using (TokenizeResult cppRes = TokenizeCpp(text, commandContentStart, commandContentLength, false)) { result.AddTokens(cppRes.Tokens); result.AddErrors(cppRes.Errors); } } } startStates.Pop(); } } else { // @TODO(final): Print error (Command end without command start) } result.AddToken(doxyToken); } else if (doxyToken.Kind == DoxygenTokenKind.TextStart) { textStartTokens.Push(doxyToken); result.AddToken(doxyToken); } else if (doxyToken.Kind == DoxygenTokenKind.TextEnd) { if (textStartTokens.Count > 0) { DoxygenToken textStartToken = textStartTokens.Pop(); Debug.Assert(doxyToken.Index >= textStartToken.Index); int textContentLen = doxyToken.Index - textStartToken.Index; using (TokenizeResult htmlRes = TokenizeHtml(text, textStartToken.Position, textContentLen)) { result.AddTokens(htmlRes.Tokens); result.AddErrors(htmlRes.Errors); result.Stats.HtmlDuration += htmlRes.Stats.HtmlDuration; } } result.AddToken(doxyToken); } else { result.AddToken(doxyToken); } curLink = curLink.Next; } return(result); }