private TokenizeResult TokenizeCpp(string text, TextPosition pos, int length, bool allowDoxy) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = new Stopwatch(); timer.Restart(); List <CppToken> cppTokens = new List <CppToken>(); using (CppLexer cppLexer = new CppLexer(text, pos, length)) { cppTokens.AddRange(cppLexer.Tokenize()); result.AddErrors(cppLexer.LexErrors); } timer.Stop(); result.Stats.CppDuration += timer.Elapsed; foreach (CppToken token in cppTokens) { if (allowDoxy && (token.Kind == CppTokenKind.MultiLineCommentDoc || token.Kind == CppTokenKind.SingleLineCommentDoc)) { result.AddToken(token); using (TokenizeResult doxyRes = TokenizeDoxy(text, token.Position, token.Length)) { result.Stats.DoxyDuration += doxyRes.Stats.DoxyDuration; result.Stats.HtmlDuration += doxyRes.Stats.HtmlDuration; result.AddTokens(doxyRes.Tokens); result.AddErrors(doxyRes.Errors); } } else { result.AddToken(token); } } return(result); }
private TokenizeResult TokenizeHtml(string text, TextPosition pos, int length) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = Stopwatch.StartNew(); using (HtmlLexer htmlLexer = new HtmlLexer(text, pos, length)) { IEnumerable <HtmlToken> htmlTokens = htmlLexer.Tokenize(); if (htmlTokens.FirstOrDefault(d => !d.IsEOF) != null) { result.AddTokens(htmlTokens); } result.AddErrors(htmlLexer.LexErrors); } timer.Stop(); result.Stats.HtmlDuration += timer.Elapsed; return(result); }
private TokenizeResult TokenizeDoxy(string text, TextPosition pos, int length) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = new Stopwatch(); timer.Restart(); List <DoxygenToken> doxyTokens = new List <DoxygenToken>(); using (DoxygenLexer doxyLexer = new DoxygenLexer(text, pos, length)) { doxyTokens.AddRange(doxyLexer.Tokenize()); result.AddErrors(doxyLexer.LexErrors); } timer.Stop(); result.Stats.DoxyDuration += timer.Elapsed; Stack <CommandStartState> startStates = new Stack <CommandStartState>(); Stack <DoxygenToken> textStartTokens = new Stack <DoxygenToken>(); var doxyTokenList = new LinkedList <DoxygenToken>(doxyTokens); var curLink = doxyTokenList.First; while (curLink != null) { var doxyToken = curLink.Value; if (doxyToken.Kind == DoxygenTokenKind.CommandStart) { result.AddToken(doxyToken); string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1); List <DoxygenToken> argTokens = new List <DoxygenToken>(); if (curLink.Next != null) { LinkedListNode <DoxygenToken> nextLink = curLink.Next; while (nextLink != null) { if (nextLink.Value.IsArgument) { argTokens.Add(nextLink.Value); } else { break; } nextLink = nextLink.Next; } curLink = nextLink; } else { curLink = null; } result.AddTokens(argTokens); CommandStartState startState = new CommandStartState(doxyToken, commandName); startState.ArgTokens.AddRange(argTokens); startStates.Push(startState); if (argTokens.Count > 0) { var last = argTokens.Last(); startState.StartPosition = new TextPosition(last.End, last.Position.Line, last.Position.Column); } else { startState.StartPosition = new TextPosition(doxyToken.End, doxyToken.Position.Line, doxyToken.Position.Column); } continue; } if (doxyToken.Kind == DoxygenTokenKind.CommandEnd) { string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1); CommandStartState topStartState = startStates.Count > 0 ? startStates.Peek() : null; if (topStartState != null) { var rule = DoxygenSyntax.GetCommandRule(commandName); Debug.Assert(rule != null && rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock); DoxygenSyntax.EndBlockCommandRule endRule = rule as DoxygenSyntax.EndBlockCommandRule; Debug.Assert(endRule != null); if (endRule.StartCommandNames.Contains(topStartState.CommandName)) { TextPosition commandContentStart = topStartState.StartPosition; TextPosition commandContentEnd = doxyToken.Position; Debug.Assert(commandContentEnd.Index >= commandContentStart.Index); int commandContentLength = commandContentEnd.Index - commandContentStart.Index; // Special handling for code block if ("code".Equals(topStartState.CommandName)) { string codeType = null; DoxygenToken firstArgToken = topStartState.ArgTokens.FirstOrDefault(); if (firstArgToken != null && firstArgToken.Kind == DoxygenTokenKind.ArgumentCaption) { codeType = text.Substring(firstArgToken.Index, firstArgToken.Length); } if ("{.c}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase) || "{.cpp}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase)) { using (TokenizeResult cppRes = TokenizeCpp(text, commandContentStart, commandContentLength, false)) { result.AddTokens(cppRes.Tokens); result.AddErrors(cppRes.Errors); } } } startStates.Pop(); } } else { // @TODO(final): Print error (Command end without command start) } result.AddToken(doxyToken); } else if (doxyToken.Kind == DoxygenTokenKind.TextStart) { textStartTokens.Push(doxyToken); result.AddToken(doxyToken); } else if (doxyToken.Kind == DoxygenTokenKind.TextEnd) { if (textStartTokens.Count > 0) { DoxygenToken textStartToken = textStartTokens.Pop(); Debug.Assert(doxyToken.Index >= textStartToken.Index); int textContentLen = doxyToken.Index - textStartToken.Index; using (TokenizeResult htmlRes = TokenizeHtml(text, textStartToken.Position, textContentLen)) { result.AddTokens(htmlRes.Tokens); result.AddErrors(htmlRes.Errors); result.Stats.HtmlDuration += htmlRes.Stats.HtmlDuration; } } result.AddToken(doxyToken); } else { result.AddToken(doxyToken); } curLink = curLink.Next; } return(result); }