public static TokenizerTimingStats operator +(TokenizerTimingStats a, TokenizerTimingStats b)
{
TokenizerTimingStats result = new TokenizerTimingStats();
result.CppDuration = a.CppDuration + b.CppDuration;
result.DoxyDuration = a.DoxyDuration + b.DoxyDuration;
result.HtmlDuration = a.HtmlDuration + b.HtmlDuration;
result.InsertDuration = a.InsertDuration + b.InsertDuration;
return(result);
}
private void Tokenize(string text)
{
Stopwatch timer = new Stopwatch();
// Push back all tokens to to pools
GiveTokensBackToPool();
// Clear tokens & errors
_tokens.Clear();
_errors.Clear();
_performanceItems.Clear();
SymbolCache.Clear(this);
Stopwatch totalLexTimer = Stopwatch.StartNew();
// C++ lexing -> Doxygen (Code -> Cpp) -> (Text -> Html)
TokenizerTimingStats totalStats = new TokenizerTimingStats();
using (TokenizeResult cppRes = TokenizeCpp(text, new TextPosition(0), text.Length, true))
{
totalStats += cppRes.Stats;
_tokens.AddRange(cppRes.Tokens);
_errors.AddRange(cppRes.Errors);
}
totalLexTimer.Stop();
Debug.WriteLine($"Lexing done (Tokens: {_tokens.Count}, Total: {totalLexTimer.Elapsed.ToMilliseconds()} ms, Insert: {totalStats.InsertDuration.ToMilliseconds()}, C++: {totalStats.CppDuration.ToMilliseconds()} ms, Doxygen: {totalStats.DoxyDuration.ToMilliseconds()} ms, Html: {totalStats.HtmlDuration.ToMilliseconds()} ms)");
int countCppTokens = _tokens.Count(t => typeof(CppToken).Equals(t.GetType()));
int countHtmlTokens = _tokens.Count(t => typeof(HtmlToken).Equals(t.GetType()));
int countDoxyTokens = _tokens.Count(t => typeof(DoxygenToken).Equals(t.GetType()));
_performanceItems.Add(new PerformanceItemModel(this, TabIndex, $"{text.Length} chars", $"{countCppTokens} tokens", "C++ lexer", totalStats.CppDuration));
_performanceItems.Add(new PerformanceItemModel(this, TabIndex, $"{text.Length} chars", $"{countDoxyTokens} tokens", "Doxygen lexer", totalStats.DoxyDuration));
_performanceItems.Add(new PerformanceItemModel(this, TabIndex, $"{text.Length} chars", $"{countHtmlTokens} tokens", "Html lexer", totalStats.HtmlDuration));
timer.Restart();
_styler.Refresh(_tokens);
timer.Stop();
Debug.WriteLine($"Styler done, took {timer.Elapsed.ToMilliseconds()} ms");
}
public TokenizeResult()
{
Stats = new TokenizerTimingStats();
}
