private TokenizeResult TokenizeCpp(string text, TextPosition pos, int length, bool allowDoxy) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = new Stopwatch(); timer.Restart(); List <CppToken> cppTokens = new List <CppToken>(); using (CppLexer cppLexer = new CppLexer(text, pos, length)) { cppTokens.AddRange(cppLexer.Tokenize()); result.AddErrors(cppLexer.LexErrors); } timer.Stop(); result.Stats.CppDuration += timer.Elapsed; foreach (CppToken token in cppTokens) { if (allowDoxy && (token.Kind == CppTokenKind.MultiLineCommentDoc || token.Kind == CppTokenKind.SingleLineCommentDoc)) { result.AddToken(token); using (TokenizeResult doxyRes = TokenizeDoxy(text, token.Position, token.Length)) { result.Stats.DoxyDuration += doxyRes.Stats.DoxyDuration; result.Stats.HtmlDuration += doxyRes.Stats.HtmlDuration; result.AddTokens(doxyRes.Tokens); result.AddErrors(doxyRes.Errors); } } else { result.AddToken(token); } } return(result); }
private TokenizeResult TokenizeHtml(string text, TextPosition pos, int length) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = Stopwatch.StartNew(); using (HtmlLexer htmlLexer = new HtmlLexer(text, pos, length)) { IEnumerable <HtmlToken> htmlTokens = htmlLexer.Tokenize(); if (htmlTokens.FirstOrDefault(d => !d.IsEOF) != null) { result.AddTokens(htmlTokens); } result.AddErrors(htmlLexer.LexErrors); } timer.Stop(); result.Stats.HtmlDuration += timer.Elapsed; return(result); }
static void Main(string[] args) { string input = String.Empty; string output; while (input != "exit") { input = Console.ReadLine(); Tokenizer tokenizer = new Tokenizer(); TokenizeResult tokenizeResult = tokenizer.TokenizeString(input); if (!tokenizeResult.Success) { output = tokenizeResult.AddtionalInfo; } else { ExpressionBuilder expressionBuilder = new ExpressionBuilder(tokenizeResult.TokenList); ExpressionizeResult expressionizeResult = expressionBuilder.GenerateExpression(); if (!expressionizeResult.Success) { output = expressionizeResult.Error.Message; } else { EvaluationResult evaluationResult = expressionizeResult.CalculatedExpression.Evaluate(null); if (!evaluationResult.Success) { output = evaluationResult.AdditionalInfo.Aggregate((item, aggregator) => $"{aggregator}\n{item}"); } else { output = evaluationResult.Value.ToString(); } } } Console.WriteLine(output); } }
private void Tokenize(string text) { Stopwatch timer = new Stopwatch(); // Push back all tokens to to pools GiveTokensBackToPool(); // Clear tokens & errors _tokens.Clear(); _errors.Clear(); _performanceItems.Clear(); SymbolCache.Clear(this); Stopwatch totalLexTimer = Stopwatch.StartNew(); // C++ lexing -> Doxygen (Code -> Cpp) -> (Text -> Html) TokenizerTimingStats totalStats = new TokenizerTimingStats(); using (TokenizeResult cppRes = TokenizeCpp(text, new TextPosition(0), text.Length, true)) { totalStats += cppRes.Stats; _tokens.AddRange(cppRes.Tokens); _errors.AddRange(cppRes.Errors); } totalLexTimer.Stop(); Debug.WriteLine($"Lexing done (Tokens: {_tokens.Count}, Total: {totalLexTimer.Elapsed.ToMilliseconds()} ms, Insert: {totalStats.InsertDuration.ToMilliseconds()}, C++: {totalStats.CppDuration.ToMilliseconds()} ms, Doxygen: {totalStats.DoxyDuration.ToMilliseconds()} ms, Html: {totalStats.HtmlDuration.ToMilliseconds()} ms)"); int countCppTokens = _tokens.Count(t => typeof(CppToken).Equals(t.GetType())); int countHtmlTokens = _tokens.Count(t => typeof(HtmlToken).Equals(t.GetType())); int countDoxyTokens = _tokens.Count(t => typeof(DoxygenToken).Equals(t.GetType())); _performanceItems.Add(new PerformanceItemModel(this, TabIndex, $"{text.Length} chars", $"{countCppTokens} tokens", "C++ lexer", totalStats.CppDuration)); _performanceItems.Add(new PerformanceItemModel(this, TabIndex, $"{text.Length} chars", $"{countDoxyTokens} tokens", "Doxygen lexer", totalStats.DoxyDuration)); _performanceItems.Add(new PerformanceItemModel(this, TabIndex, $"{text.Length} chars", $"{countHtmlTokens} tokens", "Html lexer", totalStats.HtmlDuration)); timer.Restart(); _styler.Refresh(_tokens); timer.Stop(); Debug.WriteLine($"Styler done, took {timer.Elapsed.ToMilliseconds()} ms"); }
private TokenizeResult TokenizeDoxy(string text, TextPosition pos, int length) { TokenizeResult result = new TokenizeResult(); Stopwatch timer = new Stopwatch(); timer.Restart(); List <DoxygenToken> doxyTokens = new List <DoxygenToken>(); using (DoxygenLexer doxyLexer = new DoxygenLexer(text, pos, length)) { doxyTokens.AddRange(doxyLexer.Tokenize()); result.AddErrors(doxyLexer.LexErrors); } timer.Stop(); result.Stats.DoxyDuration += timer.Elapsed; Stack <CommandStartState> startStates = new Stack <CommandStartState>(); Stack <DoxygenToken> textStartTokens = new Stack <DoxygenToken>(); var doxyTokenList = new LinkedList <DoxygenToken>(doxyTokens); var curLink = doxyTokenList.First; while (curLink != null) { var doxyToken = curLink.Value; if (doxyToken.Kind == DoxygenTokenKind.CommandStart) { result.AddToken(doxyToken); string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1); List <DoxygenToken> argTokens = new List <DoxygenToken>(); if (curLink.Next != null) { LinkedListNode <DoxygenToken> nextLink = curLink.Next; while (nextLink != null) { if (nextLink.Value.IsArgument) { argTokens.Add(nextLink.Value); } else { break; } nextLink = nextLink.Next; } curLink = nextLink; } else { curLink = null; } result.AddTokens(argTokens); CommandStartState startState = new CommandStartState(doxyToken, commandName); startState.ArgTokens.AddRange(argTokens); startStates.Push(startState); if (argTokens.Count > 0) { var last = argTokens.Last(); startState.StartPosition = new TextPosition(last.End, last.Position.Line, last.Position.Column); } else { startState.StartPosition = new TextPosition(doxyToken.End, doxyToken.Position.Line, doxyToken.Position.Column); } continue; } if (doxyToken.Kind == DoxygenTokenKind.CommandEnd) { string commandName = text.Substring(doxyToken.Index + 1, doxyToken.Length - 1); CommandStartState topStartState = startStates.Count > 0 ? startStates.Peek() : null; if (topStartState != null) { var rule = DoxygenSyntax.GetCommandRule(commandName); Debug.Assert(rule != null && rule.Kind == DoxygenSyntax.CommandKind.EndCommandBlock); DoxygenSyntax.EndBlockCommandRule endRule = rule as DoxygenSyntax.EndBlockCommandRule; Debug.Assert(endRule != null); if (endRule.StartCommandNames.Contains(topStartState.CommandName)) { TextPosition commandContentStart = topStartState.StartPosition; TextPosition commandContentEnd = doxyToken.Position; Debug.Assert(commandContentEnd.Index >= commandContentStart.Index); int commandContentLength = commandContentEnd.Index - commandContentStart.Index; // Special handling for code block if ("code".Equals(topStartState.CommandName)) { string codeType = null; DoxygenToken firstArgToken = topStartState.ArgTokens.FirstOrDefault(); if (firstArgToken != null && firstArgToken.Kind == DoxygenTokenKind.ArgumentCaption) { codeType = text.Substring(firstArgToken.Index, firstArgToken.Length); } if ("{.c}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase) || "{.cpp}".Equals(codeType, StringComparison.InvariantCultureIgnoreCase)) { using (TokenizeResult cppRes = TokenizeCpp(text, commandContentStart, commandContentLength, false)) { result.AddTokens(cppRes.Tokens); result.AddErrors(cppRes.Errors); } } } startStates.Pop(); } } else { // @TODO(final): Print error (Command end without command start) } result.AddToken(doxyToken); } else if (doxyToken.Kind == DoxygenTokenKind.TextStart) { textStartTokens.Push(doxyToken); result.AddToken(doxyToken); } else if (doxyToken.Kind == DoxygenTokenKind.TextEnd) { if (textStartTokens.Count > 0) { DoxygenToken textStartToken = textStartTokens.Pop(); Debug.Assert(doxyToken.Index >= textStartToken.Index); int textContentLen = doxyToken.Index - textStartToken.Index; using (TokenizeResult htmlRes = TokenizeHtml(text, textStartToken.Position, textContentLen)) { result.AddTokens(htmlRes.Tokens); result.AddErrors(htmlRes.Errors); result.Stats.HtmlDuration += htmlRes.Stats.HtmlDuration; } } result.AddToken(doxyToken); } else { result.AddToken(doxyToken); } curLink = curLink.Next; } return(result); }
public override bool CanFixup(TokenizeResult <WhoisResponse> result) { // Templates that this Fixup can work on return(result.Template.Name == "whois.isoc.org.il/il/Found"); }