public static string GetTokensString(this IList <IToken> tokens, IVocabulary vocabulary, TokenValueDisplayMode tokenValueDisplayMode = TokenValueDisplayMode.Show, bool onlyDefaultChannel = false, bool eachTokenOnNewLine = true) { var resultString = new StringBuilder(); foreach (var token in tokens) { if (!onlyDefaultChannel || token.Channel == 0) { resultString.Append(RenderToken(token, TokenValueDisplayMode.Show, false, vocabulary)); if (eachTokenOnNewLine) { resultString.AppendLine(); } else { resultString.Append(" "); } } } resultString.Append("EOF"); return(resultString.ToString()); }
private static string RenderToken(IToken token, TokenValueDisplayMode tokenValueDisplayMode = TokenValueDisplayMode.Show, bool showChannel = false, IVocabulary vocabulary = null) { string symbolicName = vocabulary?.GetSymbolicName(token.Type) ?? token.Type.ToString(); string value = tokenValueDisplayMode == TokenValueDisplayMode.Trim ? token.Text?.Trim() : token.Text; string tokenValue = string.Empty; if (tokenValueDisplayMode != TokenValueDisplayMode.Ignore && value != null && symbolicName != value) { tokenValue = value.Length <= MaxTokenValueLength ? value : value.Substring(0, MaxTokenValueLength) + "..."; } string channelValue = string.Empty; if (showChannel) { channelValue = "c: " + token.Channel.ToString(); // TODO: channel name instead of identifier. } string result = symbolicName; if (!string.IsNullOrEmpty(tokenValue) || !string.IsNullOrEmpty(channelValue)) { var strings = new List <string>(); if (!string.IsNullOrEmpty(tokenValue)) { strings.Add(tokenValue); } if (!string.IsNullOrEmpty(channelValue)) { strings.Add(channelValue); } result = $"{result} ({(string.Join(", ", strings))})"; } return(result); }