public static string GetTokensString(this IList <IToken> tokens, IVocabulary vocabulary,
TokenValueDisplayMode tokenValueDisplayMode = TokenValueDisplayMode.Show, bool onlyDefaultChannel = false,
bool eachTokenOnNewLine = true)
{
var resultString = new StringBuilder();
foreach (var token in tokens)
{
if (!onlyDefaultChannel || token.Channel == 0)
{
resultString.Append(RenderToken(token, TokenValueDisplayMode.Show, false, vocabulary));
if (eachTokenOnNewLine)
{
resultString.AppendLine();
}
else
{
resultString.Append(" ");
}
}
}
resultString.Append("EOF");
return(resultString.ToString());
}
private static string RenderToken(IToken token, TokenValueDisplayMode tokenValueDisplayMode = TokenValueDisplayMode.Show, bool showChannel = false, IVocabulary vocabulary = null)
{
string symbolicName = vocabulary?.GetSymbolicName(token.Type) ?? token.Type.ToString();
string value = tokenValueDisplayMode == TokenValueDisplayMode.Trim ? token.Text?.Trim() : token.Text;
string tokenValue = string.Empty;
if (tokenValueDisplayMode != TokenValueDisplayMode.Ignore && value != null && symbolicName != value)
{
tokenValue = value.Length <= MaxTokenValueLength
? value
: value.Substring(0, MaxTokenValueLength) + "...";
}
string channelValue = string.Empty;
if (showChannel)
{
channelValue = "c: " + token.Channel.ToString(); // TODO: channel name instead of identifier.
}
string result = symbolicName;
if (!string.IsNullOrEmpty(tokenValue) || !string.IsNullOrEmpty(channelValue))
{
var strings = new List <string>();
if (!string.IsNullOrEmpty(tokenValue))
{
strings.Add(tokenValue);
}
if (!string.IsNullOrEmpty(channelValue))
{
strings.Add(channelValue);
}
result = $"{result} ({(string.Join(", ", strings))})";
}
return(result);
}
