private static void dumpToken <T>(IToken <T> token, int indentLevel, List <Tuple <int, string> > state)
{
var rv = string.Format("TokenizerId:{0} StartPos:{1} Data:{2} Comment:{3}",
token.As <IHasTokenizerId <T> >(false).With(x => x.TokenizerId),
token.As <IStartEndPositionalToken <T> >(false).WithValueType(x => x.StartPos),
string.Join("", token.TokenData),
token.As <IHasCommentToken <T> >(false).With(x => x.Comment));
state.Add(new Tuple <int, string>(indentLevel, rv));
var comp = token.As <HasCompositeTokenDecoration <T> >();
if (comp != null && comp.ChildTokens != null)
{
foreach (var each in comp.ChildTokens)
{
dumpToken(each, indentLevel + 1, state);
}
}
}
