Example #1
0
 public Tokenizer(string text)
 {
     Text            = text;
     Stream          = new ParseStream(text);
     TextTokenStream = new TextTokenStream(Stream);
 }
Example #2
0
 public TextTokenStream(ParseStream stream)
 {
     this.stream = stream;
     builder     = new StringBuilder();
 }