Tokenizer inputs a stream of Unicode characters and outputs tokens for the Fantom serialization grammar.
Tokenizer Class Documentation
Beispiel #1
0
 internal void reset(Tokenizer t)
 {
     t.reset(type, val, line);
 }
Beispiel #2
0
 //////////////////////////////////////////////////////////////////////////
 // Constructor
 //////////////////////////////////////////////////////////////////////////
 /// <summary>
 /// Construct for input stream.
 /// </summary>
 public ObjDecoder(InStream @in, Map options)
 {
     tokenizer = new Tokenizer(@in);
       this.options = options;
       consume();
 }