public static List <Token> TokenizeText(string text, ScannerOptions opts) { using (StringReader reader = new StringReader(text)) { Scanner s = new Scanner(new TextReaderBag(reader), opts); return(s.Tokenize()); } }
public Scanner(TextReaderBag bag, ScannerOptions options) { ThrowIfNull(bag); ThrowIfNull(options); _readerBag = bag; _buffer = new ScannerBuffer(bag.TextReader); _options = options; }
public ExpressionEvaluator(Dictionary <string, Macro> macroMap = null) { _macroMap = macroMap ?? new Dictionary <string, Macro>(); _opts = new ScannerOptions(); _opts.HideComments = true; _opts.HideNewLines = true; _opts.HideWhitespace = true; _opts.ThrowOnEndOfStream = false; }
private ScannerOptions CreateScannerOptions() { ScannerOptions opts = new ScannerOptions(); opts.HideComments = true; opts.HideNewLines = false; opts.HideWhitespace = false; opts.ThrowOnEndOfStream = false; return(opts); }