public static List <Token> TokenizeText(string text, ScannerOptions opts) { using (StringReader reader = new StringReader(text)) { Scanner s = new Scanner(new TextReaderBag(reader), opts); return(s.Tokenize()); } }
public bool TryParse(string expression, out ExpressionNode node) { if (expression == null) { throw new ArgumentNullException("expression"); } using (var reader = new StringReader(expression)) { Scanner scanner = new Scanner(reader); scanner.Options.HideNewLines = true; scanner.Options.HideComments = true; scanner.Options.HideWhitespace = true; scanner.Options.ThrowOnEndOfStream = false; return(TryParseComplete(scanner.Tokenize(), out node)); } }