public static List <Token> TokenizeText(string text, ScannerOptions opts)
{
using (StringReader reader = new StringReader(text))
{
Scanner s = new Scanner(new TextReaderBag(reader), opts);
return(s.Tokenize());
}
}
public Scanner(TextReaderBag bag, ScannerOptions options)
{
ThrowIfNull(bag);
ThrowIfNull(options);
_readerBag = bag;
_buffer = new ScannerBuffer(bag.TextReader);
_options = options;
}
public ExpressionEvaluator(Dictionary <string, Macro> macroMap = null)
{
_macroMap = macroMap ?? new Dictionary <string, Macro>();
_opts = new ScannerOptions();
_opts.HideComments = true;
_opts.HideNewLines = true;
_opts.HideWhitespace = true;
_opts.ThrowOnEndOfStream = false;
}
private ScannerOptions CreateScannerOptions()
{
ScannerOptions opts = new ScannerOptions();
opts.HideComments = true;
opts.HideNewLines = false;
opts.HideWhitespace = false;
opts.ThrowOnEndOfStream = false;
return(opts);
}