public static ref Tokenizer NewTokenizer(@string name, io.Reader r, ref os.File file)
{
scanner.Scanner s = default;
s.Init(r);
// Newline is like a semicolon; other space characters are fine.
s.Whitespace = 1L << (int)('\t') | 1L << (int)('\r') | 1L << (int)(' ');
// Don't skip comments: we need to count newlines.
s.Mode = scanner.ScanChars | scanner.ScanFloats | scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings | scanner.ScanComments;
s.Position.Filename = name;
s.IsIdentRune = isIdentRune;
return(ref new Tokenizer(s: &s, base : src.NewFileBase(name, objabi.AbsFile(objabi.WorkingDir(), name, *flags.TrimPath)), line: 1, file: file, ));
}
