public static List <ReadOnlyMemory <char> > Tokenize(string input, TokenParsingFunc tokenParser = null, bool supportComments = true) { List <ReadOnlyMemory <char> > tokens = new List <ReadOnlyMemory <char> >(); Tokenize(tokens, input, tokenParser, supportComments); return(tokens); }
public static void Tokenize(List <ReadOnlyMemory <char> > r_Tokens, string input, TokenParsingFunc tokenParser = null, bool supportComments = true) { if (tokenParser == null) { tokenParser = DefaultTokenParser; } for (int i = 0; i < input.Length;) { SkipWhitespace(ref i, input); if (supportComments) { SkipComment(ref i, input); } if (i == input.Length) { break; } var token = tokenParser(ref i, input); if (!token.IsEmpty) { r_Tokens.Add(token); } else { i++; } } }