public static List <ReadOnlyMemory <char> > Tokenize(string input, TokenParsingFunc tokenParser = null, bool supportComments = true)
{
List <ReadOnlyMemory <char> > tokens = new List <ReadOnlyMemory <char> >();
Tokenize(tokens, input, tokenParser, supportComments);
return(tokens);
}
public static void Tokenize(List <ReadOnlyMemory <char> > r_Tokens, string input, TokenParsingFunc tokenParser = null, bool supportComments = true)
{
if (tokenParser == null)
{
tokenParser = DefaultTokenParser;
}
for (int i = 0; i < input.Length;)
{
SkipWhitespace(ref i, input);
if (supportComments)
{
SkipComment(ref i, input);
}
if (i == input.Length)
{
break;
}
var token = tokenParser(ref i, input);
if (!token.IsEmpty)
{
r_Tokens.Add(token);
}
else
{
i++;
}
}
}
