private void Validate()
{
var invalidDelimiters = Delimiters.Where(d => InvalidDelimiters.Contains(d));
if (invalidDelimiters.Any())
{
throw new InvalidDelimitersException("Invalid delimiters detected, " + string.Join(',', invalidDelimiters));
}
}
/// <summary>
/// Tokenizes the given string into a list of tokens
/// </summary>
/// <param name="input">The string to tokenize</param>
/// <returns>The list of tokens</returns>
public List <T> Tokenize(string input)
{
currentTokenIndex = 0;
List <T> tokens = new List <T>();
T currentToken = null;
insideStringLiteral = false;
var singleCharDelimiters = Delimiters.Where(d => d.Length == 1).Select(d => d[0]).OrderBy(d => d).ToList();
char currentCharacter;
char?nextCharacter;
int currentIndex = -1;
int currentLine = 1;
int currentColumn = 0;
while (TryReadCharacter(input, ref currentIndex, out currentCharacter))
{
char peeked;
if (TryPeekCharacter(input, currentIndex, out peeked))
{
nextCharacter = peeked;
}
else
{
nextCharacter = null;
}
currentColumn++;
if (currentCharacter == '\n')
{
Tokenize_Whitespace(input, ref currentIndex, ref currentCharacter, currentLine, ref currentColumn, ref currentToken, tokens);
currentLine++;
currentColumn = 0;
}
else if (IsEscapeSequence(currentCharacter, nextCharacter))
{
Tokenize_EscapeCharacter(input, ref currentIndex, ref currentCharacter, currentLine, ref currentColumn, ref currentToken, tokens);
}
else if (insideStringLiteral)
{
Tokenize_Plain(input, ref currentIndex, ref currentCharacter, currentLine, ref currentColumn, ref currentToken, tokens);
}
else if (singleCharDelimiters.BinarySearch(currentCharacter) >= 0)
{
Tokenize_DelimiterCharacter(input, ref currentIndex, ref currentCharacter, currentLine, ref currentColumn, ref currentToken, tokens);
}
else if (char.IsWhiteSpace(currentCharacter))
{
Tokenize_Whitespace(input, ref currentIndex, ref currentCharacter, currentLine, ref currentColumn, ref currentToken, tokens);
}
else
{
Tokenize_Plain(input, ref currentIndex, ref currentCharacter, currentLine, ref currentColumn, ref currentToken, tokens);
}
}
FinalizeTokenIfNotNull(ref currentToken, tokens);
return(tokens);
}