private Token GetNonTextToken(string paragraph, int position)
{
var length = paragraph.Length;
var tokenLength = 0;
var currentTokenContent = "";
while (position < length)
{
var currentSymbol = paragraph[position];
var nextContent = currentTokenContent + currentSymbol;
if (IsPartOfAnyNonTextToken(nextContent))
{
currentTokenContent = nextContent;
tokenLength++;
position++;
}
else
{
break;
}
}
return(TokenTypesTranslator.GetTokenFromString(currentTokenContent));
}
private static bool IsPartOfAnyNonTextToken(string substring)
{
return(TokenTypesTranslator.GetSupportedTokens().Any(tokenTag => tokenTag.StartsWith(substring)));
}
public Token(TokenType type)
{
Type = type;
Content = TokenTypesTranslator.GetStringFromTokenType(type);
Length = Content.Length;
}
