private Token GetNonTextToken(string paragraph, int position) { var length = paragraph.Length; var tokenLength = 0; var currentTokenContent = ""; while (position < length) { var currentSymbol = paragraph[position]; var nextContent = currentTokenContent + currentSymbol; if (IsPartOfAnyNonTextToken(nextContent)) { currentTokenContent = nextContent; tokenLength++; position++; } else { break; } } return(TokenTypesTranslator.GetTokenFromString(currentTokenContent)); }
private static bool IsPartOfAnyNonTextToken(string substring) { return(TokenTypesTranslator.GetSupportedTokens().Any(tokenTag => tokenTag.StartsWith(substring))); }
public Token(TokenType type) { Type = type; Content = TokenTypesTranslator.GetStringFromTokenType(type); Length = Content.Length; }