protected Token ParseToken(List <TokenPart> text, int position,
StringBuilder tokenValue, TokenType type)
{
var parserOperator = new ParserOperator();
if (text.Count == 0 && !IsTokenCorrupted)
{
return(CreateEmptyToken(tokenValue, position, parserOperator));
}
CollectToken(text, tokenValue, parserOperator);
var value = tokenValue.ToString();
if (text.Count != 0 && CheckCorrectTokenValue(tokenValue,
parserOperator,
text.FirstOrDefault().Value,
text.LastOrDefault().Value))
{
RecoverTokenValue(tokenValue, parserOperator);
type = TokenType.Simple;
value = tokenValue.ToString();
}
var nestedTokens = parserOperator.GetTokens();
var token = new Token(position, value, type);
token.SetNestedTokens(nestedTokens);
return(token);
}
public IEnumerable <Token> ReadTokens(string text)
{
var parserOperator = new ParserOperator();
var splittedText = text
.SplitKeepSeparators(new[] { '_', '#', '\\' })
.UnionSameStringByTwo("#")
.OperateEscaped(formattingCharacters);
foreach (var bigram in splittedText.GetBigrams())
{
parserOperator.AddTokenPart(bigram);
}
return(parserOperator.GetTokens());
}