protected virtual void CollectToken(List <TokenPart> text,
StringBuilder tokenValue, ParserOperator parserOperator)
{
var isIntoToken = false;
var offset = IsTokenCorrupted ? corruptedOffset : 0;
foreach (var bigram in text.GetBigrams())
{
var part = bigram.Previous;
if (nestedTokenValidator(part.Value) && !part.Escaped)
{
if (isIntoToken)
{
parserOperator.Position = offset;
parserOperator.AddTokenPart(bigram);
}
isIntoToken = !isIntoToken;
}
else if (!isIntoToken)
{
tokenValue.Append(part.Value);
offset += part.Value.Length;
}
if (isIntoToken)
{
parserOperator.AddTokenPart(bigram);
}
}
parserOperator.Position = offset;
}
public IEnumerable <Token> ReadTokens(string text)
{
var parserOperator = new ParserOperator();
var splittedText = text
.SplitKeepSeparators(new[] { '_', '#', '\\' })
.UnionSameStringByTwo("#")
.OperateEscaped(formattingCharacters);
foreach (var bigram in splittedText.GetBigrams())
{
parserOperator.AddTokenPart(bigram);
}
return(parserOperator.GetTokens());
}