protected virtual void CollectToken(List <TokenPart> text, StringBuilder tokenValue, ParserOperator parserOperator) { var isIntoToken = false; var offset = IsTokenCorrupted ? corruptedOffset : 0; foreach (var bigram in text.GetBigrams()) { var part = bigram.Previous; if (nestedTokenValidator(part.Value) && !part.Escaped) { if (isIntoToken) { parserOperator.Position = offset; parserOperator.AddTokenPart(bigram); } isIntoToken = !isIntoToken; } else if (!isIntoToken) { tokenValue.Append(part.Value); offset += part.Value.Length; } if (isIntoToken) { parserOperator.AddTokenPart(bigram); } } parserOperator.Position = offset; }
public IEnumerable <Token> ReadTokens(string text) { var parserOperator = new ParserOperator(); var splittedText = text .SplitKeepSeparators(new[] { '_', '#', '\\' }) .UnionSameStringByTwo("#") .OperateEscaped(formattingCharacters); foreach (var bigram in splittedText.GetBigrams()) { parserOperator.AddTokenPart(bigram); } return(parserOperator.GetTokens()); }