/// <summary> /// Figures out what token to start replacing in the old token list. /// Tokenization should start right before the returned token index. /// </summary> private static int FindTokenToStart(TokenList tokens, int textStart, char firstNewChar) { int oldTokenIndex = tokens.FindInsertIndex(textStart, beforeExisting: true); if (oldTokenIndex > 0) { // Go back until there is whitespace before the current token (oldTokenIndex) int mustEndBefore = char.IsWhiteSpace(firstNewChar) ? textStart + 1 : textStart; while (oldTokenIndex > 0 && !TokenEndsBefore(tokens[oldTokenIndex - 1], mustEndBefore)) { oldTokenIndex--; mustEndBefore = tokens[oldTokenIndex].Start; } if (oldTokenIndex < tokens.Count) { // Go back to find a non-child token (example: can't start tokenzing inside of comment text) while (oldTokenIndex > 0 && tokens[oldTokenIndex].IsChildToken) { oldTokenIndex--; } } } return(oldTokenIndex); }
/// <summary> /// Global method to compute the Result of a token change /// </summary> public static Result TokenizeChange( ICssTokenizerFactory tokenizerFactory, TokenList oldTokens, ITextProvider oldText, ITextProvider newText, int changeStart, int deletedLength, int insertedLength) { Result result = new Result(); char firstInsertedChar = (insertedLength > 0) ? newText[changeStart] : '\0'; result.NewTokens = new TokenList(); result.OldTokens = oldTokens; result.OldTokenStart = FindTokenToStart(oldTokens, changeStart, firstInsertedChar); result.OldTokenCount = oldTokens.Count - result.OldTokenStart; // assume delete to EOF result.OldTokenTextOffset = insertedLength - deletedLength; result.TokenizationStart = changeStart; if (result.OldTokenStart < oldTokens.Count) { // The first old token may start before the actual text change. // Adjust where tokenization starts: result.TokenizationStart = Math.Min(result.TokenizationStart, oldTokens[result.OldTokenStart].Start); } // Tokenize until EOF or until the new tokens start matching the old tokens bool tokenizeUntilEOF = (oldTokens.Count == 0); // Create and init a streaming tokenizer ICssTokenizer tokenizer = tokenizerFactory.CreateTokenizer(); int estimatedLength = (tokenizeUntilEOF ? newText.Length - result.TokenizationStart : insertedLength); tokenizer.InitStream(newText, result.TokenizationStart, estimatedLength, keepWhiteSpace: false); for (CssToken token = tokenizer.StreamNextToken(); true; token = tokenizer.StreamNextToken()) { if (token.TokenType != CssTokenType.EndOfFile && !tokenizeUntilEOF && token.Start >= changeStart + insertedLength) { // This could be a good token for stopping, see if it matches an old token int oldTokenStart = token.Start - result.OldTokenTextOffset; int oldTokenIndex = oldTokens.FindInsertIndex(oldTokenStart, beforeExisting: true); if (oldTokenIndex == oldTokens.Count) { tokenizeUntilEOF = true; } else { CssToken oldToken = oldTokens[oldTokenIndex]; if (oldToken.Start == oldTokenStart && CssToken.CompareTokens(token, oldToken, newText, oldText)) { result.OldTokenCount = oldTokenIndex - result.OldTokenStart; break; } } } result.NewTokens.Add(token); if (token.TokenType == CssTokenType.EndOfFile) { break; } } return(result); }