private static bool IsEndOfText( string wikiText, int startingIndex, int charOffset, IList <WikiTextTokenDef> partiallyMatchingTokens, ICollection <WikiTextTokenDef> matchedTokens) { Contract.Requires(wikiText != null); Contract.Requires(partiallyMatchingTokens != null); Contract.Requires(matchedTokens != null); if (startingIndex + charOffset < wikiText.Length) { return(false); } if (partiallyMatchingTokens.Count == 0) { return(true); } WikiTextTokenDef tokenDef = partiallyMatchingTokens[0]; if (tokenDef.IsRegexToken && charOffset > 0) { matchedTokens.Add(tokenDef); } return(true); }
public IList <WikiTextToken> TokenizeWikiText(string wikiText, WikiTokenizationSettings settings) { List <WikiTextToken> tokens = new List <WikiTextToken> (); int index = 0; int?textTokenStart = null; int scope = settings.IsWholeLine ? (int)WikiTextTokenScopes.LineStart : (int)WikiTextTokenScopes.InnerText; while (true) { if (index >= wikiText.Length) { break; } int tokenEndingIndex; WikiTextTokenDef tokenFound = LookForMatchingToken( wikiText, ref scope, index, out tokenEndingIndex); if (tokenFound != null) { AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope); string tokenText = wikiText.Substring(index, tokenEndingIndex - index); tokens.Add(new WikiTextToken(tokenFound.TokenType, tokenText, (WikiTextTokenScopes)scope)); index = tokenEndingIndex; } else { AddToTextToken(index, ref textTokenStart); // be sure to remove any LineStart scopes if ((scope & (int)WikiTextTokenScopes.LineStart) != 0) { scope = (scope | (int)WikiTextTokenScopes.InnerText) & ~(int)WikiTextTokenScopes.LineStart; } index++; } } AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope); return(tokens); }
private static WikiTextTokenDef FetchBestMatchingToken( ref int scope, int startingIndex, out int endingIndex, IList <WikiTextTokenDef> matchedTokens, int charOffset) { WikiTextTokenDef longestMatchedToken = matchedTokens[matchedTokens.Count - 1]; Contract.Assume(longestMatchedToken != null); if (longestMatchedToken.IsRegexToken) { endingIndex = startingIndex + charOffset - 1; } else { endingIndex = startingIndex + longestMatchedToken.TokenStringLength; } scope = longestMatchedToken.ModifyScope(scope) & ((int)~WikiTextTokenScopes.LineStart); return(longestMatchedToken); }
private static bool DetermineIfTokenStillMatches( IList <WikiTextTokenDef> partiallyMatchingTokens, ICollection <WikiTextTokenDef> matchedTokens, int tokenIndex, WikiTextTokenDef tokenDef, out bool tokenStillMatches, int charOffset, char textChar) { Contract.Requires(partiallyMatchingTokens != null); Contract.Requires(matchedTokens != null); Contract.Requires(tokenDef != null); if (tokenDef.IsRegexToken) { tokenStillMatches = tokenDef.TokenRegex.IsMatch(textChar.ToString(CultureInfo.InvariantCulture)); } else { char tokenChar = tokenDef.TokenString[charOffset]; if (tokenChar == textChar) { if (charOffset == tokenDef.TokenString.Length - 1) { matchedTokens.Add(tokenDef); partiallyMatchingTokens.RemoveAt(tokenIndex); tokenStillMatches = false; return(true); } tokenStillMatches = true; } else { tokenStillMatches = false; } } return(false); }
private static void HandleMatchingOrNonMatching( IList <WikiTextTokenDef> partiallyMatchingTokens, ICollection <WikiTextTokenDef> matchedTokens, ref int tokenIndex, WikiTextTokenDef tokenDef, bool tokenStillMatches, int charOffset) { if (tokenStillMatches) { tokenIndex++; } else { if (tokenDef.IsRegexToken && charOffset > 0) { matchedTokens.Add(tokenDef); } partiallyMatchingTokens.RemoveAt(tokenIndex); } }
private static void MatchCharToToken( IList <WikiTextTokenDef> partiallyMatchingTokens, ICollection <WikiTextTokenDef> matchedTokens, ref int tokenIndex, int charOffset, char textChar) { Contract.Requires(tokenIndex >= 0 && tokenIndex < partiallyMatchingTokens.Count); Contract.Ensures(tokenIndex >= 0 && tokenIndex <= partiallyMatchingTokens.Count); WikiTextTokenDef tokenDef = partiallyMatchingTokens[tokenIndex]; bool tokenStillMatches; bool tokenAlreadyRemoved = DetermineIfTokenStillMatches( partiallyMatchingTokens, matchedTokens, tokenIndex, tokenDef, out tokenStillMatches, charOffset, textChar); if (tokenAlreadyRemoved) { return; } HandleMatchingOrNonMatching(partiallyMatchingTokens, matchedTokens, ref tokenIndex, tokenDef, tokenStillMatches, charOffset); }
private static bool SelectTokensInScope(WikiTextTokenDef tokenDef, int scope) { return(((int)tokenDef.AvailableInScopes & scope) != 0); }