private static bool IsEndOfText(
string wikiText,
int startingIndex,
int charOffset,
IList <WikiTextTokenDef> partiallyMatchingTokens,
ICollection <WikiTextTokenDef> matchedTokens)
{
Contract.Requires(wikiText != null);
Contract.Requires(partiallyMatchingTokens != null);
Contract.Requires(matchedTokens != null);
if (startingIndex + charOffset < wikiText.Length)
{
return(false);
}
if (partiallyMatchingTokens.Count == 0)
{
return(true);
}
WikiTextTokenDef tokenDef = partiallyMatchingTokens[0];
if (tokenDef.IsRegexToken && charOffset > 0)
{
matchedTokens.Add(tokenDef);
}
return(true);
}
public IList <WikiTextToken> TokenizeWikiText(string wikiText, WikiTokenizationSettings settings)
{
List <WikiTextToken> tokens = new List <WikiTextToken> ();
int index = 0;
int?textTokenStart = null;
int scope = settings.IsWholeLine ? (int)WikiTextTokenScopes.LineStart : (int)WikiTextTokenScopes.InnerText;
while (true)
{
if (index >= wikiText.Length)
{
break;
}
int tokenEndingIndex;
WikiTextTokenDef tokenFound = LookForMatchingToken(
wikiText, ref scope, index, out tokenEndingIndex);
if (tokenFound != null)
{
AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);
string tokenText = wikiText.Substring(index, tokenEndingIndex - index);
tokens.Add(new WikiTextToken(tokenFound.TokenType, tokenText, (WikiTextTokenScopes)scope));
index = tokenEndingIndex;
}
else
{
AddToTextToken(index, ref textTokenStart);
// be sure to remove any LineStart scopes
if ((scope & (int)WikiTextTokenScopes.LineStart) != 0)
{
scope = (scope | (int)WikiTextTokenScopes.InnerText) & ~(int)WikiTextTokenScopes.LineStart;
}
index++;
}
}
AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);
return(tokens);
}
private static WikiTextTokenDef FetchBestMatchingToken(
ref int scope, int startingIndex, out int endingIndex, IList <WikiTextTokenDef> matchedTokens, int charOffset)
{
WikiTextTokenDef longestMatchedToken = matchedTokens[matchedTokens.Count - 1];
Contract.Assume(longestMatchedToken != null);
if (longestMatchedToken.IsRegexToken)
{
endingIndex = startingIndex + charOffset - 1;
}
else
{
endingIndex = startingIndex + longestMatchedToken.TokenStringLength;
}
scope = longestMatchedToken.ModifyScope(scope) & ((int)~WikiTextTokenScopes.LineStart);
return(longestMatchedToken);
}
private static bool DetermineIfTokenStillMatches(
IList <WikiTextTokenDef> partiallyMatchingTokens,
ICollection <WikiTextTokenDef> matchedTokens,
int tokenIndex,
WikiTextTokenDef tokenDef,
out bool tokenStillMatches,
int charOffset,
char textChar)
{
Contract.Requires(partiallyMatchingTokens != null);
Contract.Requires(matchedTokens != null);
Contract.Requires(tokenDef != null);
if (tokenDef.IsRegexToken)
{
tokenStillMatches = tokenDef.TokenRegex.IsMatch(textChar.ToString(CultureInfo.InvariantCulture));
}
else
{
char tokenChar = tokenDef.TokenString[charOffset];
if (tokenChar == textChar)
{
if (charOffset == tokenDef.TokenString.Length - 1)
{
matchedTokens.Add(tokenDef);
partiallyMatchingTokens.RemoveAt(tokenIndex);
tokenStillMatches = false;
return(true);
}
tokenStillMatches = true;
}
else
{
tokenStillMatches = false;
}
}
return(false);
}
private static void HandleMatchingOrNonMatching(
IList <WikiTextTokenDef> partiallyMatchingTokens,
ICollection <WikiTextTokenDef> matchedTokens,
ref int tokenIndex,
WikiTextTokenDef tokenDef,
bool tokenStillMatches,
int charOffset)
{
if (tokenStillMatches)
{
tokenIndex++;
}
else
{
if (tokenDef.IsRegexToken && charOffset > 0)
{
matchedTokens.Add(tokenDef);
}
partiallyMatchingTokens.RemoveAt(tokenIndex);
}
}
private static void MatchCharToToken(
IList <WikiTextTokenDef> partiallyMatchingTokens,
ICollection <WikiTextTokenDef> matchedTokens,
ref int tokenIndex,
int charOffset,
char textChar)
{
Contract.Requires(tokenIndex >= 0 && tokenIndex < partiallyMatchingTokens.Count);
Contract.Ensures(tokenIndex >= 0 && tokenIndex <= partiallyMatchingTokens.Count);
WikiTextTokenDef tokenDef = partiallyMatchingTokens[tokenIndex];
bool tokenStillMatches;
bool tokenAlreadyRemoved = DetermineIfTokenStillMatches(
partiallyMatchingTokens, matchedTokens, tokenIndex, tokenDef, out tokenStillMatches, charOffset, textChar);
if (tokenAlreadyRemoved)
{
return;
}
HandleMatchingOrNonMatching(partiallyMatchingTokens, matchedTokens, ref tokenIndex, tokenDef, tokenStillMatches, charOffset);
}
private static bool SelectTokensInScope(WikiTextTokenDef tokenDef, int scope)
{
return(((int)tokenDef.AvailableInScopes & scope) != 0);
}
