Beispiel #1
0
        private static bool IsEndOfText(
            string wikiText,
            int startingIndex,
            int charOffset,
            IList <WikiTextTokenDef> partiallyMatchingTokens,
            ICollection <WikiTextTokenDef> matchedTokens)
        {
            Contract.Requires(wikiText != null);
            Contract.Requires(partiallyMatchingTokens != null);
            Contract.Requires(matchedTokens != null);

            if (startingIndex + charOffset < wikiText.Length)
            {
                return(false);
            }

            if (partiallyMatchingTokens.Count == 0)
            {
                return(true);
            }

            WikiTextTokenDef tokenDef = partiallyMatchingTokens[0];

            if (tokenDef.IsRegexToken && charOffset > 0)
            {
                matchedTokens.Add(tokenDef);
            }

            return(true);
        }
Beispiel #2
0
        public IList <WikiTextToken> TokenizeWikiText(string wikiText, WikiTokenizationSettings settings)
        {
            List <WikiTextToken> tokens = new List <WikiTextToken> ();

            int index          = 0;
            int?textTokenStart = null;

            int scope = settings.IsWholeLine ? (int)WikiTextTokenScopes.LineStart : (int)WikiTextTokenScopes.InnerText;

            while (true)
            {
                if (index >= wikiText.Length)
                {
                    break;
                }

                int tokenEndingIndex;

                WikiTextTokenDef tokenFound = LookForMatchingToken(
                    wikiText, ref scope, index, out tokenEndingIndex);
                if (tokenFound != null)
                {
                    AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);

                    string tokenText = wikiText.Substring(index, tokenEndingIndex - index);
                    tokens.Add(new WikiTextToken(tokenFound.TokenType, tokenText, (WikiTextTokenScopes)scope));
                    index = tokenEndingIndex;
                }
                else
                {
                    AddToTextToken(index, ref textTokenStart);
                    // be sure to remove any LineStart scopes
                    if ((scope & (int)WikiTextTokenScopes.LineStart) != 0)
                    {
                        scope = (scope | (int)WikiTextTokenScopes.InnerText) & ~(int)WikiTextTokenScopes.LineStart;
                    }
                    index++;
                }
            }

            AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);

            return(tokens);
        }
Beispiel #3
0
        private static WikiTextTokenDef FetchBestMatchingToken(
            ref int scope, int startingIndex, out int endingIndex, IList <WikiTextTokenDef> matchedTokens, int charOffset)
        {
            WikiTextTokenDef longestMatchedToken = matchedTokens[matchedTokens.Count - 1];

            Contract.Assume(longestMatchedToken != null);

            if (longestMatchedToken.IsRegexToken)
            {
                endingIndex = startingIndex + charOffset - 1;
            }
            else
            {
                endingIndex = startingIndex + longestMatchedToken.TokenStringLength;
            }

            scope = longestMatchedToken.ModifyScope(scope) & ((int)~WikiTextTokenScopes.LineStart);
            return(longestMatchedToken);
        }
Beispiel #4
0
        private static bool DetermineIfTokenStillMatches(
            IList <WikiTextTokenDef> partiallyMatchingTokens,
            ICollection <WikiTextTokenDef> matchedTokens,
            int tokenIndex,
            WikiTextTokenDef tokenDef,
            out bool tokenStillMatches,
            int charOffset,
            char textChar)
        {
            Contract.Requires(partiallyMatchingTokens != null);
            Contract.Requires(matchedTokens != null);
            Contract.Requires(tokenDef != null);

            if (tokenDef.IsRegexToken)
            {
                tokenStillMatches = tokenDef.TokenRegex.IsMatch(textChar.ToString(CultureInfo.InvariantCulture));
            }
            else
            {
                char tokenChar = tokenDef.TokenString[charOffset];
                if (tokenChar == textChar)
                {
                    if (charOffset == tokenDef.TokenString.Length - 1)
                    {
                        matchedTokens.Add(tokenDef);
                        partiallyMatchingTokens.RemoveAt(tokenIndex);
                        tokenStillMatches = false;
                        return(true);
                    }

                    tokenStillMatches = true;
                }
                else
                {
                    tokenStillMatches = false;
                }
            }

            return(false);
        }
Beispiel #5
0
        private static void HandleMatchingOrNonMatching(
            IList <WikiTextTokenDef> partiallyMatchingTokens,
            ICollection <WikiTextTokenDef> matchedTokens,
            ref int tokenIndex,
            WikiTextTokenDef tokenDef,
            bool tokenStillMatches,
            int charOffset)
        {
            if (tokenStillMatches)
            {
                tokenIndex++;
            }
            else
            {
                if (tokenDef.IsRegexToken && charOffset > 0)
                {
                    matchedTokens.Add(tokenDef);
                }

                partiallyMatchingTokens.RemoveAt(tokenIndex);
            }
        }
Beispiel #6
0
        private static void MatchCharToToken(
            IList <WikiTextTokenDef> partiallyMatchingTokens,
            ICollection <WikiTextTokenDef> matchedTokens,
            ref int tokenIndex,
            int charOffset,
            char textChar)
        {
            Contract.Requires(tokenIndex >= 0 && tokenIndex < partiallyMatchingTokens.Count);
            Contract.Ensures(tokenIndex >= 0 && tokenIndex <= partiallyMatchingTokens.Count);

            WikiTextTokenDef tokenDef = partiallyMatchingTokens[tokenIndex];

            bool tokenStillMatches;
            bool tokenAlreadyRemoved = DetermineIfTokenStillMatches(
                partiallyMatchingTokens, matchedTokens, tokenIndex, tokenDef, out tokenStillMatches, charOffset, textChar);

            if (tokenAlreadyRemoved)
            {
                return;
            }

            HandleMatchingOrNonMatching(partiallyMatchingTokens, matchedTokens, ref tokenIndex, tokenDef, tokenStillMatches, charOffset);
        }
Beispiel #7
0
 private static bool SelectTokensInScope(WikiTextTokenDef tokenDef, int scope)
 {
     return(((int)tokenDef.AvailableInScopes & scope) != 0);
 }