示例#1
0
        IList <WikiTextToken> IWikiTextTokenizer.TokenizeWikiText(string wikiText, WikiTokenizationSettings settings)
        {
            Contract.Requires(wikiText != null);
            Contract.Requires(settings != null);
            Contract.Ensures(Contract.Result <IList <WikiTextToken> >() != null);
            Contract.Ensures(Contract.ForAll(Contract.Result <IList <WikiTextToken> >(), x => x != null));

            throw new System.NotImplementedException();
        }
示例#2
0
        private void ProcessLine(
            DocumentDefBuilder docBuilder,
            ParsingContext context,
            string lineText)
        {
            Contract.Requires(docBuilder != null);
            Contract.Requires(lineText != null);

            WikiTokenizationSettings tokenizationSettings = new WikiTokenizationSettings();

            tokenizationSettings.IsWholeLine = true;
            IList <WikiTextToken> tokens = tokenizer.TokenizeWikiText(lineText, tokenizationSettings);

            if (tokens.Count == 0)
            {
                throw new NotImplementedException("todo next:");
            }

            TokenBuffer tokenBuffer = new TokenBuffer(tokens);

            WikiTextToken firstToken = tokenBuffer.Token;

            switch (firstToken.Type)
            {
            case WikiTextToken.TokenType.Heading1Start:
            case WikiTextToken.TokenType.Heading2Start:
            case WikiTextToken.TokenType.Heading3Start:
            case WikiTextToken.TokenType.Heading4Start:
            case WikiTextToken.TokenType.Heading5Start:
            case WikiTextToken.TokenType.Heading6Start:
                docBuilder.FinalizeCurrentParagraph();
                HandleHeadingLine(docBuilder, context, tokenBuffer);
                break;

            case WikiTextToken.TokenType.Text:
            case WikiTextToken.TokenType.DoubleApostrophe:
            case WikiTextToken.TokenType.TripleApostrophe:
            case WikiTextToken.TokenType.DoubleSquareBracketsOpen:
            case WikiTextToken.TokenType.SingleSquareBracketsOpen:
            case WikiTextToken.TokenType.BulletList:
            case WikiTextToken.TokenType.NumberedList:
            case WikiTextToken.TokenType.Indent:
                HandleText(docBuilder, context, tokenBuffer);
                break;

            default:
                throw new NotImplementedException("todo next: {0}".Fmt(firstToken.Type));
            }
        }
示例#3
0
        public IList <WikiTextToken> TokenizeWikiText(string wikiText, WikiTokenizationSettings settings)
        {
            List <WikiTextToken> tokens = new List <WikiTextToken> ();

            int index          = 0;
            int?textTokenStart = null;

            int scope = settings.IsWholeLine ? (int)WikiTextTokenScopes.LineStart : (int)WikiTextTokenScopes.InnerText;

            while (true)
            {
                if (index >= wikiText.Length)
                {
                    break;
                }

                int tokenEndingIndex;

                WikiTextTokenDef tokenFound = LookForMatchingToken(
                    wikiText, ref scope, index, out tokenEndingIndex);
                if (tokenFound != null)
                {
                    AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);

                    string tokenText = wikiText.Substring(index, tokenEndingIndex - index);
                    tokens.Add(new WikiTextToken(tokenFound.TokenType, tokenText, (WikiTextTokenScopes)scope));
                    index = tokenEndingIndex;
                }
                else
                {
                    AddToTextToken(index, ref textTokenStart);
                    // be sure to remove any LineStart scopes
                    if ((scope & (int)WikiTextTokenScopes.LineStart) != 0)
                    {
                        scope = (scope | (int)WikiTextTokenScopes.InnerText) & ~(int)WikiTextTokenScopes.LineStart;
                    }
                    index++;
                }
            }

            AddTextTokenIfAny(wikiText, tokens, ref textTokenStart, index, (WikiTextTokenScopes)scope);

            return(tokens);
        }