コード例 #1
0
        // Scan the input string, creating tokens for anything special
        public void Tokenize(string str, int start, int len)
        {
            // Prepare
            base.Reset(str, start, len);
            m_Tokens.Clear();

            List <Token> emphasis_marks = null;

            List <Abbreviation> Abbreviations = m_Markdown.GetAbbreviations();
            bool ExtraMode = m_Markdown.ExtraMode;

            // Scan string
            int start_text_token = Position;

            while (!Eof)
            {
                int end_text_token = Position;

                // Work out token
                Token token        = null;
                int   positionSave = Position;
                switch (Current)
                {
                case '*':
                case '_':
                    // Create emphasis mark
                    token = CreateEmphasisMark();

                    if (token != null)
                    {
                        // Store marks in a separate list the we'll resolve later
                        switch (token.type)
                        {
                        case TokenType.internal_mark:
                        case TokenType.opening_mark:
                        case TokenType.closing_mark:
                            if (emphasis_marks == null)
                            {
                                emphasis_marks = new List <Token>();
                            }
                            emphasis_marks.Add(token);
                            break;
                        }
                    }
                    break;

                case '~':
                    if (m_Markdown.HnDMode)
                    {
                        // scan for ~~text~~.
                        token = ProcessStrikeThrough();
                    }
                    break;

                case '`':
                    token = ProcessCodeSpan();
                    break;

                case '[':
                case '!':
                    // Process link reference
                    int linkpos = Position;
                    token = ProcessLinkOrImageOrFootnote();

                    // Rewind if invalid syntax
                    // (the '[' or '!' will be treated as a regular character and processed below)
                    if (token == null)
                    {
                        Position = linkpos;
                    }
                    break;

                case '<':
                    // Is it a valid html tag?
                    HtmlTag tag = HtmlTag.Parse(this);
                    if (tag != null)
                    {
                        if (!m_Markdown.SafeMode || tag.IsSafe())
                        {
                            // Yes, create a token for it
                            token = CreateToken(TokenType.HtmlTag, positionSave, Position - positionSave);
                        }
                        else
                        {
                            // No, rewrite and encode it
                            Position = positionSave;
                        }
                    }
                    else
                    {
                        // No, rewind and check if it's a valid autolink eg: <google.com>
                        Position = positionSave;
                        token    = ProcessAutoLink();

                        if (token == null)
                        {
                            Position = positionSave;
                        }
                    }
                    break;

                case '&':
                    // Is it a valid html entity
                    string unused = null;
                    if (SkipHtmlEntity(ref unused))
                    {
                        // Yes, create a token for it
                        token = CreateToken(TokenType.Html, positionSave, Position - positionSave);
                    }

                    break;

                case ' ':
                    // Check for double space at end of a line
                    if (CharAtOffset(1) == ' ' && IsLineEnd(CharAtOffset(2)))
                    {
                        // Yes, skip it
                        SkipForward(2);

                        // Don't put br's at the end of a paragraph
                        if (!Eof)
                        {
                            SkipEol();
                            token = CreateToken(TokenType.br, end_text_token, 0);
                        }
                    }
                    break;

                case '\\':
                    // Special handling for escaping <autolinks>

                    /*
                     * if (CharAtOffset(1) == '<')
                     * {
                     *      // Is it an autolink?
                     *      int savepos = position;
                     *      SkipForward(1);
                     *      bool AutoLink = ProcessAutoLink() != null;
                     *      position = savepos;
                     *
                     *      if (AutoLink)
                     *      {
                     *              token = CreateToken(TokenType.Text, position + 1, 1);
                     *              SkipForward(2);
                     *      }
                     * }
                     * else
                     */
                {
                    // Check followed by an escapable character
                    if (Utils.IsEscapableChar(CharAtOffset(1), ExtraMode))
                    {
                        token = CreateToken(TokenType.Text, Position + 1, 1);
                        SkipForward(2);
                    }
                }
                break;

                case '@':
                    if (m_Markdown.DocNetMode || m_Markdown.HnDMode)
                    {
                        if (this.DoesMatch("@fa-"))
                        {
                            // expect FontAwesome.
                            string iconName    = string.Empty;
                            int    newPosition = 0;
                            if (Utils.SkipFontAwesome(this.Input, this.Position, out newPosition, out iconName))
                            {
                                // token should be just the iconname, so adjust position specification to that.
                                token         = CreateToken(TokenType.font_awesome, newPosition - iconName.Length, iconName.Length);
                                this.Position = newPosition;
                            }
                        }
                    }
                    break;

                case ':':
                    if (m_Markdown.HnDMode && m_Markdown.EmojiFilePerName != null)
                    {
                        // scan till next ':' and stop if EOL or whitespace is seen.
                        string emojiName   = string.Empty;
                        int    newPosition = 0;
                        if (Utils.ParseEmoji(this.Input, this.Position, out newPosition, out emojiName))
                        {
                            token         = CreateToken(TokenType.emoji, emojiName);
                            this.Position = newPosition;
                        }
                        else
                        {
                            // try smiley mapping
                            token = ParseSmileyMapping(positionSave);
                        }
                    }
                    break;

                default:
                    token = ParseSmileyMapping(positionSave);
                    break;
                }

                // Look for abbreviations.
                if (token == null && Abbreviations != null && !Char.IsLetterOrDigit(CharAtOffset(-1)))
                {
                    var savepos = Position;
                    foreach (var abbr in Abbreviations)
                    {
                        if (SkipString(abbr.Abbr) && !Char.IsLetterOrDigit(Current))
                        {
                            token = CreateToken(TokenType.abbreviation, abbr);
                            break;
                        }

                        Position = savepos;
                    }
                }

                // If token found, append any preceeding text and the new token to the token list
                if (token != null)
                {
                    // Create a token for everything up to the special character
                    if (end_text_token > start_text_token)
                    {
                        m_Tokens.Add(CreateToken(TokenType.Text, start_text_token, end_text_token - start_text_token));
                    }

                    // Add the new token
                    m_Tokens.Add(token);

                    // Remember where the next text token starts
                    start_text_token = Position;
                }
                else
                {
                    // Skip a single character and keep looking
                    SkipForward(1);
                }
            }

            // Append a token for any trailing text after the last token.
            if (Position > start_text_token)
            {
                m_Tokens.Add(CreateToken(TokenType.Text, start_text_token, Position - start_text_token));
            }

            // Do we need to resolve and emphasis marks?
            if (emphasis_marks != null)
            {
                ResolveEmphasisMarks(m_Tokens, emphasis_marks);
            }
        }