public virtual void MarkTokens(IDocument document)
        {
            if (Rules.Count == 0)
            {
                return;
            }

            int lineNumber = 0;

            while (lineNumber < document.TotalNumberOfLines)
            {
                LineSegment previousLine = (lineNumber > 0 ? document.GetLineSegment(lineNumber - 1) : null);

                if (lineNumber >= document.LineSegmentCollection.Count)
                {                 // may be, if the last line ends with a delimiter
                    break;        // then the last line is not in the collection :)
                }

                currentSpanStack = previousLine != null && previousLine.HighlightSpanStack != null?previousLine.HighlightSpanStack.Clone() : null;

                if (currentSpanStack != null)
                {
                    while (!currentSpanStack.IsEmpty && currentSpanStack.Peek().StopEOL)
                    {
                        currentSpanStack.Pop();
                    }

                    if (currentSpanStack.IsEmpty)
                    {
                        currentSpanStack = null;
                    }
                }

                currentLine = document.LineSegmentCollection[lineNumber];

                if (currentLine.Length == -1)
                {                 // happens when buffer is empty !
                    return;
                }

                currentLineNumber = lineNumber;
                List <TextWord> words = ParseLine(document);
                // Alex: clear old words

                if (currentLine.Words != null)
                {
                    currentLine.Words.Clear();
                }

                currentLine.Words = words;
                currentLine.HighlightSpanStack = currentSpanStack == null || currentSpanStack.IsEmpty ? null : currentSpanStack;

                ++lineNumber;
            }

            document.RequestUpdate(new TextAreaUpdate(TextAreaUpdateType.WholeTextArea));
            document.CommitUpdate();
            currentLine = null;
        }
Exemplo n.º 2
0
        public SpanStack Clone()
        {
            SpanStack n = new SpanStack
            {
                top = this.top
            };

            return(n);
        }
Exemplo n.º 3
0
        public virtual void MarkTokens(IDocument document)
        {
            if (Rules.Count == 0)
            {
                return;
            }

            var lineNumber = 0;

            while (lineNumber < document.TotalNumberOfLines)
            {
                var previousLine = lineNumber > 0 ? document.GetLineSegment(lineNumber - 1) : null;
                if (lineNumber >= document.LineSegmentCollection.Count)
                {
                    break; // then the last line is not in the collection :)
                }
                currentSpanStack = previousLine?.HighlightSpanStack?.Clone();

                if (currentSpanStack != null)
                {
                    while (!currentSpanStack.IsEmpty && currentSpanStack.Peek().StopEOL)
                    {
                        currentSpanStack.Pop();
                    }
                    if (currentSpanStack.IsEmpty)
                    {
                        currentSpanStack = null;
                    }
                }

                currentLine = document.LineSegmentCollection[lineNumber];

                if (currentLine.Length == -1)
                {
                    return;
                }

                currentLineNumber = lineNumber;
                var words = ParseLine(document);
                // Alex: clear old words
                currentLine.Words?.Clear();
                currentLine.Words = words;
                currentLine.HighlightSpanStack = currentSpanStack == null || currentSpanStack.IsEmpty ? null : currentSpanStack;

                ++lineNumber;
            }

            document.RequestUpdate(new TextAreaUpdate(TextAreaUpdateType.WholeTextArea));
            document.CommitUpdate();
            currentLine = null;
        }
Exemplo n.º 4
0
        List <TextWord> ParseLine(IDocument document)
        {
            List <TextWord> words    = new List <TextWord>();
            HighlightColor  markNext = null;

            currentOffset = 0;
            currentLength = 0;
            UpdateSpanStateVariables();

            int currentLineLength = currentLine.Length;
            int currentLineOffset = currentLine.Offset;

            for (int i = 0; i < currentLineLength; ++i)
            {
                char ch = document.GetCharAt(currentLineOffset + i);
                switch (ch)
                {
                case '\n':
                case '\r':
                    PushCurWord(document, ref markNext, words);
                    ++currentOffset;
                    break;

                case ' ':
                    PushCurWord(document, ref markNext, words);
                    if (activeSpan != null && activeSpan.Color.HasBackground)
                    {
                        words.Add(new TextWord.SpaceTextWord(activeSpan.Color));
                    }
                    else
                    {
                        words.Add(TextWord.Space);
                    }
                    ++currentOffset;
                    break;

                case '\t':
                    PushCurWord(document, ref markNext, words);
                    if (activeSpan != null && activeSpan.Color.HasBackground)
                    {
                        words.Add(new TextWord.TabTextWord(activeSpan.Color));
                    }
                    else
                    {
                        words.Add(TextWord.Tab);
                    }
                    ++currentOffset;
                    break;

                default:
                {
                    // handle escape characters
                    char escapeCharacter = '\0';
                    if (activeSpan != null && activeSpan.EscapeCharacter != '\0')
                    {
                        escapeCharacter = activeSpan.EscapeCharacter;
                    }
                    else if (activeRuleSet != null)
                    {
                        escapeCharacter = activeRuleSet.EscapeCharacter;
                    }
                    if (escapeCharacter != '\0' && escapeCharacter == ch)
                    {
                        // we found the escape character
                        if (activeSpan != null && activeSpan.End != null && activeSpan.End.Length == 1 &&
                            escapeCharacter == activeSpan.End[0])
                        {
                            // the escape character is a end-doubling escape character
                            // it may count as escape only when the next character is the escape, too
                            if (i + 1 < currentLineLength)
                            {
                                if (document.GetCharAt(currentLineOffset + i + 1) == escapeCharacter)
                                {
                                    currentLength += 2;
                                    PushCurWord(document, ref markNext, words);
                                    ++i;
                                    continue;
                                }
                            }
                        }
                        else
                        {
                            // this is a normal \-style escape
                            ++currentLength;
                            if (i + 1 < currentLineLength)
                            {
                                ++currentLength;
                            }
                            PushCurWord(document, ref markNext, words);
                            ++i;
                            continue;
                        }
                    }

                    // highlight digits
                    if (!inSpan && (Char.IsDigit(ch) || (ch == '.' && i + 1 < currentLineLength && Char.IsDigit(document.GetCharAt(currentLineOffset + i + 1)))) && currentLength == 0)
                    {
                        bool ishex           = false;
                        bool isfloatingpoint = false;

                        if (ch == '0' && i + 1 < currentLineLength && Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1)) == 'X')   // hex digits
                        {
                            const string hex = "0123456789ABCDEF";
                            ++currentLength;
                            ++i; // skip 'x'
                            ++currentLength;
                            ishex = true;
                            while (i + 1 < currentLineLength && hex.IndexOf(Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1))) != -1)
                            {
                                ++i;
                                ++currentLength;
                            }
                        }
                        else
                        {
                            ++currentLength;
                            while (i + 1 < currentLineLength && Char.IsDigit(document.GetCharAt(currentLineOffset + i + 1)))
                            {
                                ++i;
                                ++currentLength;
                            }
                        }
                        if (!ishex && i + 1 < currentLineLength && document.GetCharAt(currentLineOffset + i + 1) == '.')
                        {
                            isfloatingpoint = true;
                            ++i;
                            ++currentLength;
                            while (i + 1 < currentLineLength && Char.IsDigit(document.GetCharAt(currentLineOffset + i + 1)))
                            {
                                ++i;
                                ++currentLength;
                            }
                        }

                        if (i + 1 < currentLineLength && Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1)) == 'E')
                        {
                            isfloatingpoint = true;
                            ++i;
                            ++currentLength;
                            if (i + 1 < currentLineLength && (document.GetCharAt(currentLineOffset + i + 1) == '+' || document.GetCharAt(currentLine.Offset + i + 1) == '-'))
                            {
                                ++i;
                                ++currentLength;
                            }
                            while (i + 1 < currentLine.Length && Char.IsDigit(document.GetCharAt(currentLineOffset + i + 1)))
                            {
                                ++i;
                                ++currentLength;
                            }
                        }

                        if (i + 1 < currentLine.Length)
                        {
                            char nextch = Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1));
                            if (nextch == 'F' || nextch == 'M' || nextch == 'D')
                            {
                                isfloatingpoint = true;
                                ++i;
                                ++currentLength;
                            }
                        }

                        if (!isfloatingpoint)
                        {
                            bool isunsigned = false;
                            if (i + 1 < currentLineLength && Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1)) == 'U')
                            {
                                ++i;
                                ++currentLength;
                                isunsigned = true;
                            }
                            if (i + 1 < currentLineLength && Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1)) == 'L')
                            {
                                ++i;
                                ++currentLength;
                                if (!isunsigned && i + 1 < currentLineLength && Char.ToUpper(document.GetCharAt(currentLineOffset + i + 1)) == 'U')
                                {
                                    ++i;
                                    ++currentLength;
                                }
                            }
                        }

                        words.Add(new TextWord(document, currentLine, currentOffset, currentLength, DigitColor, false));
                        currentOffset += currentLength;
                        currentLength  = 0;
                        continue;
                    }

                    // Check for SPAN ENDs
                    if (inSpan)
                    {
                        if (activeSpan.End != null && activeSpan.End.Length > 0)
                        {
                            if (MatchExpr(currentLine, activeSpan.End, i, document, activeSpan.IgnoreCase))
                            {
                                PushCurWord(document, ref markNext, words);
                                string regex = GetRegString(currentLine, activeSpan.End, i, document);
                                currentLength += regex.Length;
                                words.Add(new TextWord(document, currentLine, currentOffset, currentLength, activeSpan.EndColor, false));
                                currentOffset += currentLength;
                                currentLength  = 0;
                                i             += regex.Length - 1;
                                currentSpanStack.Pop();
                                UpdateSpanStateVariables();
                                continue;
                            }
                        }
                    }

                    // check for SPAN BEGIN
                    if (activeRuleSet != null)
                    {
                        foreach (Span span in activeRuleSet.Spans)
                        {
                            if ((!span.IsBeginSingleWord || currentLength == 0) &&
                                (!span.IsBeginStartOfLine.HasValue || span.IsBeginStartOfLine.Value == (currentLength == 0 && words.TrueForAll(delegate(TextWord textWord)
                                {
                                    return(textWord.Type != TextWordType.Word);
                                }))) &&
                                MatchExpr(currentLine, span.Begin, i, document, activeRuleSet.IgnoreCase))
                            {
                                PushCurWord(document, ref markNext, words);
                                string regex = GetRegString(currentLine, span.Begin, i, document);

                                if (!OverrideSpan(regex, document, words, span, ref i))
                                {
                                    currentLength += regex.Length;
                                    words.Add(new TextWord(document, currentLine, currentOffset, currentLength, span.BeginColor, false));
                                    currentOffset += currentLength;
                                    currentLength  = 0;

                                    i += regex.Length - 1;
                                    if (currentSpanStack == null)
                                    {
                                        currentSpanStack = new SpanStack();
                                    }
                                    currentSpanStack.Push(span);
                                    span.IgnoreCase = activeRuleSet.IgnoreCase;

                                    UpdateSpanStateVariables();
                                }

                                goto skip;
                            }
                        }
                    }

                    // check if the char is a delimiter
                    if (activeRuleSet != null && (int)ch < 256 && activeRuleSet.Delimiters[(int)ch])
                    {
                        PushCurWord(document, ref markNext, words);
                        if (currentOffset + currentLength + 1 < currentLine.Length)
                        {
                            ++currentLength;
                            PushCurWord(document, ref markNext, words);
                            goto skip;
                        }
                    }

                    ++currentLength;
                    skip : continue;
                }
                }
            }

            PushCurWord(document, ref markNext, words);

            OnParsedLine(document, currentLine, words);

            return(words);
        }
Exemplo n.º 5
0
        bool MarkTokensInLine(IDocument document, int lineNumber, ref bool spanChanged)
        {
            currentLineNumber = lineNumber;
            bool        processNextLine = false;
            LineSegment previousLine    = (lineNumber > 0 ? document.GetLineSegment(lineNumber - 1) : null);

            currentSpanStack = ((previousLine != null && previousLine.HighlightSpanStack != null) ? previousLine.HighlightSpanStack.Clone() : null);
            if (currentSpanStack != null)
            {
                while (!currentSpanStack.IsEmpty && currentSpanStack.Peek().StopEOL)
                {
                    currentSpanStack.Pop();
                }
                if (currentSpanStack.IsEmpty)
                {
                    currentSpanStack = null;
                }
            }

            currentLine = (LineSegment)document.LineSegmentCollection[lineNumber];

            if (currentLine.Length == -1)   // happens when buffer is empty !
            {
                return(false);
            }

            List <TextWord> words = ParseLine(document);

            if (currentSpanStack != null && currentSpanStack.IsEmpty)
            {
                currentSpanStack = null;
            }

            // Check if the span state has changed, if so we must re-render the next line
            // This check may seem utterly complicated but I didn't want to introduce any function calls
            // or allocations here for perf reasons.
            if (currentLine.HighlightSpanStack != currentSpanStack)
            {
                if (currentLine.HighlightSpanStack == null)
                {
                    processNextLine = false;
                    foreach (Span sp in currentSpanStack)
                    {
                        if (!sp.StopEOL)
                        {
                            spanChanged     = true;
                            processNextLine = true;
                            break;
                        }
                    }
                }
                else if (currentSpanStack == null)
                {
                    processNextLine = false;
                    foreach (Span sp in currentLine.HighlightSpanStack)
                    {
                        if (!sp.StopEOL)
                        {
                            spanChanged     = true;
                            processNextLine = true;
                            break;
                        }
                    }
                }
                else
                {
                    SpanStack.Enumerator e1 = currentSpanStack.GetEnumerator();
                    SpanStack.Enumerator e2 = currentLine.HighlightSpanStack.GetEnumerator();
                    bool done = false;
                    while (!done)
                    {
                        bool blockSpanIn1 = false;
                        while (e1.MoveNext())
                        {
                            if (!((Span)e1.Current).StopEOL)
                            {
                                blockSpanIn1 = true;
                                break;
                            }
                        }
                        bool blockSpanIn2 = false;
                        while (e2.MoveNext())
                        {
                            if (!((Span)e2.Current).StopEOL)
                            {
                                blockSpanIn2 = true;
                                break;
                            }
                        }
                        if (blockSpanIn1 || blockSpanIn2)
                        {
                            if (blockSpanIn1 && blockSpanIn2)
                            {
                                if (e1.Current != e2.Current)
                                {
                                    done            = true;
                                    processNextLine = true;
                                    spanChanged     = true;
                                }
                            }
                            else
                            {
                                spanChanged     = true;
                                done            = true;
                                processNextLine = true;
                            }
                        }
                        else
                        {
                            done            = true;
                            processNextLine = false;
                        }
                    }
                }
            }
            else
            {
                processNextLine = false;
            }

            //// Alex: remove old words
            if (currentLine.Words != null)
            {
                currentLine.Words.Clear();
            }
            currentLine.Words = words;
            currentLine.HighlightSpanStack = (currentSpanStack != null && !currentSpanStack.IsEmpty) ? currentSpanStack : null;

            return(processNextLine);
        }
Exemplo n.º 6
0
        private bool MarkTokensInLine(IDocument document, int lineNumber, ref bool spanChanged)
        {
            currentLineNumber = lineNumber;
            var processNextLine = false;
            var previousLine    = lineNumber > 0 ? document.GetLineSegment(lineNumber - 1) : null;

            currentSpanStack = previousLine != null && previousLine.HighlightSpanStack != null?previousLine.HighlightSpanStack.Clone() : null;

            if (currentSpanStack != null)
            {
                while (!currentSpanStack.IsEmpty && currentSpanStack.Peek().StopEOL)
                {
                    currentSpanStack.Pop();
                }
                if (currentSpanStack.IsEmpty)
                {
                    currentSpanStack = null;
                }
            }

            currentLine = document.LineSegmentCollection[lineNumber];

            if (currentLine.Length == -1)
            {
                return(false);
            }

            var words = ParseLine(document);

            if (currentSpanStack != null && currentSpanStack.IsEmpty)
            {
                currentSpanStack = null;
            }

            // Check if the span state has changed, if so we must re-render the next line
            // This check may seem utterly complicated but I didn't want to introduce any function calls
            // or allocations here for perf reasons.
            if (currentLine.HighlightSpanStack != currentSpanStack)
            {
                if (currentLine.HighlightSpanStack == null)
                {
                    foreach (var sp in currentSpanStack)
                    {
                        if (!sp.StopEOL)
                        {
                            spanChanged     = true;
                            processNextLine = true;
                            break;
                        }
                    }
                }
                else if (currentSpanStack == null)
                {
                    foreach (var sp in currentLine.HighlightSpanStack)
                    {
                        if (!sp.StopEOL)
                        {
                            spanChanged     = true;
                            processNextLine = true;
                            break;
                        }
                    }
                }
                else
                {
                    var e1   = currentSpanStack.GetEnumerator();
                    var e2   = currentLine.HighlightSpanStack.GetEnumerator();
                    var done = false;
                    while (!done)
                    {
                        var blockSpanIn1 = false;
                        while (e1.MoveNext())
                        {
                            if (!e1.Current.StopEOL)
                            {
                                blockSpanIn1 = true;
                                break;
                            }
                        }

                        var blockSpanIn2 = false;
                        while (e2.MoveNext())
                        {
                            if (!e2.Current.StopEOL)
                            {
                                blockSpanIn2 = true;
                                break;
                            }
                        }

                        if (blockSpanIn1 || blockSpanIn2)
                        {
                            if (blockSpanIn1 && blockSpanIn2)
                            {
                                if (e1.Current != e2.Current)
                                {
                                    done            = true;
                                    processNextLine = true;
                                    spanChanged     = true;
                                }
                            }
                            else
                            {
                                spanChanged     = true;
                                done            = true;
                                processNextLine = true;
                            }
                        }
                        else
                        {
                            done = true;
                        }
                    }
                }
            }

            //// Alex: remove old words
            currentLine.Words?.Clear();
            currentLine.Words = words;
            currentLine.HighlightSpanStack = currentSpanStack != null && !currentSpanStack.IsEmpty ? currentSpanStack : null;

            return(processNextLine);
        }
        List <TextWord> ParseLine(IDocument document)
        {
            List <TextWord> words    = new List <TextWord>();
            HighlightColor  markNext = null;

            currentOffset = 0;
            currentLength = 0;
            UpdateSpanStateVariables();

            for (int i = 0; i < currentLine.Length; ++i)
            {
                char ch = document.GetCharAt(currentLine.Offset + i);
                switch (ch)
                {
                case '\n':
                case '\r':
                    PushCurWord(document, ref markNext, words);
                    ++currentOffset;
                    break;

                case ' ':
                    PushCurWord(document, ref markNext, words);
                    if (activeSpan != null && activeSpan.Color.HasBackground)
                    {
                        words.Add(new TextWord.SpaceTextWord(activeSpan.Color));
                    }
                    else
                    {
                        words.Add(TextWord.Space);
                    }
                    ++currentOffset;
                    break;

                case '\t':
                    PushCurWord(document, ref markNext, words);
                    if (activeSpan != null && activeSpan.Color.HasBackground)
                    {
                        words.Add(new TextWord.TabTextWord(activeSpan.Color));
                    }
                    else
                    {
                        words.Add(TextWord.Tab);
                    }
                    ++currentOffset;
                    break;

                case '\\': // handle escape chars
                    if ((activeRuleSet != null && activeRuleSet.NoEscapeSequences) ||
                        (activeSpan != null && activeSpan.NoEscapeSequences))
                    {
                        goto default;
                    }
                    ++currentLength;
                    if (i + 1 < currentLine.Length)
                    {
                        ++currentLength;
                    }
                    PushCurWord(document, ref markNext, words);
                    ++i;
                    continue;

                default: {
                    // highlight digits
                    if (!inSpan && (Char.IsDigit(ch) || (ch == '.' && i + 1 < currentLine.Length && Char.IsDigit(document.GetCharAt(currentLine.Offset + i + 1)))) && currentLength == 0)
                    {
                        bool ishex           = false;
                        bool isfloatingpoint = false;

                        if (ch == '0' && i + 1 < currentLine.Length && Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1)) == 'X') // hex digits
                        {
                            const string hex = "0123456789ABCDEF";
                            ++currentLength;
                            ++i; // skip 'x'
                            ++currentLength;
                            ishex = true;
                            while (i + 1 < currentLine.Length && hex.IndexOf(Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1))) != -1)
                            {
                                ++i;
                                ++currentLength;
                            }
                        }
                        else
                        {
                            ++currentLength;
                            while (i + 1 < currentLine.Length && Char.IsDigit(document.GetCharAt(currentLine.Offset + i + 1)))
                            {
                                ++i;
                                ++currentLength;
                            }
                        }
                        if (!ishex && i + 1 < currentLine.Length && document.GetCharAt(currentLine.Offset + i + 1) == '.')
                        {
                            isfloatingpoint = true;
                            ++i;
                            ++currentLength;
                            while (i + 1 < currentLine.Length && Char.IsDigit(document.GetCharAt(currentLine.Offset + i + 1)))
                            {
                                ++i;
                                ++currentLength;
                            }
                        }

                        if (i + 1 < currentLine.Length && Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1)) == 'E')
                        {
                            isfloatingpoint = true;
                            ++i;
                            ++currentLength;
                            if (i + 1 < currentLine.Length && (document.GetCharAt(currentLine.Offset + i + 1) == '+' || document.GetCharAt(currentLine.Offset + i + 1) == '-'))
                            {
                                ++i;
                                ++currentLength;
                            }
                            while (i + 1 < currentLine.Length && Char.IsDigit(document.GetCharAt(currentLine.Offset + i + 1)))
                            {
                                ++i;
                                ++currentLength;
                            }
                        }

                        if (i + 1 < currentLine.Length)
                        {
                            char nextch = Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1));
                            if (nextch == 'F' || nextch == 'M' || nextch == 'D')
                            {
                                isfloatingpoint = true;
                                ++i;
                                ++currentLength;
                            }
                        }

                        if (!isfloatingpoint)
                        {
                            bool isunsigned = false;
                            if (i + 1 < currentLine.Length && Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1)) == 'U')
                            {
                                ++i;
                                ++currentLength;
                                isunsigned = true;
                            }
                            if (i + 1 < currentLine.Length && Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1)) == 'L')
                            {
                                ++i;
                                ++currentLength;
                                if (!isunsigned && i + 1 < currentLine.Length && Char.ToUpperInvariant(document.GetCharAt(currentLine.Offset + i + 1)) == 'U')
                                {
                                    ++i;
                                    ++currentLength;
                                }
                            }
                        }

                        words.Add(new TextWord(document, currentLine, currentOffset, currentLength, DigitColor, false));
                        currentOffset += currentLength;
                        currentLength  = 0;
                        continue;
                    }

                    // Check for SPAN ENDs
                    if (inSpan)
                    {
                        if (activeSpan.End != null && !activeSpan.End.Equals(""))
                        {
                            if (currentLine.MatchExpr(activeSpan.End, i, document, activeSpan.IgnoreCase))
                            {
                                PushCurWord(document, ref markNext, words);
                                string regex = currentLine.GetRegString(activeSpan.End, i, document);
                                currentLength += regex.Length;
                                words.Add(new TextWord(document, currentLine, currentOffset, currentLength, activeSpan.EndColor, false));
                                currentOffset += currentLength;
                                currentLength  = 0;
                                i             += regex.Length - 1;
                                currentSpanStack.Pop();
                                UpdateSpanStateVariables();
                                continue;
                            }
                        }
                    }

                    // check for SPAN BEGIN
                    if (activeRuleSet != null)
                    {
                        foreach (Span span in activeRuleSet.Spans)
                        {
                            if ((!span.IsBeginSingleWord || currentLength == 0) && currentLine.MatchExpr(span.Begin, i, document, activeRuleSet.IgnoreCase))
                            {
                                PushCurWord(document, ref markNext, words);
                                string regex = currentLine.GetRegString(span.Begin, i, document);

                                if (!OverrideSpan(regex, document, words, span, ref i))
                                {
                                    currentLength += regex.Length;
                                    words.Add(new TextWord(document, currentLine, currentOffset, currentLength, span.BeginColor, false));
                                    currentOffset += currentLength;
                                    currentLength  = 0;

                                    i += regex.Length - 1;
                                    if (currentSpanStack == null)
                                    {
                                        currentSpanStack = new SpanStack();
                                    }
                                    currentSpanStack.Push(span);
                                    span.IgnoreCase = activeRuleSet.IgnoreCase;

                                    UpdateSpanStateVariables();
                                }

                                goto skip;
                            }
                        }
                    }

                    // check if the char is a delimiter
                    if (activeRuleSet != null && (int)ch < 256 && activeRuleSet.Delimiters[(int)ch])
                    {
                        PushCurWord(document, ref markNext, words);
                        if (currentOffset + currentLength + 1 < currentLine.Length)
                        {
                            ++currentLength;
                            PushCurWord(document, ref markNext, words);
                            goto skip;
                        }
                    }

                    ++currentLength;
                    skip : continue;
                }
                }
            }

            PushCurWord(document, ref markNext, words);

            OnParsedLine(document, currentLine, words);

            return(words);
        }
        private List <TextWord> ParseLine(IDocument document)
        {
            List <TextWord> textWords      = new List <TextWord>();
            HighlightColor  highlightColor = null;

            this.currentOffset = 0;
            this.currentLength = 0;
            this.UpdateSpanStateVariables();
            int length = this.currentLine.Length;
            int offset = this.currentLine.Offset;

            for (int i = 0; i < length; i++)
            {
                char charAt = document.GetCharAt(offset + i);
                char chr    = charAt;
                switch (chr)
                {
                case '\t':
                {
                    this.PushCurWord(document, ref highlightColor, textWords);
                    if (this.activeSpan == null || !this.activeSpan.Color.HasBackground)
                    {
                        textWords.Add(TextWord.Tab);
                    }
                    else
                    {
                        textWords.Add(new TextWord.TabTextWord(this.activeSpan.Color));
                    }
                    this.currentOffset++;
                    break;
                }

                case '\n':
                case '\r':
                {
                    this.PushCurWord(document, ref highlightColor, textWords);
                    this.currentOffset++;
                    break;
                }

                case '\v':
                case '\f':
                {
                    char escapeCharacter = '\0';
                    if (this.activeSpan != null && this.activeSpan.EscapeCharacter != 0)
                    {
                        escapeCharacter = this.activeSpan.EscapeCharacter;
                    }
                    else if (this.activeRuleSet != null)
                    {
                        escapeCharacter = this.activeRuleSet.EscapeCharacter;
                    }
                    if (escapeCharacter != 0 && escapeCharacter == charAt)
                    {
                        if (this.activeSpan == null || this.activeSpan.End == null || (int)this.activeSpan.End.Length != 1 || escapeCharacter != this.activeSpan.End[0])
                        {
                            this.currentLength++;
                            if (i + 1 < length)
                            {
                                this.currentLength++;
                            }
                            this.PushCurWord(document, ref highlightColor, textWords);
                            i++;
                            break;
                        }
                        else if (i + 1 < length && document.GetCharAt(offset + i + 1) == escapeCharacter)
                        {
                            this.currentLength += 2;
                            this.PushCurWord(document, ref highlightColor, textWords);
                            i++;
                            break;
                        }
                    }
                    if (!this.inSpan && (char.IsDigit(charAt) || charAt == '.' && i + 1 < length && char.IsDigit(document.GetCharAt(offset + i + 1))) && this.currentLength == 0)
                    {
                        bool flag  = false;
                        bool flag1 = false;
                        if (charAt != '0' || i + 1 >= length || char.ToUpper(document.GetCharAt(offset + i + 1)) != 'X')
                        {
                            this.currentLength++;
                            while (i + 1 < length && char.IsDigit(document.GetCharAt(offset + i + 1)))
                            {
                                i++;
                                this.currentLength++;
                            }
                        }
                        else
                        {
                            this.currentLength++;
                            i++;
                            this.currentLength++;
                            flag = true;
                            while (i + 1 < length)
                            {
                                if ("0123456789ABCDEF".IndexOf(char.ToUpper(document.GetCharAt(offset + i + 1))) != -1)
                                {
                                    i++;
                                    this.currentLength++;
                                }
                                else
                                {
                                    break;
                                }
                            }
                        }
                        if (!flag && i + 1 < length && document.GetCharAt(offset + i + 1) == '.')
                        {
                            flag1 = true;
                            i++;
                            this.currentLength++;
                            while (i + 1 < length && char.IsDigit(document.GetCharAt(offset + i + 1)))
                            {
                                i++;
                                this.currentLength++;
                            }
                        }
                        if (i + 1 < length && char.ToUpper(document.GetCharAt(offset + i + 1)) == 'E')
                        {
                            flag1 = true;
                            i++;
                            this.currentLength++;
                            if (i + 1 < length && (document.GetCharAt(offset + i + 1) == '+' || document.GetCharAt(this.currentLine.Offset + i + 1) == '-'))
                            {
                                i++;
                                this.currentLength++;
                            }
                            while (i + 1 < this.currentLine.Length && char.IsDigit(document.GetCharAt(offset + i + 1)))
                            {
                                i++;
                                this.currentLength++;
                            }
                        }
                        if (i + 1 < this.currentLine.Length)
                        {
                            char upper = char.ToUpper(document.GetCharAt(offset + i + 1));
                            if (upper == 'F' || upper == 'M' || upper == 'D')
                            {
                                flag1 = true;
                                i++;
                                this.currentLength++;
                            }
                        }
                        if (!flag1)
                        {
                            bool flag2 = false;
                            if (i + 1 < length && char.ToUpper(document.GetCharAt(offset + i + 1)) == 'U')
                            {
                                i++;
                                this.currentLength++;
                                flag2 = true;
                            }
                            if (i + 1 < length && char.ToUpper(document.GetCharAt(offset + i + 1)) == 'L')
                            {
                                i++;
                                this.currentLength++;
                                if (!flag2 && i + 1 < length && char.ToUpper(document.GetCharAt(offset + i + 1)) == 'U')
                                {
                                    i++;
                                    this.currentLength++;
                                }
                            }
                        }
                        textWords.Add(new TextWord(document, this.currentLine, this.currentOffset, this.currentLength, this.DigitColor, false));
                        this.currentOffset += this.currentLength;
                        this.currentLength  = 0;
                        break;
                    }
                    else if (!this.inSpan || this.activeSpan.End == null || (int)this.activeSpan.End.Length <= 0 || !DefaultHighlightingStrategy.MatchExpr(this.currentLine, this.activeSpan.End, i, document, this.activeSpan.IgnoreCase))
                    {
                        if (this.activeRuleSet != null)
                        {
                            foreach (Span span in this.activeRuleSet.Spans)
                            {
                                if (span.IsBeginSingleWord && this.currentLength != 0)
                                {
                                    continue;
                                }
                                if (span.IsBeginStartOfLine.HasValue)
                                {
                                    if (span.IsBeginStartOfLine.Value != (this.currentLength != 0 ? false : textWords.TrueForAll((TextWord textWord) => textWord.Type != TextWordType.Word)))
                                    {
                                        continue;
                                    }
                                }
                                if (!DefaultHighlightingStrategy.MatchExpr(this.currentLine, span.Begin, i, document, this.activeRuleSet.IgnoreCase))
                                {
                                    continue;
                                }
                                this.PushCurWord(document, ref highlightColor, textWords);
                                string regString = DefaultHighlightingStrategy.GetRegString(this.currentLine, span.Begin, i, document);
                                if (!this.OverrideSpan(regString, document, textWords, span, ref i))
                                {
                                    this.currentLength += regString.Length;
                                    textWords.Add(new TextWord(document, this.currentLine, this.currentOffset, this.currentLength, span.BeginColor, false));
                                    this.currentOffset += this.currentLength;
                                    this.currentLength  = 0;
                                    i = i + (regString.Length - 1);
                                    if (this.currentSpanStack == null)
                                    {
                                        this.currentSpanStack = new SpanStack();
                                    }
                                    this.currentSpanStack.Push(span);
                                    span.IgnoreCase = this.activeRuleSet.IgnoreCase;
                                    this.UpdateSpanStateVariables();
                                }
                                goto label0;
                            }
                        }
                        if (this.activeRuleSet != null && charAt < 'Ā' && this.activeRuleSet.Delimiters[charAt])
                        {
                            this.PushCurWord(document, ref highlightColor, textWords);
                            if (this.currentOffset + this.currentLength + 1 < this.currentLine.Length)
                            {
                                this.currentLength++;
                                this.PushCurWord(document, ref highlightColor, textWords);
                                break;
                            }
                        }
                        this.currentLength++;
                        break;
                    }
                    else
                    {
                        this.PushCurWord(document, ref highlightColor, textWords);
                        string str = DefaultHighlightingStrategy.GetRegString(this.currentLine, this.activeSpan.End, i, document);
                        this.currentLength += str.Length;
                        textWords.Add(new TextWord(document, this.currentLine, this.currentOffset, this.currentLength, this.activeSpan.EndColor, false));
                        this.currentOffset += this.currentLength;
                        this.currentLength  = 0;
                        i = i + (str.Length - 1);
                        this.currentSpanStack.Pop();
                        this.UpdateSpanStateVariables();
                        break;
                    }
                }

                default:
                {
                    if (chr == ' ')
                    {
                        this.PushCurWord(document, ref highlightColor, textWords);
                        if (this.activeSpan == null || !this.activeSpan.Color.HasBackground)
                        {
                            textWords.Add(TextWord.Space);
                        }
                        else
                        {
                            textWords.Add(new TextWord.SpaceTextWord(this.activeSpan.Color));
                        }
                        this.currentOffset++;
                        break;
                    }
                    else
                    {
                        goto case '\f';
                    }
                }
                }
                label0 :;
            }
            this.PushCurWord(document, ref highlightColor, textWords);
            this.OnParsedLine(document, this.currentLine, textWords);
            return(textWords);
        }
        private bool MarkTokensInLine(IDocument document, int lineNumber, ref bool spanChanged)
        {
            LineSegment lineSegment;
            SpanStack   spanStacks;
            SpanStack   spanStacks1;

            this.currentLineNumber = lineNumber;
            bool flag = false;

            if (lineNumber > 0)
            {
                lineSegment = document.GetLineSegment(lineNumber - 1);
            }
            else
            {
                lineSegment = null;
            }
            LineSegment lineSegment1 = lineSegment;

            if (lineSegment1 == null || lineSegment1.HighlightSpanStack == null)
            {
                spanStacks = null;
            }
            else
            {
                spanStacks = lineSegment1.HighlightSpanStack.Clone();
            }
            this.currentSpanStack = spanStacks;
            if (this.currentSpanStack != null)
            {
                while (!this.currentSpanStack.IsEmpty && this.currentSpanStack.Peek().StopEOL)
                {
                    this.currentSpanStack.Pop();
                }
                if (this.currentSpanStack.IsEmpty)
                {
                    this.currentSpanStack = null;
                }
            }
            this.currentLine = document.LineSegmentCollection[lineNumber];
            if (this.currentLine.Length == -1)
            {
                return(false);
            }
            List <TextWord> textWords = this.ParseLine(document);

            if (this.currentSpanStack != null && this.currentSpanStack.IsEmpty)
            {
                this.currentSpanStack = null;
            }
            if (this.currentLine.HighlightSpanStack == this.currentSpanStack)
            {
                flag = false;
            }
            else if (this.currentLine.HighlightSpanStack == null)
            {
                flag = false;
                foreach (Span span in this.currentSpanStack)
                {
                    if (span.StopEOL)
                    {
                        continue;
                    }
                    spanChanged = true;
                    flag        = true;
                    break;
                }
            }
            else if (this.currentSpanStack != null)
            {
                SpanStack.Enumerator enumerator  = this.currentSpanStack.GetEnumerator();
                SpanStack.Enumerator enumerator1 = this.currentLine.HighlightSpanStack.GetEnumerator();
                bool flag1 = false;
                while (!flag1)
                {
                    bool flag2 = false;
                    while (enumerator.MoveNext())
                    {
                        if (enumerator.Current.StopEOL)
                        {
                            continue;
                        }
                        flag2 = true;
                        break;
                    }
                    bool flag3 = false;
                    while (enumerator1.MoveNext())
                    {
                        if (enumerator1.Current.StopEOL)
                        {
                            continue;
                        }
                        flag3 = true;
                        break;
                    }
                    if (!flag2 && !flag3)
                    {
                        flag1 = true;
                        flag  = false;
                    }
                    else if (!flag2 || !flag3)
                    {
                        spanChanged = true;
                        flag1       = true;
                        flag        = true;
                    }
                    else
                    {
                        if (enumerator.Current == enumerator1.Current)
                        {
                            continue;
                        }
                        flag1       = true;
                        flag        = true;
                        spanChanged = true;
                    }
                }
            }
            else
            {
                flag = false;
                foreach (Span highlightSpanStack in this.currentLine.HighlightSpanStack)
                {
                    if (highlightSpanStack.StopEOL)
                    {
                        continue;
                    }
                    spanChanged = true;
                    flag        = true;
                    break;
                }
            }
            if (this.currentLine.Words != null)
            {
                this.currentLine.Words.Clear();
            }
            this.currentLine.Words = textWords;
            LineSegment lineSegment2 = this.currentLine;

            if (this.currentSpanStack == null || this.currentSpanStack.IsEmpty)
            {
                spanStacks1 = null;
            }
            else
            {
                spanStacks1 = this.currentSpanStack;
            }
            lineSegment2.HighlightSpanStack = spanStacks1;
            return(flag);
        }
        public virtual void MarkTokens(IDocument document)
        {
            LineSegment lineSegment;
            SpanStack   spanStacks;
            SpanStack   spanStacks1;

            if (this.Rules.Count == 0)
            {
                return;
            }
            for (int i = 0; i < document.TotalNumberOfLines; i++)
            {
                if (i > 0)
                {
                    lineSegment = document.GetLineSegment(i - 1);
                }
                else
                {
                    lineSegment = null;
                }
                LineSegment lineSegment1 = lineSegment;
                if (i >= document.LineSegmentCollection.Count)
                {
                    break;
                }
                if (lineSegment1 == null || lineSegment1.HighlightSpanStack == null)
                {
                    spanStacks = null;
                }
                else
                {
                    spanStacks = lineSegment1.HighlightSpanStack.Clone();
                }
                this.currentSpanStack = spanStacks;
                if (this.currentSpanStack != null)
                {
                    while (!this.currentSpanStack.IsEmpty && this.currentSpanStack.Peek().StopEOL)
                    {
                        this.currentSpanStack.Pop();
                    }
                    if (this.currentSpanStack.IsEmpty)
                    {
                        this.currentSpanStack = null;
                    }
                }
                this.currentLine = document.LineSegmentCollection[i];
                if (this.currentLine.Length == -1)
                {
                    return;
                }
                this.currentLineNumber = i;
                List <TextWord> textWords = this.ParseLine(document);
                if (this.currentLine.Words != null)
                {
                    this.currentLine.Words.Clear();
                }
                this.currentLine.Words = textWords;
                LineSegment lineSegment2 = this.currentLine;
                if (this.currentSpanStack == null || this.currentSpanStack.IsEmpty)
                {
                    spanStacks1 = null;
                }
                else
                {
                    spanStacks1 = this.currentSpanStack;
                }
                lineSegment2.HighlightSpanStack = spanStacks1;
            }
            document.RequestUpdate(new TextAreaUpdate(TextAreaUpdateType.WholeTextArea));
            document.CommitUpdate();
            this.currentLine = null;
        }