示例#1
0
        private void ConsumeIgnored(ReadOnlyMemory <char> source)
        {
            while (CurrentPosition < source.Length)
            {
                if (IgnoreWhiteSpace)
                {
                    var currentCharacter = source.At(CurrentPosition);
                    if (WhiteSpaces.Contains(currentCharacter))
                    {
                        CurrentPosition++;
                        CurrentColumn++;
                        continue;
                    }
                }

                if (IgnoreEOL)
                {
                    var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
                    if (eol != EOLType.No)
                    {
                        CurrentPosition += eol == EOLType.Windows ? 2 : 1;
                        CurrentColumn    = 0;
                        CurrentLine++;
                        continue;
                    }
                }

                break;
            }
        }
示例#2
0
        private void ConsumeIgnored(ReadOnlyMemory <char> source, LexerPosition position)
        {
            while (position.Index < source.Length)
            {
                if (IgnoreWhiteSpace)
                {
                    var currentCharacter = source.At(position.Index);
                    if (WhiteSpaces.Contains(currentCharacter))
                    {
                        position.Index++;
                        position.Column++;
                        continue;
                    }
                }

                if (IgnoreEOL)
                {
                    var eol = EOLManager.IsEndOfLine(source, position.Index);
                    if (eol != EOLType.No)
                    {
                        position.Index += eol == EOLType.Windows ? 2 : 1;
                        position.Column = 0;
                        position.Line++;
                        continue;
                    }
                }

                break;
            }
        }
示例#3
0
        public static EOLType IsEndOfLine(ReadOnlyMemory <char> value, int position)
        {
            var end = EOLType.No;
            var n   = value.At(position);

            if (n == '\n')
            {
                end = EOLType.Nix;
            }
            else if (n == '\r')
            {
                if (value.At(position + 1) == '\n')
                {
                    end = EOLType.Windows;
                }
                else
                {
                    end = EOLType.Mac;
                }
            }

            return(end);
        }
        public static bool CheckDate(ReadOnlyMemory <char> value)
        {
            var ok = false;

            if (value.Length == 6)
            {
                ok = char.IsDigit(value.At(0));
                ok = ok && char.IsDigit(value.At(1));
                ok = ok && value.At(2) == '.';
                ok = ok && char.IsDigit(value.At(3));
                ok = ok && char.IsDigit(value.At(4));
                ok = ok && value.At(5) == '.';
            }

            return(ok);
        }
示例#5
0
        public FSMMatch <N> Run(ReadOnlyMemory <char> source, LexerPosition lexerPosition)
        {
            ConsumeIgnored(source, lexerPosition);

            // End of token stream
            if (lexerPosition.Index >= source.Length)
            {
                return(new FSMMatch <N>(false));
            }

            // Make a note of where current token starts
            var position = lexerPosition.Clone();

            FSMMatch <N> result      = null;
            var          currentNode = Nodes[0];

            while (lexerPosition.Index < source.Length)
            {
                var currentCharacter = source.At(lexerPosition);
                var currentValue     = source.Slice(position.Index, lexerPosition.Index - position.Index + 1);
                currentNode = Move(currentNode, currentCharacter, currentValue);
                if (currentNode == null)
                {
                    // No more viable transitions, so exit loop
                    break;
                }

                if (currentNode.IsEnd)
                {
                    // Remember the possible match
                    result = new FSMMatch <N>(true, currentNode.Value, currentValue, position, currentNode.Id, lexerPosition, currentNode.IsLineEnding);
                }

                lexerPosition.Index++;
                lexerPosition.Column++;
            }

            if (result != null)
            {
                // Backtrack
                var length = result.Result.Value.Length;
                lexerPosition.Index  = result.Result.Position.Index + length;
                lexerPosition.Column = result.Result.Position.Column + length;

                if (HasCallback(result.NodeId))
                {
                    result = Callbacks[result.NodeId](result);
                }

                return(result);
            }

            if (lexerPosition.Index >= source.Length)
            {
                // Failed on last character, so need to backtrack
                lexerPosition.Index  -= 1;
                lexerPosition.Column -= 1;
            }

            var errorChar = source.Slice(lexerPosition.Index, 1);
            var ko        = new FSMMatch <N>(false, default(N), errorChar, lexerPosition, -1, lexerPosition, false);

            return(ko);
        }
示例#6
0
文件: FSMLexer.cs 项目: magne/csly
        public FSMMatch <N> Run(ReadOnlyMemory <char> source, int start)
        {
            var value           = "";
            int tokenStartIndex = start;
            int tokenLength     = 0;
            var result          = new FSMMatch <N>(false);
            var successes       = new Stack <FSMMatch <N> >();

            CurrentPosition = start;
            var           currentNode = Nodes[0];
            var           lastNode    = 0;
            TokenPosition position    = null;

            var tokenStarted = false;


            if (CurrentPosition < source.Length)
            {
                var currentCharacter = source.At(CurrentPosition);

                while (CurrentPosition < source.Length && currentNode != null)
                {
                    currentCharacter = source.Span[CurrentPosition];

                    var consumeSkipped = true;

                    while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
                    {
                        currentCharacter = source.At(CurrentPosition);
                        if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentCharacter))
                        {
                            if (successes.Any())
                            {
                                currentNode = null;
                            }
                            else
                            {
                                currentNode = Nodes[0];
                            }
                            CurrentPosition++;
                            CurrentColumn++;
                        }
                        else
                        {
                            var eol = EOLManager.IsEndOfLine(source, CurrentPosition);

                            if (IgnoreEOL && eol != EOLType.No)
                            {
                                if (successes.Any())
                                {
                                    currentNode = null;
                                }
                                else
                                {
                                    currentNode = Nodes[0];
                                }
                                CurrentPosition += eol == EOLType.Windows ? 2 : 1;
                                CurrentColumn    = 0;
                                CurrentLine++;
                            }
                            else
                            {
                                consumeSkipped = false;
                            }
                        }
                    }


                    currentNode = Move(currentNode, currentCharacter, value);
                    if (currentNode != null)
                    {
                        lastNode = currentNode.Id;
                        value   += currentCharacter;
                        tokenLength++;

                        if (!tokenStarted)
                        {
                            tokenStarted = true;
                            position     = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
                        }

                        if (currentNode.IsEnd)
                        {
                            var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position, currentNode.Id);
                            successes.Push(resultInter);
                        }

                        if (HasAction(currentNode.Id))
                        {
                            value = Actions[currentNode.Id](value);
                        }
                        CurrentPosition++;
                        CurrentColumn++;
                    }
                    else
                    {
                        if (!successes.Any() && CurrentPosition < source.Length)
                        {
                            throw new LexerException(new LexicalError(CurrentLine, CurrentColumn,
                                                                      source.At(CurrentPosition)));
                        }
                    }
                }
            }


            if (successes.Any())
            {
                result = successes.Pop();
                if (HasCallback(result.NodeId))
                {
                    result = Callbacks[result.NodeId](result);
                }
            }

            return(result);
        }
示例#7
0
        public LexerResult <JsonToken> Tokenize(ReadOnlyMemory <char> source)
        {
            var tokens        = new List <Token <JsonToken> >();
            var position      = 0;
            var currentLine   = 1;
            var currentColumn = 1;
            var length        = source.Length;

            var currentTokenLine     = 1;
            var currentTokenColumn   = 1;
            var currentTokenPosition = 1;
            var currentValue         = "";

            var InString    = false;
            var InNum       = false;
            var InNull      = false;
            var InTrue      = false;
            var InFalse     = false;
            var NumIsDouble = false;

            int tokenStartIndex = 0;
            int tokenLength     = 0;

            Func <JsonToken, Token <JsonToken> > NewToken = tok =>
            {
                var token = new Token <JsonToken>();
                token.Position  = new TokenPosition(currentTokenPosition, currentTokenLine, currentTokenColumn);
                token.SpanValue = source.Slice(tokenStartIndex, tokenLength);
                tokenStartIndex = tokenStartIndex + tokenLength;
                token.TokenID   = tok;
                tokens.Add(token);
                currentValue = "";
                return(token);
            };


            while (position < length)
            {
                var current = source.At(position);
                if (InString)
                {
                    currentValue += current;
                    if (current == '"')
                    {
                        InString = false;

                        NewToken(JsonToken.STRING);
                        position++;
                    }
                    else
                    {
                        position++;
                    }
                }
                else if (InNum)
                {
                    if (current == '.')
                    {
                        NumIsDouble   = true;
                        currentValue += current;
                    }
                    else if (char.IsDigit(current))
                    {
                        currentValue += current;
                        var type = NumIsDouble ? JsonToken.DOUBLE : JsonToken.INT;
                        if (position == length - 1)
                        {
                            NewToken(type);
                        }
                    }
                    else
                    {
                        InNum = false;
                        var type = NumIsDouble ? JsonToken.DOUBLE : JsonToken.INT;
                        NewToken(type);
                        position--;
                    }

                    position++;
                }
                else if (InNull)
                {
                    if (current == "null"[currentValue.Length])
                    {
                        currentValue += current;
                        if (currentValue.Length == 4)
                        {
                            NewToken(JsonToken.NULL);
                            InNull = false;
                        }
                    }

                    position++;
                }
                else if (InFalse)
                {
                    if (current == "false"[currentValue.Length])
                    {
                        currentValue += current;
                        if (currentValue.Length == 5)
                        {
                            NewToken(JsonToken.BOOLEAN);
                            InFalse = false;
                        }
                        else
                        {
                            position++;
                        }
                    }
                }
                else if (InTrue)
                {
                    if (current == "true"[currentValue.Length])
                    {
                        currentValue += current;
                        if (currentValue.Length == 5)
                        {
                            NewToken(JsonToken.BOOLEAN);
                            InFalse = false;
                        }
                    }

                    position++;
                }
                else
                {
                    currentValue += current;
                    if (current == '"')
                    {
                        InString           = true;
                        currentValue      += current;
                        currentTokenColumn = currentColumn;
                        currentTokenLine   = currentLine;
                    }
                    else if (char.IsDigit(current))
                    {
                        InNum = true;
                    }
                    else if (current == 't')
                    {
                        InTrue = true;
                    }
                    else if (current == 'f')
                    {
                        InFalse = true;
                    }
                    else if (current == 'n')
                    {
                        InNull = true;
                    }
                    else if (current == '[')
                    {
                        NewToken(JsonToken.CROG);
                    }
                    else if (current == ']')
                    {
                        NewToken(JsonToken.CROD);
                    }
                    else if (current == '{')
                    {
                        NewToken(JsonToken.ACCG);
                    }
                    else if (current == '}')
                    {
                        NewToken(JsonToken.ACCD);
                    }
                    else if (current == ':')
                    {
                        NewToken(JsonToken.COLON);
                    }
                    else if (current == ',')
                    {
                        NewToken(JsonToken.COMMA);
                    }
                    else if (char.IsWhiteSpace(current))
                    {
                        currentValue = "";
                    }
                    else if (current == '\n' || current == '\r')
                    {
                        currentLine++;
                        currentValue  = ";;";
                        currentColumn = 1;
                    }

                    currentColumn++;
                    position++;
                }
            }
            return(new LexerResult <JsonToken>(tokens));
        }