コード例 #1
0
ファイル: FSMLexer.cs プロジェクト: thulyacloud/csly
        private void ConsumeIgnored(ReadOnlyMemory <char> source, LexerPosition position)
        {
            while (position.Index < source.Length)
            {
                if (IgnoreWhiteSpace)
                {
                    var currentCharacter = source.At(position.Index);
                    if (WhiteSpaces.Contains(currentCharacter))
                    {
                        position.Index++;
                        position.Column++;
                        continue;
                    }
                }

                if (IgnoreEOL)
                {
                    var eol = EOLManager.IsEndOfLine(source, position.Index);
                    if (eol != EOLType.No)
                    {
                        position.Index += eol == EOLType.Windows ? 2 : 1;
                        position.Column = 0;
                        position.Line++;
                        continue;
                    }
                }

                break;
            }
        }
コード例 #2
0
        private void ConsumeIgnored(ReadOnlyMemory <char> source)
        {
            while (CurrentPosition < source.Length)
            {
                if (IgnoreWhiteSpace)
                {
                    var currentCharacter = source.At(CurrentPosition);
                    if (WhiteSpaces.Contains(currentCharacter))
                    {
                        CurrentPosition++;
                        CurrentColumn++;
                        continue;
                    }
                }

                if (IgnoreEOL)
                {
                    var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
                    if (eol != EOLType.No)
                    {
                        CurrentPosition += eol == EOLType.Windows ? 2 : 1;
                        CurrentColumn    = 0;
                        CurrentLine++;
                        continue;
                    }
                }

                break;
            }
        }
コード例 #3
0
 public static string RemoveWhiteSpaces(this string value) => value == null ? null : WhiteSpaces.Replace(value, string.Empty);
コード例 #4
0
ファイル: FSMLexer.cs プロジェクト: magne/csly
        public FSMMatch <N> Run(ReadOnlyMemory <char> source, int start)
        {
            var value           = "";
            int tokenStartIndex = start;
            int tokenLength     = 0;
            var result          = new FSMMatch <N>(false);
            var successes       = new Stack <FSMMatch <N> >();

            CurrentPosition = start;
            var           currentNode = Nodes[0];
            var           lastNode    = 0;
            TokenPosition position    = null;

            var tokenStarted = false;


            if (CurrentPosition < source.Length)
            {
                var currentCharacter = source.At(CurrentPosition);

                while (CurrentPosition < source.Length && currentNode != null)
                {
                    currentCharacter = source.Span[CurrentPosition];

                    var consumeSkipped = true;

                    while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
                    {
                        currentCharacter = source.At(CurrentPosition);
                        if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentCharacter))
                        {
                            if (successes.Any())
                            {
                                currentNode = null;
                            }
                            else
                            {
                                currentNode = Nodes[0];
                            }
                            CurrentPosition++;
                            CurrentColumn++;
                        }
                        else
                        {
                            var eol = EOLManager.IsEndOfLine(source, CurrentPosition);

                            if (IgnoreEOL && eol != EOLType.No)
                            {
                                if (successes.Any())
                                {
                                    currentNode = null;
                                }
                                else
                                {
                                    currentNode = Nodes[0];
                                }
                                CurrentPosition += eol == EOLType.Windows ? 2 : 1;
                                CurrentColumn    = 0;
                                CurrentLine++;
                            }
                            else
                            {
                                consumeSkipped = false;
                            }
                        }
                    }


                    currentNode = Move(currentNode, currentCharacter, value);
                    if (currentNode != null)
                    {
                        lastNode = currentNode.Id;
                        value   += currentCharacter;
                        tokenLength++;

                        if (!tokenStarted)
                        {
                            tokenStarted = true;
                            position     = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
                        }

                        if (currentNode.IsEnd)
                        {
                            var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position, currentNode.Id);
                            successes.Push(resultInter);
                        }

                        if (HasAction(currentNode.Id))
                        {
                            value = Actions[currentNode.Id](value);
                        }
                        CurrentPosition++;
                        CurrentColumn++;
                    }
                    else
                    {
                        if (!successes.Any() && CurrentPosition < source.Length)
                        {
                            throw new LexerException(new LexicalError(CurrentLine, CurrentColumn,
                                                                      source.At(CurrentPosition)));
                        }
                    }
                }
            }


            if (successes.Any())
            {
                result = successes.Pop();
                if (HasCallback(result.NodeId))
                {
                    result = Callbacks[result.NodeId](result);
                }
            }

            return(result);
        }
コード例 #5
0
ファイル: FSMLexer.cs プロジェクト: dotted/csly
        public FSMMatch <N> Run(string source, int start)
        {
            string value     = "";
            var    result    = new FSMMatch <N>(false);
            var    successes = new Stack <FSMMatch <N> >();

            CurrentPosition = start;
            FSMNode <N>   currentNode = Nodes[0];
            int           lastNode    = 0;
            TokenPosition position    = null;

            bool tokenStarted = false;


            if (CurrentPosition < source.Length)
            {
                char currentToken = source[CurrentPosition];

                while (CurrentPosition < source.Length && currentNode != null)
                {
                    currentToken = source[CurrentPosition];

                    bool consumeSkipped = true;

                    while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
                    {
                        currentToken = source[CurrentPosition];
                        if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentToken))
                        {
                            if (successes.Any())
                            {
                                currentNode = null;
                            }
                            else
                            {
                                currentNode = Nodes[0];
                            }
                            CurrentPosition++;
                            CurrentColumn++;
                        }
                        else
                        {
                            var eol = EOLManager.IsEndOfLine(source, CurrentPosition);

                            if (IgnoreEOL && eol != EOLType.No)
                            {
                                if (successes.Any())
                                {
                                    currentNode = null;
                                }
                                else
                                {
                                    currentNode = Nodes[0];
                                }
                                CurrentPosition += (eol == EOLType.Windows ? 2 : 1);
                                CurrentColumn    = 0;
                                CurrentLine++;
                            }
                            else
                            {
                                consumeSkipped = false;
                            }
                        }
                    }



                    currentNode = Move(currentNode, currentToken, value);
                    if (currentNode != null)
                    {
                        lastNode = currentNode.Id;
                        value   += currentToken;

                        if (!tokenStarted)
                        {
                            tokenStarted = true;
                            position     = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
                        }
                        if (currentNode.IsEnd)
                        {
                            var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position);
                            successes.Push(resultInter);
                        }
                        if (HasAction(currentNode.Id))
                        {
                            value = Actions[currentNode.Id](value);
                        }
                        CurrentPosition++;
                        CurrentColumn++;
                    }
                    else
                    {
                        if (lastNode == 0 && !tokenStarted && !successes.Any() && CurrentPosition < source.Length)
                        {
                            throw new LexerException(new LexicalError(CurrentLine, CurrentColumn, source[CurrentPosition]));
                        }
                        ;
                    }
                }
            }


            if (successes.Any())
            {
                result = successes.Pop();
                if (HasCallback(lastNode))
                {
                    result = Callbacks[lastNode](result);
                }
            }

            return(result);
        }