예제 #1
0
        public LexerPosition ConsumeComment(Token <GenericToken> comment, ReadOnlyMemory <char> source, LexerPosition lexerPosition)
        {
            ReadOnlyMemory <char> commentValue;

            if (comment.IsSingleLineComment)
            {
                var position = lexerPosition.Index;
                commentValue      = EOLManager.GetToEndOfLine(source, position);
                position          = position + commentValue.Length;
                comment.SpanValue = commentValue;
                return(new LexerPosition(position, lexerPosition.Line + 1, 0));
                //LexerFsm.MovePosition(position, LexerFsm.CurrentLine + 1, 0);
            }
            else if (comment.IsMultiLineComment)
            {
                var position = lexerPosition.Index;

                var end = source.Span.Slice(position).IndexOf(MultiLineCommentEnd.AsSpan());
                if (end < 0)
                {
                    position = source.Length;
                }
                else
                {
                    position = end + position;
                }
                commentValue      = source.Slice(lexerPosition.Index, position - lexerPosition.Index);
                comment.SpanValue = commentValue;

                var newPosition = lexerPosition.Index + commentValue.Length + MultiLineCommentEnd.Length;
                var lines       = EOLManager.GetLinesLength(commentValue);
                var newLine     = lexerPosition.Line + lines.Count - 1;
                int newColumn;
                if (lines.Count > 1)
                {
                    newColumn = lines.Last() + MultiLineCommentEnd.Length;
                }
                else
                {
                    newColumn = lexerPosition.Column + lines[0] + MultiLineCommentEnd.Length;
                }

                return(new LexerPosition(newPosition, newLine, newColumn));
                // LexerFsm.MovePosition(newPosition, newLine, newColumn);
            }

            return(lexerPosition);
        }
예제 #2
0
파일: Token.cs 프로젝트: thulyacloud/csly
 public Token()
 {
     IsEOS    = true;
     End      = true;
     Position = new LexerPosition(0, 0, 0);
 }
예제 #3
0
        public LexerResult <IN> Tokenize(ReadOnlyMemory <char> memorySource)
        {
            LexerPosition position = new LexerPosition();

            var tokens = new List <Token <IN> >();

            var r = LexerFsm.Run(memorySource, new LexerPosition());

            if (!r.IsSuccess && !r.IsEOS)
            {
                var result = r.Result;
                var error  = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
                return(new LexerResult <IN>(error));
            }
            if (r.IsSuccess && r.Result.IsComment)
            {
                position = r.NewPosition;
                position = ConsumeComment(r.Result, memorySource, position);
            }
            else if (r.IsSuccess && !r.Result.IsComment)
            {
                position = r.NewPosition;
            }

            while (r.IsSuccess)
            {
                ComputePositionWhenIgnoringEOL(r, tokens);
                var transcoded = Transcode(r);
                if (CallBacks.TryGetValue(transcoded.TokenID, out var callback))
                {
                    transcoded = callback(transcoded);
                }

                tokens.Add(transcoded);

                r = LexerFsm.Run(memorySource, position);
                if (!r.IsSuccess && !r.IsEOS)
                {
                    var result = r.Result;
                    var error  = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
                    return(new LexerResult <IN>(error));
                }

                if (r.IsSuccess && r.Result.IsComment)
                {
                    position = r.NewPosition;
                    position = ConsumeComment(r.Result, memorySource, position);
                }
            }

            var eos  = new Token <IN>();
            var prev = tokens.LastOrDefault();

            if (prev == null)
            {
                eos.Position = new LexerPosition(1, 0, 0);
            }
            else
            {
                eos.Position = new LexerPosition(prev.Position.Index + 1, prev.Position.Line,
                                                 prev.Position.Column + prev.Value.Length);
            }
            tokens.Add(eos);
            return(new LexerResult <IN>(tokens));
        }
예제 #4
0
파일: Token.cs 프로젝트: thulyacloud/csly
 public Token(T token, string value, LexerPosition position, bool isCommentStart = false,
              CommentType commentType = CommentType.Single) : this(token, new ReadOnlyMemory <char>(value.ToCharArray()), position, isCommentStart, commentType)
 {
 }