public LexerPosition ConsumeComment(Token <GenericToken> comment, ReadOnlyMemory <char> source, LexerPosition lexerPosition)
{
ReadOnlyMemory <char> commentValue;
if (comment.IsSingleLineComment)
{
var position = lexerPosition.Index;
commentValue = EOLManager.GetToEndOfLine(source, position);
position = position + commentValue.Length;
comment.SpanValue = commentValue;
return(new LexerPosition(position, lexerPosition.Line + 1, 0));
//LexerFsm.MovePosition(position, LexerFsm.CurrentLine + 1, 0);
}
else if (comment.IsMultiLineComment)
{
var position = lexerPosition.Index;
var end = source.Span.Slice(position).IndexOf(MultiLineCommentEnd.AsSpan());
if (end < 0)
{
position = source.Length;
}
else
{
position = end + position;
}
commentValue = source.Slice(lexerPosition.Index, position - lexerPosition.Index);
comment.SpanValue = commentValue;
var newPosition = lexerPosition.Index + commentValue.Length + MultiLineCommentEnd.Length;
var lines = EOLManager.GetLinesLength(commentValue);
var newLine = lexerPosition.Line + lines.Count - 1;
int newColumn;
if (lines.Count > 1)
{
newColumn = lines.Last() + MultiLineCommentEnd.Length;
}
else
{
newColumn = lexerPosition.Column + lines[0] + MultiLineCommentEnd.Length;
}
return(new LexerPosition(newPosition, newLine, newColumn));
// LexerFsm.MovePosition(newPosition, newLine, newColumn);
}
return(lexerPosition);
}
public Token()
{
IsEOS = true;
End = true;
Position = new LexerPosition(0, 0, 0);
}
public LexerResult <IN> Tokenize(ReadOnlyMemory <char> memorySource)
{
LexerPosition position = new LexerPosition();
var tokens = new List <Token <IN> >();
var r = LexerFsm.Run(memorySource, new LexerPosition());
if (!r.IsSuccess && !r.IsEOS)
{
var result = r.Result;
var error = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
return(new LexerResult <IN>(error));
}
if (r.IsSuccess && r.Result.IsComment)
{
position = r.NewPosition;
position = ConsumeComment(r.Result, memorySource, position);
}
else if (r.IsSuccess && !r.Result.IsComment)
{
position = r.NewPosition;
}
while (r.IsSuccess)
{
ComputePositionWhenIgnoringEOL(r, tokens);
var transcoded = Transcode(r);
if (CallBacks.TryGetValue(transcoded.TokenID, out var callback))
{
transcoded = callback(transcoded);
}
tokens.Add(transcoded);
r = LexerFsm.Run(memorySource, position);
if (!r.IsSuccess && !r.IsEOS)
{
var result = r.Result;
var error = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
return(new LexerResult <IN>(error));
}
if (r.IsSuccess && r.Result.IsComment)
{
position = r.NewPosition;
position = ConsumeComment(r.Result, memorySource, position);
}
}
var eos = new Token <IN>();
var prev = tokens.LastOrDefault();
if (prev == null)
{
eos.Position = new LexerPosition(1, 0, 0);
}
else
{
eos.Position = new LexerPosition(prev.Position.Index + 1, prev.Position.Line,
prev.Position.Column + prev.Value.Length);
}
tokens.Add(eos);
return(new LexerResult <IN>(tokens));
}
public Token(T token, string value, LexerPosition position, bool isCommentStart = false,
CommentType commentType = CommentType.Single) : this(token, new ReadOnlyMemory <char>(value.ToCharArray()), position, isCommentStart, commentType)
{
}