public IEnumerable <Token <IN> > Tokenize(string source)
{
var tokens = new List <Token <IN> >();
var r = LexerFsm.Run(source, 0);
while (r.IsSuccess)
{
var transcoded = Transcode(r);
Func <Token <IN>, Token <IN> > callback = null;
if (CallBacks.TryGetValue(transcoded.TokenID, out callback))
{
transcoded = callback(transcoded);
}
tokens.Add(transcoded);
r = LexerFsm.Run(source);
if (r.IsSuccess && r.Result.IsComment)
{
ConsumeComment(r.Result, source);
}
}
var eos = new Token <IN>();
var prev = tokens.Last();
eos.Position = new TokenPosition(prev.Position.Index + 1, prev.Position.Line,
prev.Position.Column + prev.Value.Length);
tokens.Add(eos);
return(tokens);
}
public LexerResult <IN> Tokenize(ReadOnlyMemory <char> memorySource)
{
var tokens = new List <Token <IN> >();
var r = LexerFsm.Run(memorySource, 0);
if (!r.IsSuccess && !r.IsEOS)
{
var result = r.Result;
var error = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
return(new LexerResult <IN>(error));
}
while (r.IsSuccess)
{
var transcoded = Transcode(r);
if (CallBacks.TryGetValue(transcoded.TokenID, out var callback))
{
transcoded = callback(transcoded);
}
tokens.Add(transcoded);
r = LexerFsm.Run(memorySource);
if (!r.IsSuccess && !r.IsEOS)
{
var result = r.Result;
var error = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
return(new LexerResult <IN>(error));
}
if (r.IsSuccess && r.Result.IsComment)
{
ConsumeComment(r.Result, memorySource);
}
}
var eos = new Token <IN>();
var prev = tokens.LastOrDefault();
if (prev == null)
{
eos.Position = new TokenPosition(1, 0, 0);
}
else
{
eos.Position = new TokenPosition(prev.Position.Index + 1, prev.Position.Line,
prev.Position.Column + prev.Value.Length);
}
tokens.Add(eos);
return(new LexerResult <IN>(tokens));
}
public IEnumerable <Token <IN> > Tokenize(string source)
{
List <Token <IN> > tokens = new List <Token <IN> >();
var r = LexerFsm.Run(source, 0);
while (r.IsSuccess)
{
var transcoded = Transcode(r);
tokens.Add(transcoded);
r = LexerFsm.Run(source);
if (r.Result.IsComment)
{
ConsumeComment(r.Result, source);
}
}
var eos = new Token <IN>();
var prev = tokens.Last();
eos.Position = new TokenPosition(prev.Position.Index + 1, prev.Position.Line, prev.Position.Column + prev.Value.Length);
tokens.Add(eos);
return(tokens);
}