public TokenizerTest()
{
InitializeComponent();
errors = new List <EchelonScriptErrorMessage> ();
tokenizer = new EchelonScriptTokenizer(errors);
textMarkerService = new TextMarkerService(codeText);
}
public EchelonScriptErrorMessage(ReadOnlySpan <char> srcText, ReadOnlyMemory <char> fileName, ES_AstNodeBounds bounds, string?message = null)
{
Message = message;
StartPos = bounds.StartPos;
Length = bounds.EndPos - bounds.StartPos;
EchelonScriptTokenizer.CalcLine(srcText, StartPos, out var curLine, out var curLineStart);
FileName = fileName.Span.GetPooledString();
Line = curLine;
Column = EchelonScriptTokenizer.CalcColumn(srcText, curLineStart, StartPos);
}
public override bool ParseToken(EchelonScriptTokenizer tokenizer, ReadOnlySpan <char> peekedChars, ref EchelonScriptToken retToken)
{
var startPos = tokenizer.curPos;
tokenizer.ReadChars(2);
if (tokenizer.PeekChar() == NumberSeparator)
{
tokenizer.Errors.Add(new EchelonScriptErrorMessage(retToken, ES_FrontendErrors.InvalidHexLiteral));
}
else
{
tokenizer.ReadStringWhile(c => IsHexDigit(c) || c == NumberSeparator);
}
tokenizer.TryReadIntSuffix();
retToken.Type = EchelonScriptTokenType.HexIntegerLiteral;
retToken.Text = tokenizer.data [startPos..tokenizer.curPos];
public override bool IsStartValid(EchelonScriptTokenizer tokenizer, ReadOnlySpan <char> peekedChars)
=> peekedChars.StartsWith("0x", StringComparison.InvariantCultureIgnoreCase);
public abstract bool ParseToken(EchelonScriptTokenizer tokenizer, ReadOnlySpan <char> peekedChars, ref EchelonScriptToken retToken);
public abstract bool IsStartValid(EchelonScriptTokenizer tokenizer, ReadOnlySpan <char> peekedChars);
