public List <Token> Tokenize(ITracker tracker, int untilIndex = 0)
{
Guid newestValidResetPoint = tracker.GetResetPoint();
index = tracker.Index;
Level = 0;
if (0 == untilIndex || tracker.StopIndex < untilIndex)
{
untilIndex = tracker.StopIndex;
}
while (index < untilIndex)
{
if (Level >= Tokenizers.Count)
{
Errors.Reverse();
TokenizeError specific = (from e in Errors
orderby e.Specificity descending
select e).First();
throw new TokenizeException(tracker.Raw, specific.Region, tracker.Last, specific.Message);
}
if (!Tokenizers[Level].Selector(tracker.Raw[index]))
{
AddError("unknown char: '" + tracker.Raw[index] + "'", 0, new TextRegion(index), Tokenizers[Level]);
Level++;
continue;
}
// Reset errors from last iteration in the tracker
if (!tracker.ResetErrorsToLastResetPoint(newestValidResetPoint))
{
throw new TokenizeException(tracker.Raw, new TextRegion(0, 0), tracker.Last, "Could not reset to last resetpoint - that should not be possible.");
}
// Try next Tokenizer
TokenizerResult result = Tokenizers[Level].Tokenize(tracker);
if (result == TokenizerResult.Success)
{
newestValidResetPoint = tracker.GetResetPoint();
index = tracker.Index;
Level = 0;
Errors.Clear();
}
else
{
if (!tracker.HasErrors)
{
AddError(Tokenizers[Level] + ": Error!", 0, new TextRegion(index), Tokenizers[Level]);
}
else
{
Errors.AddRange(tracker.Errors);
}
tracker.Reset(newestValidResetPoint);
Level++;
}
}
return(tracker.Tokens.ToList());
}
public TrackerErrorItem(TokenizeError error, Guid resetPoint)
{
Error = error;
ResetPoint = resetPoint;
IsMarker = false;
}