示例#1
0
        public List <Token> Tokenize(ITracker tracker, int untilIndex = 0)
        {
            Guid newestValidResetPoint = tracker.GetResetPoint();

            index = tracker.Index;
            Level = 0;

            if (0 == untilIndex || tracker.StopIndex < untilIndex)
            {
                untilIndex = tracker.StopIndex;
            }

            while (index < untilIndex)
            {
                if (Level >= Tokenizers.Count)
                {
                    Errors.Reverse();
                    TokenizeError specific = (from e in Errors
                                              orderby e.Specificity descending
                                              select e).First();

                    throw new TokenizeException(tracker.Raw, specific.Region, tracker.Last, specific.Message);
                }

                if (!Tokenizers[Level].Selector(tracker.Raw[index]))
                {
                    AddError("unknown char: '" + tracker.Raw[index] + "'", 0, new TextRegion(index), Tokenizers[Level]);
                    Level++;
                    continue;
                }

                // Reset errors from last iteration in the tracker
                if (!tracker.ResetErrorsToLastResetPoint(newestValidResetPoint))
                {
                    throw new TokenizeException(tracker.Raw, new TextRegion(0, 0), tracker.Last, "Could not reset to last resetpoint - that should not be possible.");
                }

                // Try next Tokenizer
                TokenizerResult result = Tokenizers[Level].Tokenize(tracker);

                if (result == TokenizerResult.Success)
                {
                    newestValidResetPoint = tracker.GetResetPoint();
                    index = tracker.Index;

                    Level = 0;
                    Errors.Clear();
                }
                else
                {
                    if (!tracker.HasErrors)
                    {
                        AddError(Tokenizers[Level] + ": Error!", 0, new TextRegion(index), Tokenizers[Level]);
                    }
                    else
                    {
                        Errors.AddRange(tracker.Errors);
                    }
                    tracker.Reset(newestValidResetPoint);
                    Level++;
                }
            }

            return(tracker.Tokens.ToList());
        }
示例#2
0
 public TrackerErrorItem(TokenizeError error, Guid resetPoint)
 {
     Error      = error;
     ResetPoint = resetPoint;
     IsMarker   = false;
 }