public UnrecognizedTokenException(LeskContext context)
        {
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            Context = context;
        }
示例#2
0
        public List<Token> Tokenize(string input)
        {
            var context = new LeskContext(input);
            while (context.HasMore())
            {
                var allResults = new List<ConsumeResult>();
                foreach (var consumerBuilder in Consumers)
                {
                    InputConsumer consumer = consumerBuilder.Item1();
                    var clonedContext = context.Clone();
                    ConsumeResult result = consumer.Consume(clonedContext);
                    result.TokenBuilder = consumerBuilder.Item2;
                    allResults.Add(result);
                }

                if (allResults.All(r => r.Success == false))
                {
                    throw new UnrecognizedTokenException(context);
                }

                var succeded = allResults.Where(r => r.Success);

                if (succeded.Count() == 1)
                {
                    ConsumeResult theOne = succeded.FirstOrDefault();
                    context.Apply(theOne);
                }
                else
                {
                    var bestMatches = succeded.Where(r => r.ConsumedLength == succeded.Max(r2 => r2.ConsumedLength));
                    ConsumeResult final = null;

                    if (bestMatches.Count() > 1)
                    {
                        Trace.WriteLine("Warning : Multiple rules matched. Selecting the rule defined first.");
                    }

                    final = bestMatches.FirstOrDefault();
                    if (final == null)
                    {
                        throw new InvalidOperationException("Unexpected error.");
                    }

                    context.Apply(final);
                }
            }

            return context.Tokens;
        }
示例#3
0
文件: LeskContext.cs 项目: k0dat/Lesk
 public LeskContext Clone()
 {
     var context = new LeskContext(Input);
     context._position = _position;
     return context;
 }