public UnrecognizedTokenException(LeskContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
Context = context;
}
public List<Token> Tokenize(string input)
{
var context = new LeskContext(input);
while (context.HasMore())
{
var allResults = new List<ConsumeResult>();
foreach (var consumerBuilder in Consumers)
{
InputConsumer consumer = consumerBuilder.Item1();
var clonedContext = context.Clone();
ConsumeResult result = consumer.Consume(clonedContext);
result.TokenBuilder = consumerBuilder.Item2;
allResults.Add(result);
}
if (allResults.All(r => r.Success == false))
{
throw new UnrecognizedTokenException(context);
}
var succeded = allResults.Where(r => r.Success);
if (succeded.Count() == 1)
{
ConsumeResult theOne = succeded.FirstOrDefault();
context.Apply(theOne);
}
else
{
var bestMatches = succeded.Where(r => r.ConsumedLength == succeded.Max(r2 => r2.ConsumedLength));
ConsumeResult final = null;
if (bestMatches.Count() > 1)
{
Trace.WriteLine("Warning : Multiple rules matched. Selecting the rule defined first.");
}
final = bestMatches.FirstOrDefault();
if (final == null)
{
throw new InvalidOperationException("Unexpected error.");
}
context.Apply(final);
}
}
return context.Tokens;
}
public LeskContext Clone()
{
var context = new LeskContext(Input);
context._position = _position;
return context;
}