public TokenizationResult Tokenize(string input)
{
var producedTokens = new List <Token>();
RecoverableException exception = null;
while (input.Length > 0)
{
var success = false;
foreach (var proposition in tokens)
{
var regex = proposition.ApplicabilityCondition;
var match = regex.Match(input);
if (!match.Success)
{
continue;
}
success = true;
Token producedToken;
try
{
producedToken = proposition.Factory(input.Substring(0, match.Length));
}
catch (RecoverableException e)
{
success = false;
exception = exception ?? e;
break;
}
input = input.Substring(match.Length);
if (producedToken != null)
{
producedTokens.Add(producedToken);
}
break;
}
if (!success)
{
break;
}
}
return(new TokenizationResult(input.Length, producedTokens, exception));
}
public TokenizationResult(int unmatchedCharactersLeft, IEnumerable <Token> tokens, RecoverableException e)
{
UnmatchedCharactersLeft = unmatchedCharactersLeft;
Tokens = tokens;
Exception = e;
}
private void ProcessRecoverableException(RecoverableException exception)
{
this.errorMessage = exception.Message;
this.recoveryChunkOffset = exception.RecoveryChunkOffset;
this.recoveryRecordOffset = exception.RecoveryRecordOffset;
}
