public TokenizationResult Tokenize(string input) { var producedTokens = new List <Token>(); RecoverableException exception = null; while (input.Length > 0) { var success = false; foreach (var proposition in tokens) { var regex = proposition.ApplicabilityCondition; var match = regex.Match(input); if (!match.Success) { continue; } success = true; Token producedToken; try { producedToken = proposition.Factory(input.Substring(0, match.Length)); } catch (RecoverableException e) { success = false; exception = exception ?? e; break; } input = input.Substring(match.Length); if (producedToken != null) { producedTokens.Add(producedToken); } break; } if (!success) { break; } } return(new TokenizationResult(input.Length, producedTokens, exception)); }
public TokenizationResult(int unmatchedCharactersLeft, IEnumerable <Token> tokens, RecoverableException e) { UnmatchedCharactersLeft = unmatchedCharactersLeft; Tokens = tokens; Exception = e; }
private void ProcessRecoverableException(RecoverableException exception) { this.errorMessage = exception.Message; this.recoveryChunkOffset = exception.RecoveryChunkOffset; this.recoveryRecordOffset = exception.RecoveryRecordOffset; }