public string[] Tokenize(string input) { var preprocessed_input = preprocessor.Processor(input); var tokenized_input = tokenizer.Tokenize(preprocessed_input); return(tokenized_input .Select(token => postprocessor.Processor(token)) .ToArray()); }
public string[] Tokenize(string input) { var preprocessed_input = preprocessor.Processor(input); var tokenized_input = tokenizer.Tokenize(preprocessed_input); return(tokenized_input .Select(token => postprocessor.Processor(token)) .ToArray()); }