using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.Recognizers.Text; using Microsoft.Recognizers.Text.Number; public class TokenizeExample { public static void Main(string[] args) { var input = "The price for this item is $5.99"; var tokenizer = new NumberTokenizer(); tokenizer.Init(input); while (tokenizer.MoveNext()) { Console.WriteLine(tokenizer.Current.Value); } } }
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; public class CustomTokenizer : ITokenizer { private string[] _tokens; private int _index; public void Init(string input) { _tokens = input.Split(' '); _index = -1; } public Token Current { get { if (_index >= 0 && _index < _tokens.Length) { return new Token(_tokens[_index]); } else { return null; } } } public bool MoveNext() { if (_index < _tokens.Length - 1) { _index++; return true; } else { return false; } } } public class TokenizeExample { public static void Main(string[] args) { var input = "This is a custom tokenizer example"; var tokenizer = new CustomTokenizer(); tokenizer.Init(input); while (tokenizer.MoveNext()) { Console.WriteLine(tokenizer.Current.Value); } } }In this example, we define a custom tokenizer that implements the ITokenizer interface. The Init method splits the input string into an array of tokens, and the MoveNext method moves the pointer to the next token in the array. We use the custom tokenizer to tokenize a simple input string, and then print out the value of each token using a while loop and the Current property of the tokenizer. The package library used in both examples is Microsoft.Recognizers.Text, which provides a set of tools for working with natural language expressions.