private void cmdManaged_Click(object sender, EventArgs e) { //some lang eg. Thai, Lao, need dictionary breaking //we use dic data from icu-project //1. create dictionary based breaking engine //TODO: dic should be read once var dicProvider = new IcuSimpleTextFileDictionaryProvider() { DataDir = "../../../icu58/brkitr_src/dictionaries" }; CustomBreakerBuilder.Setup(dicProvider); CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker(); char[] test = this.textBox1.Text.ToCharArray(); this.listBox1.Items.Clear(); breaker1.BreakWords(test, 0, test.Length); foreach (BreakSpan span in breaker1.GetBreakSpanIter()) { string s = new string(test, span.startAt, span.len); this.listBox1.Items.Add(span.startAt + " " + s); } }
private void cmdManaged_Click(object sender, EventArgs e) { CustomBreakerBuilder.DataDir = "../../../icu58/brkitr_src/dictionaries"; CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker(); //2. create dictionary based breaking engine // char[] test = this.textBox1.Text.ToCharArray(); this.listBox1.Items.Clear(); breaker1.BreakWords(test, 0); foreach (var span in breaker1.GetBreakSpanIter()) { string s = new string(test, span.startAt, span.len); this.listBox1.Items.Add(span.startAt + " " + s); } }
void ParseWithManaged(int ntimes) { //------------------- CustomBreakerBuilder.DataDir = "../../../icu58/brkitr_src/dictionaries"; CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker(); char[] test = this.textBox1.Text.ToCharArray(); //------------- for (int i = ntimes - 1; i >= 0; --i) { breaker1.BreakWords(test, 0); foreach (var span in breaker1.GetBreakSpanIter()) { } } }
public void WordKindTest() { var breaker = new CustomBreaker { ThrowIfCharOutOfRange = true }; var breakList = new List<BreakSpan>(); #warning Use `breaker.BreakWords("«Maître leçon»", breakList);` once #156 is merged breaker.BreakWords("«Maître leçon»"); breakList.AddRange(breaker.GetBreakSpanIter()); Assert.AreEqual(breakList.Count, 5); void BreakSpanEqual(BreakSpan actual, BreakSpan expected) { Assert.AreEqual(expected.startAt, actual.startAt); Assert.AreEqual(expected.len, actual.len); Assert.AreEqual(expected.wordKind, actual.wordKind); } BreakSpanEqual(breakList[0], new BreakSpan { startAt = 0, len = 1, wordKind = WordKind.Punc }); BreakSpanEqual(breakList[1], new BreakSpan { startAt = 1, len = 6, wordKind = WordKind.Text }); BreakSpanEqual(breakList[2], new BreakSpan { startAt = 7, len = 1, wordKind = WordKind.Whitespace }); BreakSpanEqual(breakList[3], new BreakSpan { startAt = 8, len = 5, wordKind = WordKind.Text }); BreakSpanEqual(breakList[4], new BreakSpan { startAt = 13, len = 1, wordKind = WordKind.Punc }); }
private void cmdManaged_Click(object sender, EventArgs e) { //some lang eg. Thai, Lao, need dictionary breaking //we use dic data from icu-project //1. create dictionary based breaking engine CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker(); char[] test = this.textBox1.Text.ToCharArray(); this.listBox1.Items.Clear(); breaker1.BreakWords(test, 0, test.Length); foreach (BreakSpan span in breaker1.GetBreakSpanIter()) { string s = new string(test, span.startAt, span.len); this.listBox1.Items.Add(span.startAt + " " + s); } }
void ParseWithManaged(int ntimes) { //------------------- var dicProvider = new IcuSimpleTextFileDictionaryProvider() { DataDir = "../../../icu58/brkitr_src/dictionaries" }; CustomBreakerBuilder.Setup(dicProvider); CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker(); char[] test = this.textBox1.Text.ToCharArray(); //------------- for (int i = ntimes - 1; i >= 0; --i) { breaker1.BreakWords(test, 0, test.Length); foreach (var span in breaker1.GetBreakSpanIter()) { } } }