public static void SetICUDataFile(string icudatafile) { lock (s_dataLoadLock) { if (s_isDataLoaded) { return; } } if (s_isDataLoaded) { return; } s_isDataLoaded = true; s_icuDataFile = icudatafile; //---------- int major, minor, revision; NativeTextBreakerLib.MyFtLibGetFullVersion(out major, out minor, out revision); if (dataHolder == null) { dataHolder = new InMemoryIcuDataHolder(icudatafile); dataHolder.Use(); } }
public void Use() { int errCode; unsafe { NativeTextBreakerLib.MyFt_IcuSetData((void *)unmanagedICUMemData, out errCode); } }
public static void Test1() { int major, minor, revision; NativeTextBreakerLib.MyFtLibGetFullVersion(out major, out minor, out revision); NativeTextBreaker.SetICUDataFile(@"icudt57l\icudt57l.dat"); string str = "ABCD EFGH IJKL\0"; var textBreaker = new NativeTextBreaker(TextBreakKind.Word, "en-US"); List <SplitBound> tokens = new List <SplitBound>(); textBreaker.DoBreak(str, splitBound => { tokens.Add(splitBound); }); }
public override void DoBreak(char[] input, int start, int len, OnBreak onbreak) { //1. UBreakIteratorType type = UBreakIteratorType.WORD; switch (BreakKind) { default: case TextBreakKind.Word: type = UBreakIteratorType.WORD; break; case TextBreakKind.Sentence: type = UBreakIteratorType.SENTENCE; break; } //------------------------ int errCode = 0; //break all string unsafe { fixed(char *h = &input[start]) { IntPtr nativeIter = NativeTextBreakerLib.MtFt_UbrkOpen(type, localebuff, h, len, out errCode); int cur = NativeTextBreakerLib.MtFt_UbrkFirst(nativeIter); while (cur != DONE) { int next = NativeTextBreakerLib.MtFt_UbrkNext(nativeIter); int status = NativeTextBreakerLib.MtFt_UbrkGetRuleStatus(nativeIter); if (next != DONE && AddToken(type, status)) { onbreak(new SplitBound(cur, next - cur)); } cur = next; } NativeTextBreakerLib.MtFt_UbrkClose(nativeIter); } } }