public void TestRoundTrip() { String[] HANGUL = { "\uAC03\uC2F8", "\uC544\uC5B4" }; Transliterator latinJamo = Transliterator.GetInstance("Latin-Jamo"); Transliterator jamoLatin = latinJamo.GetInverse(); Transliterator jamoHangul = Transliterator.GetInstance("NFC"); Transliterator hangulJamo = Transliterator.GetInstance("NFD"); StringBuffer buf = new StringBuffer(); for (int i = 0; i < HANGUL.Length; ++i) { String hangul = HANGUL[i]; String jamo = hangulJamo.Transliterate(hangul); String latin = jamoLatin.Transliterate(jamo); String jamo2 = latinJamo.Transliterate(latin); String hangul2 = jamoHangul.Transliterate(jamo2); buf.Length = (0); buf.Append(hangul + " => " + jamoToName(jamo) + " => " + latin + " => " + jamoToName(jamo2) + " => " + hangul2 ); if (!hangul.Equals(hangul2)) { Errln("FAIL: " + Utility.Escape(buf.ToString())); } else { Logln(Utility.Escape(buf.ToString())); } } }
public virtual Transliterator GetInstance() { List <Transliterator> transliterators = new List <Transliterator>(); int passNumber = 1; int limit = Math.Max(idBlockVector.Count, dataVector.Count); for (int i = 0; i < limit; i++) { if (i < idBlockVector.Count) { string idBlock = idBlockVector[i]; if (idBlock.Length > 0) { transliterators.Add(Transliterator.GetInstance(idBlock)); } } if (i < dataVector.Count) { #pragma warning disable 612, 618 Data data = dataVector[i]; transliterators.Add(new RuleBasedTransliterator("%Pass" + passNumber++, data, null)); #pragma warning restore 612, 618 } } Transliterator t = new CompoundTransliterator(transliterators, passNumber - 1); t.ID = id; if (compoundFilter != null) { t.Filter = compoundFilter; } return(t); }
public void TestForWidth() { Transliterator widen = Transliterator.GetInstance("halfwidth-fullwidth"); Transliterator narrow = Transliterator.GetInstance("fullwidth-halfwidth"); UnicodeSet ASCII = new UnicodeSet("[:ascii:]"); string lettersAndSpace = "abc def"; string punctOnly = "( )"; String wideLettersAndSpace = widen.Transform(lettersAndSpace); String widePunctOnly = widen.Transform(punctOnly); assertContainsNone("Should be wide", ASCII, wideLettersAndSpace); assertContainsNone("Should be wide", ASCII, widePunctOnly); String back; back = narrow.Transform(wideLettersAndSpace); assertEquals("Should be narrow", lettersAndSpace, back); back = narrow.Transform(widePunctOnly); assertEquals("Should be narrow", punctOnly, back); Transliterator latin = Transliterator.GetInstance("any-Latn"); back = latin.Transform(wideLettersAndSpace); assertEquals("Should be ascii", lettersAndSpace, back); back = latin.Transform(widePunctOnly); assertEquals("Should be ascii", punctOnly, back); // Han-Latin is now forward-only per CLDR ticket #5630 //Transliterator t2 = Transliterator.GetInstance("any-Han"); //back = t2.transform(widePunctOnly); //assertEquals("Should be same", widePunctOnly, back); }
public void TestAnyTranslit() { Transliterator tx = Transliterator.GetInstance("Any-Latin"); List <Thread> threads = new List <Thread>(); for (int i = 0; i < 8; i++) { threads.Add(new Thread(() => { tx.Transliterate("διαφορετικούς"); })); } foreach (Thread th in threads) { th.Start(); } foreach (Thread th in threads) { #if !NETCOREAPP1_0 try { #endif th.Join(); #if !NETCOREAPP1_0 } catch (ThreadInterruptedException e) { Errln("Uexpected exception: " + e); } #endif } }
public void TestIncrementalProgress(string lang, string text) { var targets = Transliterator.GetAvailableTargets(lang); foreach (string target in targets) { var variants = Transliterator.GetAvailableVariants(lang, target); foreach (var variant in variants) { String id = lang + "-" + target + "/" + variant; Logln("id: " + id); Transliterator t = Transliterator.GetInstance(id); CheckIncrementalAux(t, text); String rev = t.Transliterate(text); // Special treatment: This transliterator has no registered inverse, skip for now. if (id.Equals("Devanagari-Arabic/")) { continue; } Transliterator inv = t.GetInverse(); CheckIncrementalAux(inv, rev); } } }
public void TestRandomStrings() { Transliterator transform = Transliterator.GetInstance("Any-Latin"); using Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); return(new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, transform))); }); CheckRandomData(Random, a, 1000 * RandomMultiplier); }
public override void Run() { Transliterator tx = Transliterator.GetInstance("Latin-Thai"); for (int loop = 0; loop < outerInstance.iterationCount; loop++) { foreach (String s in WORDS) { count += tx.Transliterate(s).Length; } } }
// TODO: add support for custom rules /// <summary>Creates a new <see cref="ICUTransformFilterFactory"/>.</summary> public ICUTransformFilterFactory(IDictionary <string, string> args) : base(args) { string id = Require(args, "id"); string direction = Get(args, "direction", new string[] { "forward", "reverse" }, "forward", false); TransliterationDirection dir = "forward".Equals(direction, StringComparison.Ordinal) ? Transliterator.Forward : Transliterator.Reverse; transliterator = Transliterator.GetInstance(id, dir); if (args.Count > 0) { throw new ArgumentException(string.Format(J2N.Text.StringFormatter.CurrentCulture, "Unknown parameters: {0}", args)); } }
// TODO: add support for custom rules /// <summary>Creates a new <see cref="ICUTransformFilterFactory"/>.</summary> public ICUTransformFilterFactory(IDictionary <string, string> args) : base(args) { string id = Require(args, "id"); string direction = Get(args, "direction", new string[] { "forward", "reverse" }, "forward", false); TransliterationDirection dir = "forward".Equals(direction) ? Transliterator.Forward : Transliterator.Reverse; transliterator = Transliterator.GetInstance(id, dir); if (args.Count != 0) { throw new ArgumentException("Unknown parameters: " + args); } }
public void TestBasicFunctionality() { CheckToken(Transliterator.GetInstance("Traditional-Simplified"), "簡化字", "简化字"); CheckToken(Transliterator.GetInstance("Katakana-Hiragana"), "ヒラガナ", "ひらがな"); CheckToken(Transliterator.GetInstance("Fullwidth-Halfwidth"), "アルアノリウ", "アルアノリウ"); CheckToken(Transliterator.GetInstance("Any-Latin"), "Αλφαβητικός Κατάλογος", "Alphabētikós Katálogos"); CheckToken(Transliterator.GetInstance("NFD; [:Nonspacing Mark:] Remove"), "Alphabētikós Katálogos", "Alphabetikos Katalogos"); CheckToken(Transliterator.GetInstance("Han-Latin"), "中国", "zhōng guó"); }
public void TestCommonDigits() { UnicodeSet westernDigitSet = new UnicodeSet("[0-9]"); UnicodeSet westernDigitSetAndMarks = new UnicodeSet("[[0-9][:Mn:]]"); UnicodeSet arabicDigitSet = new UnicodeSet("[[:Nd:]&[:block=Arabic:]]"); Transliterator latin = Transliterator.GetInstance("Any-Latn"); Transliterator arabic = Transliterator.GetInstance("Any-Arabic"); String westernDigits = getList(westernDigitSet); String arabicDigits = getList(arabicDigitSet); String fromArabic = latin.Transform(arabicDigits); assertContainsAll("Any-Latin transforms Arabic digits", westernDigitSetAndMarks, fromArabic); if (false) { // we don't require conversion to Arabic digits String fromLatin = arabic.Transform(westernDigits); assertContainsAll("Any-Arabic transforms Western digits", arabicDigitSet, fromLatin); } }
public void TestGetTransliterator() { Logln("Testing the getTransliterator() API of CompoundTransliterator"); String ID = "Latin-Greek;Greek-Latin;Latin-Devanagari;Devanagari-Latin;Latin-Cyrillic;Cyrillic-Latin;Any-Hex;Hex-Any"; Transliterator ct1 = null; try { //ct1=new CompoundTransliterator(ID); ct1 = Transliterator.GetInstance(ID); } catch (ArgumentException iae) { Errln("CompoundTransliterator construction failed for ID=" + ID); throw iae; } //int count=ct1.getCount(); Transliterator[] elems = ct1.GetElements(); int count = elems.Length; String[] array = split(ID, ';'); if (count != array.Length) { Errln("Error: getCount() failed. Expected:" + array.Length + " got:" + count); } for (int i = 0; i < count; i++) { //String child= ct1.getTransliterator(i).getID(); String child = elems[i].ID; if (!child.Equals(array[i])) { Errln("Error getTransliterator() failed: Expected->" + array[i] + " Got->" + child); } else { Logln("OK: getTransliterator() passed: Expected->" + array[i] + " Got->" + child); } } }
public void TestRBTErrors() { String rules = "ab>y"; String id = "MyRandom-YReverse"; String goodPattern = "[[:L:]&[\\u0000-\\uFFFF]]"; /* all BMP letters */ UnicodeSet set = null; try { set = new UnicodeSet(goodPattern); try { Transliterator t = Transliterator.CreateFromRules(id, rules, Transliterator.REVERSE); t.Filter = (set); Transliterator.RegisterClass(id, t.GetType(), null); Transliterator.Unregister(id); try { Transliterator.GetInstance(id, Transliterator.REVERSE); Errln("FAIL: construction of unregistered ID should have failed."); } catch (ArgumentException e) { } } catch (ArgumentException e) { Errln("FAIL: Was not able to create a good RBT to test registration."); } } catch (ArgumentException e) { Errln("FAIL: Was not able to create a good UnicodeSet based on valid patterns."); return; } }
public void TestPiecemeal() { String hangul = "\uBC0F"; String jamo = nameToJamo("(Mi)(I)(Cf)"); String latin = "mic"; String latin2 = "mich"; Transliterator t = null; t = Transliterator.GetInstance("NFD"); // was Hangul-Jamo TransliteratorTest.Expect(t, hangul, jamo); t = Transliterator.GetInstance("NFC"); // was Jamo-Hangul TransliteratorTest.Expect(t, jamo, hangul); t = Transliterator.GetInstance("Latin-Jamo"); TransliteratorTest.Expect(t, latin, jamo); t = Transliterator.GetInstance("Jamo-Latin"); TransliteratorTest.Expect(t, jamo, latin2); t = Transliterator.GetInstance("Hangul-Latin"); TransliteratorTest.Expect(t, hangul, latin2); t = Transliterator.GetInstance("Latin-Hangul"); TransliteratorTest.Expect(t, latin, hangul); t = Transliterator.GetInstance("Hangul-Latin; Latin-Jamo"); TransliteratorTest.Expect(t, hangul, jamo); t = Transliterator.GetInstance("Jamo-Latin; Latin-Hangul"); TransliteratorTest.Expect(t, jamo, hangul); t = Transliterator.GetInstance("Hangul-Latin; Latin-Hangul"); TransliteratorTest.Expect(t, hangul, hangul); }
internal void Check(String transliteratorName, String test, String shouldProduceStyles) { TestReplaceable tr = new TestReplaceable(test, null); String original = tr.ToString(); Transliterator t; if (transliteratorName.StartsWith("*", StringComparison.Ordinal)) { transliteratorName = transliteratorName.Substring(1); t = Transliterator.CreateFromRules("test", transliteratorName, Transliterator.FORWARD); } else { t = Transliterator.GetInstance(transliteratorName); } t.Transliterate(tr); String newStyles = tr.GetStyles(); if (!newStyles.Equals(shouldProduceStyles)) { Errln("FAIL Styles: " + transliteratorName + " ( " + original + " ) => " + tr.ToString() + "; should be {" + shouldProduceStyles + "}!"); } else { Logln("OK: " + transliteratorName + " ( " + original + " ) => " + tr.ToString()); } if (!tr.HasMetaData || tr.Chars.HasMetaData || tr.Styles.HasMetaData) { Errln("Fail hasMetaData()"); } }
public void TestScripts() { // get a couple of characters of each script for testing StringBuffer testBuffer = new StringBuffer(); for (int script = 0; script < UScript.CodeLimit; ++script) { UnicodeSet test = new UnicodeSet().ApplyPropertyAlias("script", UScript.GetName(script)); int count = Math.Min(20, test.Count); for (int i = 0; i < count; ++i) { testBuffer.Append(UTF16.ValueOf(test[i])); } } { String test = testBuffer.ToString(); Logln("Test line: " + test); int inclusion = TestFmwk.GetExhaustiveness(); bool testedUnavailableScript = false; for (int script = 0; script < UScript.CodeLimit; ++script) { if (script == UScript.Common || script == UScript.Inherited) { continue; } // if the inclusion rate is not 10, skip all but a small number of items. // Make sure, however, that we test at least one unavailable script if (inclusion < 10 && script != UScript.Latin && script != UScript.Han && script != UScript.Hiragana && testedUnavailableScript ) { continue; } String scriptName = UScript.GetName(script); // long name UCultureInfo locale = new UCultureInfo(scriptName); if (locale.Language.Equals("new") || locale.Language.Equals("pau")) { if (logKnownIssue("11171", "long script name loosely looks like a locale ID with a known likely script")) { continue; } } Transliterator t; try { t = Transliterator.GetInstance("any-" + scriptName); } catch (Exception e) { testedUnavailableScript = true; Logln("Skipping unavailable: " + scriptName); continue; // we don't handle all scripts } Logln("Checking: " + scriptName); if (t != null) { t.Transform(test); // just verify we don't crash } String shortScriptName = UScript.GetShortName(script); // 4-letter script code try { t = Transliterator.GetInstance("any-" + shortScriptName); } catch (Exception e) { Errln("Transliterator.GetInstance() worked for \"any-" + scriptName + "\" but not for \"any-" + shortScriptName + '\"'); } t.Transform(test); // just verify we don't crash } } }
public void TestEmptyTerm() { using (Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer tokenizer = new KeywordTokenizer(reader); return(new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, Transliterator.GetInstance("Any-Latin")))); })) { CheckOneTerm(a, "", ""); } }
public void TestConstruction() { Logln("Testing the construction of the compound Transliterator"); String[] names = { "Greek-Latin", "Latin-Devanagari", "Devanagari-Latin", "Latin-Greek" }; try { Transliterator.GetInstance(names[0]); Transliterator.GetInstance(names[1]); Transliterator.GetInstance(names[2]); Transliterator.GetInstance(names[3]); } catch (ArgumentException ex) { Errln("FAIL: Transliterator construction failed" + ex.ToString()); throw ex; } string[] IDs = { names[0], names[0] + ";" + names[3], names[3] + ";" + names[1] + ";" + names[2], names[0] + ";" + names[1] + ";" + names[2] + ";" + names[3] }; for (int i = 0; i < 4; i++) { try { Transliterator.GetInstance(IDs[i]); } catch (ArgumentException ex1) { Errln("FAIL: construction using CompoundTransliterator(String ID) failed for " + IDs[i]); throw ex1; } try { Transliterator.GetInstance(IDs[i], Transliterator.Forward); } catch (ArgumentException ex2) { Errln("FAIL: construction using CompoundTransliterator(String ID, int direction=FORWARD) failed for " + IDs[i]); throw ex2; } try { Transliterator.GetInstance(IDs[i], Transliterator.Reverse); } catch (ArgumentException ex3) { Errln("FAIL: construction using CompoundTransliterator(String ID, int direction=REVERSE) failed for " + IDs[i]); throw ex3; } // try{ // CompoundTransliterator cpdtrans=new CompoundTransliterator(IDs[i], Transliterator.FORWARD, null); // cpdtrans = null; // }catch(IllegalArgumentException ex4) { // Errln("FAIL: construction using CompoundTransliterator(String ID, int direction=FORWARD," + // "UnicodeFilter adoptedFilter=0) failed for " + IDs[i]); // throw ex4; // } // // // try{ // CompoundTransliterator cpdtrans2=new CompoundTransliterator(transarray[i], null); // cpdtrans2 = null; // }catch(IllegalArgumentException ex5) { // Errln("FAIL: Construction using CompoundTransliterator(Transliterator transliterators[]," + // "UnicodeFilter adoptedFilter = 0) failed"); // throw ex5; // } } }
public void TestTransliterate() { Logln("Testing the handleTransliterate() API of CompoundTransliterator"); Transliterator ct1 = null; try { ct1 = Transliterator.GetInstance("Any-Hex;Hex-Any"); } catch (ArgumentException iae) { Errln("FAIL: construction using CompoundTransliterator(String ID) failed for " + "Any-Hex;Hex-Any"); throw iae; } String s = "abcabc"; expect(ct1, s, s); TransliterationPosition index = new TransliterationPosition(); ReplaceableString rsource2 = new ReplaceableString(s); String expectedResult = s; ct1.Transliterate(rsource2, index); ct1.FinishTransliteration(rsource2, index); String result = rsource2.ToString(); expectAux(ct1.ID + ":ReplaceableString, index(0,0,0,0)", s + "->" + rsource2, result.Equals(expectedResult), expectedResult); TransliterationPosition index2 = new TransliterationPosition(1, 3, 2, 3); ReplaceableString rsource3 = new ReplaceableString(s); ct1.Transliterate(rsource3, index2); ct1.FinishTransliteration(rsource3, index2); result = rsource3.ToString(); expectAux(ct1.ID + ":String, index2(1,2,2,3)", s + "->" + rsource3, result.Equals(expectedResult), expectedResult); String[] Data = { //ID, input string, transliterated string "Any-Hex;Hex-Any;Any-Hex", "hello", "\\u0068\\u0065\\u006C\\u006C\\u006F", "Any-Hex;Hex-Any", "hello! How are you?", "hello! How are you?", "Devanagari-Latin;Latin-Devanagari", "\u092D\u0948'\u0930'\u0935", "\u092D\u0948\u0930\u0935", // quotes lost "Latin-Cyrillic;Cyrillic-Latin", "a'b'k'd'e'f'g'h'i'j'Shch'shch'zh'h", "a'b'k'd'e'f'g'h'i'j'Shch'shch'zh'h", "Latin-Greek;Greek-Latin", "ABGabgAKLMN", "ABGabgAKLMN", //"Latin-Arabic;Arabic-Latin", "Ad'r'a'b'i'k'dh'dd'gh", "Adrabikdhddgh", "Hiragana-Katakana", "\u3041\u308f\u3099\u306e\u304b\u3092\u3099", "\u30A1\u30f7\u30ce\u30ab\u30fa", "Hiragana-Katakana;Katakana-Hiragana", "\u3041\u308f\u3099\u306e\u304b\u3051", "\u3041\u308f\u3099\u306e\u304b\u3051", "Katakana-Hiragana;Hiragana-Katakana", "\u30A1\u30f7\u30ce\u30f5\u30f6", "\u30A1\u30f7\u30ce\u30ab\u30b1", "Latin-Katakana;Katakana-Latin", "vavivuvevohuzizuzoninunasesuzezu", "vavivuvevohuzizuzoninunasesuzezu", }; Transliterator ct2 = null; for (int i = 0; i < Data.Length; i += 3) { try { ct2 = Transliterator.GetInstance(Data[i + 0]); } catch (ArgumentException iae2) { Errln("FAIL: CompoundTransliterator construction failed for " + Data[i + 0]); throw iae2; } expect(ct2, Data[i + 1], Data[i + 2]); } }
public void TestJamo() { Transliterator latinJamo = Transliterator.GetInstance("Latin-Jamo"); Transliterator jamoLatin = latinJamo.GetInverse(); String[] CASE = { // Column 1 is the latin text L1 to be fed to Latin-Jamo // to yield output J. // Column 2 is expected value of J. J is fed to // Jamo-Latin to yield output L2. // Column 3 is expected value of L2. If the expected // value of L2 is L1, then L2 is null. // add tests for the update to fix problems where it didn't follow the standard // see also http://www.unicode.org/cldr/data/charts/transforms/Latin-Hangul.html "gach", "(Gi)(A)(Cf)", null, "geumhui", "(Gi)(EU)(Mf)(Hi)(YI)", null, "choe", "(Ci)(OE)", null, "wo", "(IEUNG)(WEO)", null, "Wonpil", "(IEUNG)(WEO)(Nf)(Pi)(I)(L)", "wonpil", "GIPPEUM", "(Gi)(I)(BB)(EU)(Mf)", "gippeum", "EUTTEUM", "(IEUNG)(EU)(DD)(EU)(Mf)", "eutteum", "KKOTNAE", "(GGi)(O)(Tf)(Ni)(AE)", "kkotnae", "gaga", "(Gi)(A)(Gi)(A)", null, "gag-a", "(Gi)(A)(Gf)(IEUNG)(A)", null, "gak-ka", "(Gi)(A)(Kf)(Ki)(A)", null, "gakka", "(Gi)(A)(GGi)(A)", null, "gakk-a", "(Gi)(A)(GGf)(IEUNG)(A)", null, "gakkka", "(Gi)(A)(GGf)(Ki)(A)", null, "gak-kka", "(Gi)(A)(Kf)(GGi)(A)", null, "bab", "(Bi)(A)(Bf)", null, "babb", "(Bi)(A)(Bf)(Bi)(EU)", "babbeu", "babbba", "(Bi)(A)(Bf)(Bi)(EU)(Bi)(A)", "babbeuba", "bagg", "(Bi)(A)(Gf)(Gi)(EU)", "baggeu", "baggga", "(Bi)(A)(Gf)(Gi)(EU)(Gi)(A)", "baggeuga", //"bag"+SEP+"gga", "(Bi)(A)(Gf)"+SEP+"(Gi)(EU)(Gi)(A)", "bag"+SEP+"geuga", "kabsa", "(Ki)(A)(Bf)(Si)(A)", null, "kabska", "(Ki)(A)(BS)(Ki)(A)", null, "gabsbka", "(Gi)(A)(BS)(Bi)(EU)(Ki)(A)", "gabsbeuka", // not (Kf) "gga", "(Gi)(EU)(Gi)(A)", "geuga", "bsa", "(Bi)(EU)(Si)(A)", "beusa", "agg", "(IEUNG)(A)(Gf)(Gi)(EU)", "aggeu", "agga", "(IEUNG)(A)(Gf)(Gi)(A)", null, "la", "(R)(A)", null, "bs", "(Bi)(EU)(Sf)", "beus", "kalgga", "(Ki)(A)(L)(Gi)(EU)(Gi)(A)", "kalgeuga", // 'r' in a final position is treated like 'l' "karka", "(Ki)(A)(L)(Ki)(A)", "kalka", }; for (int i = 0; i < CASE.Length; i += 3) { String jamo = nameToJamo(CASE[i + 1]); if (CASE[i + 2] == null) { TransliteratorTest.Expect(latinJamo, CASE[i], jamo, jamoLatin); } else { // Handle case where round-trip is expected to fail TransliteratorTest.Expect(latinJamo, CASE[i], jamo); TransliteratorTest.Expect(jamoLatin, jamo, CASE[i + 2]); } } }
/// <summary> /// Returns a transliterator from the given source to our target or /// target/variant. Returns NULL if the source is the same as our /// target script, or if the source is <see cref="UScript.InvalidCode"/>. /// Caches the result and returns the same transliterator the next /// time. The caller does NOT own the result and must not delete /// it. /// </summary> private Transliterator GetTransliterator(int source) { if (source == targetScript || source == UScript.InvalidCode) { if (IsWide(targetScript)) { return(null); } else { return(widthFix); } } int key = (int)source; Transliterator t = cache.Get(key); if (!cache.TryGetValue(key, out t) || t == null) { string sourceName = UScript.GetName(source); string id = sourceName + TARGET_SEP + target; try { t = Transliterator.GetInstance(id, FORWARD); } catch (Exception e) { } if (t == null) { // Try to pivot around Latin, our most common script id = sourceName + LATIN_PIVOT + target; try { t = Transliterator.GetInstance(id, FORWARD); } catch (Exception e) { } } if (t != null) { if (!IsWide(targetScript)) { IList <Transliterator> v = new List <Transliterator>(); v.Add(widthFix); v.Add(t); t = new CompoundTransliterator(v); } //Transliterator prevCachedT = cache.putIfAbsent(key, t); Transliterator prevCachedT; // ICU4N: This is to simulate putIfAbsent // ICU4N TODO: If this works, make it into a PutIfAbsent extension method so we can go back to using ConcurrentDictionary elsewhere if (!cache.TryGetValue(key, out prevCachedT)) { // If another thread beat us here, set the prevCachedT // value to NullTransliterator to indicate it already exists if (!cache.TryAdd(key, t)) { prevCachedT = new NullTransliterator(); } } if (prevCachedT != null) { t = prevCachedT; } } else if (!IsWide(targetScript)) { return(widthFix); } } return(t); }
public void TestOptimizer2() { CheckToken(Transliterator.GetInstance("Traditional-Simplified; CaseFold"), "ABCDE", "abcde"); }
public void TestRealText() { Transliterator latinJamo = Transliterator.GetInstance("Latin-Jamo"); Transliterator jamoLatin = latinJamo.GetInverse(); Transliterator jamoHangul = Transliterator.GetInstance("NFC"); Transliterator hangulJamo = Transliterator.GetInstance("NFD"); //Transliterator rt = new CompoundTransliterator(new Transliterator[] { // hangulJamo, jamoLatin, latinJamo, jamoHangul }); Transliterator rt = Transliterator.GetInstance("NFD;Jamo-Latin;Latin-Jamo;NFC"); int pos = 0; StringBuffer buf = new StringBuffer(); int total = 0; int errors = 0; while (pos < WHAT_IS_UNICODE.Length) { int space = WHAT_IS_UNICODE.IndexOf(' ', pos + 1); if (space < 0) { space = WHAT_IS_UNICODE.Length; } if (pos < space) { ++total; String hangul = WHAT_IS_UNICODE.Substring(pos, space - pos); // ICU4N: Corrected 2nd parameter String hangulX = rt.Transliterate(hangul); if (!hangul.Equals(hangulX)) { ++errors; String jamo = hangulJamo.Transliterate(hangul); String latin = jamoLatin.Transliterate(jamo); String jamo2 = latinJamo.Transliterate(latin); String hangul2 = jamoHangul.Transliterate(jamo2); if (hangul.Equals(hangul2)) { buf.Length = (0); buf.Append("FAIL (Compound transliterator problem): "); buf.Append(hangul + " => " + jamoToName(jamo) + " => " + latin + " => " + jamoToName(jamo2) + " => " + hangul2 + "; but " + hangul + " =cpd=> " + jamoToName(hangulX) ); Errln(Utility.Escape(buf.ToString())); } else if (jamo.Equals(jamo2)) { buf.Length = (0); buf.Append("FAIL (Jamo<>Hangul problem): "); if (!hangul2.Equals(hangulX)) { buf.Append("(Weird: " + hangulX + " != " + hangul2 + ")"); } buf.Append(hangul + " => " + jamoToName(jamo) + " => " + latin + " => " + jamoToName(jamo2) + " => " + hangul2 ); Errln(Utility.Escape(buf.ToString())); } else { buf.Length = (0); buf.Append("FAIL: "); if (!hangul2.Equals(hangulX)) { buf.Append("(Weird: " + hangulX + " != " + hangul2 + ")"); } // The Hangul-Jamo conversion is not usually the // bug here, so we hide it from display. // Uncomment lines to see the Hangul. buf.Append(//hangul + " => " + jamoToName(jamo) + " => " + latin + " => " + jamoToName(jamo2) //+ " => " + hangul2 ); Errln(Utility.Escape(buf.ToString())); } } } pos = space + 1; } if (errors != 0) { Errln("Test word failures: " + errors + " out of " + total); } else { Logln("All " + total + " test words passed"); } }
public void TestTransliteratorErrors() { String trans = "Latin-Greek"; String bogusID = "LATINGREEK-GREEKLATIN"; String newID = "Bogus-Latin"; String newIDRules = "zzz > Z; f <> ph"; String bogusRules = "a } [b-g m-p "; ReplaceableString testString = new ReplaceableString("A quick fox jumped over the lazy dog."); String insertString = "cats and dogs"; int stoppedAt = 0, len; Transliterator.Position pos = new Transliterator.Position(); Transliterator t = Transliterator.GetInstance(trans, Transliterator.FORWARD); if (t == null) { Errln("FAIL: construction of Latin-Greek"); return; } len = testString.Length; stoppedAt = t.Transliterate(testString, 0, 100); if (stoppedAt != -1) { Errln("FAIL: Out of bounds check failed (1)."); } else if (testString.Length != len) { testString = new ReplaceableString("A quick fox jumped over the lazy dog."); Errln("FAIL: Transliterate fails and the target string was modified."); } stoppedAt = t.Transliterate(testString, 100, testString.Length - 1); if (stoppedAt != -1) { Errln("FAIL: Out of bounds check failed (2)."); } else if (testString.Length != len) { testString = new ReplaceableString("A quick fox jumped over the lazy dog."); Errln("FAIL: Transliterate fails and the target string was modified."); } pos.Start = 100; pos.Limit = testString.Length; try { t.Transliterate(testString, pos); Errln("FAIL: Start offset is out of bounds, error not reported."); } catch (ArgumentException e) { Logln("Start offset is out of bounds and detected."); } pos.Limit = 100; pos.Start = 0; try { t.Transliterate(testString, pos); Errln("FAIL: Limit offset is out of bounds, error not reported.\n"); } catch (ArgumentException e) { Logln("Start offset is out of bounds and detected."); } len = pos.ContextLimit = testString.Length; pos.ContextStart = 0; pos.Limit = len - 1; pos.Start = 5; try { t.Transliterate(testString, pos, insertString); if (len == pos.Limit) { Errln("FAIL: Test insertion with string: the transliteration position limit didn't change as expected."); } } catch (ArgumentException e) { Errln("Insertion test with string failed for some reason."); } pos.ContextStart = 0; pos.ContextLimit = testString.Length; pos.Limit = testString.Length - 1; pos.Start = 5; try { t.Transliterate(testString, pos, 0x0061); if (len == pos.Limit) { Errln("FAIL: Test insertion with character: the transliteration position limit didn't change as expected."); } } catch (ArgumentException e) { Errln("FAIL: Insertion test with UTF-16 code point failed for some reason."); } len = pos.Limit = testString.Length; pos.ContextStart = 0; pos.ContextLimit = testString.Length - 1; pos.Start = 5; try { t.Transliterate(testString, pos, insertString); Errln("FAIL: Out of bounds check failed (3)."); if (testString.Length != len) { Errln("FAIL: The input string was modified though the offsets were out of bounds."); } } catch (ArgumentException e) { Logln("Insertion test with out of bounds indexes."); } Transliterator t1 = null; try { t1 = Transliterator.GetInstance(bogusID, Transliterator.FORWARD); if (t1 != null) { Errln("FAIL: construction of bogus ID \"LATINGREEK-GREEKLATIN\""); } } catch (ArgumentException e) { } //try { // unneeded - Exception cannot be thrown Transliterator t2 = Transliterator.CreateFromRules( newID, newIDRules, Transliterator.FORWARD); try { Transliterator t3 = t2.GetInverse(); Errln("FAIL: The newID transliterator was not registered so createInverse should fail."); if (t3 != null) { Errln("FAIL: The newID transliterator was not registered so createInverse should fail."); } } catch (Exception e) { } //} catch (Exception e) { } try { Transliterator t4 = Transliterator.CreateFromRules( newID, bogusRules, Transliterator.FORWARD); if (t4 != null) { Errln("FAIL: The rules is malformed but error was not reported."); } } catch (Exception e) { } }