// Was testTrieRanges in ICU4C. Renamed to not conflict with ICU4J test framework.
private void checkTrieRanges(String testName, String serializedName, bool withClone,
int[][] setRanges, int[][] checkRanges)
{
string ns =
#if FEATURE_TYPEEXTENSIONS_GETTYPEINFO
typeof(Trie2Test).GetTypeInfo().Namespace;
#else
typeof(Trie2Test).Namespace;
#endif
// Run tests against Tries that were built by ICU4C and serialized.
String fileName16 = ns + ".Trie2Test." + serializedName + ".16.tri2";
String fileName32 = ns + ".Trie2Test." + serializedName + ".32.tri2";
#if FEATURE_TYPEEXTENSIONS_GETTYPEINFO
Assembly assembly = typeof(Trie2Test).GetTypeInfo().Assembly;
#else
Assembly assembly = typeof(Trie2Test).Assembly;
#endif
Stream @is = assembly.GetManifestResourceStream(fileName16);
Trie2 trie16;
try
{
trie16 = Trie2.CreateFromSerialized(ICUBinary.GetByteBufferFromStreamAndDisposeStream(@is));
}
finally
{
@is.Dispose();
}
trieGettersTest(testName, trie16, checkRanges);
@is = assembly.GetManifestResourceStream(fileName32);
Trie2 trie32;
try
{
trie32 = Trie2.CreateFromSerialized(ICUBinary.GetByteBufferFromStreamAndDisposeStream(@is));
}
finally
{
@is.Dispose();
}
trieGettersTest(testName, trie32, checkRanges);
// Run the same tests against locally contructed Tries.
Trie2Writable trieW = genTrieFromSetRanges(setRanges);
trieGettersTest(testName, trieW, checkRanges);
assertEquals("", trieW, trie16); // Locally built tries must be
assertEquals("", trieW, trie32); // the same as those imported from ICU4C
Trie2_32 trie32a = trieW.ToTrie2_32();
trieGettersTest(testName, trie32a, checkRanges);
Trie2_16 trie16a = trieW.ToTrie2_16();
trieGettersTest(testName, trie16a, checkRanges);
}
/// <summary>
/// Get an <see cref="RBBIDataWrapper"/> from an InputStream onto a pre-compiled set
/// of RBBI rules.
/// </summary>
/// <param name="bytes"></param>
/// <returns></returns>
internal static RBBIDataWrapper Get(ByteBuffer bytes)
{
RBBIDataWrapper This = new RBBIDataWrapper();
ICUBinary.ReadHeader(bytes, DATA_FORMAT, IS_ACCEPTABLE);
This.isBigEndian = bytes.Order == ByteOrder.BigEndian;
// Read in the RBBI data header...
This.fHeader = new RBBIDataHeader();
This.fHeader.fMagic = bytes.GetInt32();
This.fHeader.fFormatVersion[0] = bytes.Get();
This.fHeader.fFormatVersion[1] = bytes.Get();
This.fHeader.fFormatVersion[2] = bytes.Get();
This.fHeader.fFormatVersion[3] = bytes.Get();
This.fHeader.fLength = bytes.GetInt32();
This.fHeader.fCatCount = bytes.GetInt32();
This.fHeader.fFTable = bytes.GetInt32();
This.fHeader.fFTableLen = bytes.GetInt32();
This.fHeader.fRTable = bytes.GetInt32();
This.fHeader.fRTableLen = bytes.GetInt32();
This.fHeader.fSFTable = bytes.GetInt32();
This.fHeader.fSFTableLen = bytes.GetInt32();
This.fHeader.fSRTable = bytes.GetInt32();
This.fHeader.fSRTableLen = bytes.GetInt32();
This.fHeader.fTrie = bytes.GetInt32();
This.fHeader.fTrieLen = bytes.GetInt32();
This.fHeader.fRuleSource = bytes.GetInt32();
This.fHeader.fRuleSourceLen = bytes.GetInt32();
This.fHeader.fStatusTable = bytes.GetInt32();
This.fHeader.fStatusTableLen = bytes.GetInt32();
ICUBinary.SkipBytes(bytes, 6 * 4); // uint32_t fReserved[6];
if (This.fHeader.fMagic != 0xb1a0 || !IS_ACCEPTABLE.IsDataVersionAcceptable(This.fHeader.fFormatVersion))
{
throw new IOException("Break Iterator Rule Data Magic Number Incorrect, or unsupported data version.");
}
// Current position in the buffer.
int pos = 24 * 4; // offset of end of header, which has 24 fields, all int32_t (4 bytes)
//
// Read in the Forward state transition table as an array of shorts.
//
// Quick Sanity Check
if (This.fHeader.fFTable < pos || This.fHeader.fFTable > This.fHeader.fLength)
{
throw new IOException("Break iterator Rule data corrupt");
}
// Skip over any padding preceding this table
ICUBinary.SkipBytes(bytes, This.fHeader.fFTable - pos);
pos = This.fHeader.fFTable;
This.fFTable = ICUBinary.GetInt16s(
bytes, This.fHeader.fFTableLen / 2, This.fHeader.fFTableLen & 1);
pos += This.fHeader.fFTableLen;
//
// Read in the Reverse state table
//
// Skip over any padding in the file
ICUBinary.SkipBytes(bytes, This.fHeader.fRTable - pos);
pos = This.fHeader.fRTable;
// Create & fill the table itself.
This.fRTable = ICUBinary.GetInt16s(
bytes, This.fHeader.fRTableLen / 2, This.fHeader.fRTableLen & 1);
pos += This.fHeader.fRTableLen;
//
// Read in the Safe Forward state table
//
if (This.fHeader.fSFTableLen > 0)
{
// Skip over any padding in the file
ICUBinary.SkipBytes(bytes, This.fHeader.fSFTable - pos);
pos = This.fHeader.fSFTable;
// Create & fill the table itself.
This.fSFTable = ICUBinary.GetInt16s(
bytes, This.fHeader.fSFTableLen / 2, This.fHeader.fSFTableLen & 1);
pos += This.fHeader.fSFTableLen;
}
//
// Read in the Safe Reverse state table
//
if (This.fHeader.fSRTableLen > 0)
{
// Skip over any padding in the file
ICUBinary.SkipBytes(bytes, This.fHeader.fSRTable - pos);
pos = This.fHeader.fSRTable;
// Create & fill the table itself.
This.fSRTable = ICUBinary.GetInt16s(
bytes, This.fHeader.fSRTableLen / 2, This.fHeader.fSRTableLen & 1);
pos += This.fHeader.fSRTableLen;
}
// Rule Compatibility Hacks
// If a rule set includes reverse rules but does not explicitly include safe reverse rules,
// the reverse rules are to be treated as safe reverse rules.
if (This.fSRTable == null && This.fRTable != null)
{
This.fSRTable = This.fRTable;
This.fRTable = null;
}
//
// Unserialize the Character categories TRIE
// Because we can't be absolutely certain where the Trie deserialize will
// leave the buffer, leave position unchanged.
// The seek to the start of the next item following the TRIE will get us
// back in sync.
//
ICUBinary.SkipBytes(bytes, This.fHeader.fTrie - pos); // seek buffer from end of
pos = This.fHeader.fTrie; // previous section to the start of the trie
bytes.Mark(); // Mark position of start of TRIE in the input
// and tell Java to keep the mark valid so long
// as we don't go more than 100 bytes past the
// past the end of the TRIE.
This.fTrie = Trie2.CreateFromSerialized(bytes); // Deserialize the TRIE, leaving buffer
// at an unknown position, preceding the
// padding between TRIE and following section.
bytes.Reset(); // Move buffer back to marked position at
// the start of the serialized TRIE. Now our
// "pos" variable and the buffer are in
// agreement.
//
// Read the Rule Status Table
//
if (pos > This.fHeader.fStatusTable)
{
throw new IOException("Break iterator Rule data corrupt");
}
ICUBinary.SkipBytes(bytes, This.fHeader.fStatusTable - pos);
pos = This.fHeader.fStatusTable;
This.fStatusTable = ICUBinary.GetInt32s(
bytes, This.fHeader.fStatusTableLen / 4, This.fHeader.fStatusTableLen & 3);
pos += This.fHeader.fStatusTableLen;
//
// Put the break rule source into a String
//
if (pos > This.fHeader.fRuleSource)
{
throw new IOException("Break iterator Rule data corrupt");
}
ICUBinary.SkipBytes(bytes, This.fHeader.fRuleSource - pos);
pos = This.fHeader.fRuleSource;
This.fRuleSource = ICUBinary.GetString(
bytes, This.fHeader.fRuleSourceLen / 2, This.fHeader.fRuleSourceLen & 1);
if (RuleBasedBreakIterator.fDebugEnv != null && RuleBasedBreakIterator.fDebugEnv.IndexOf("data", StringComparison.Ordinal) >= 0)
{
This.Dump(Console.Out);
}
return(This);
}
public void TestTrie2API()
{
// Trie2.createFromSerialized()
// This function is well exercised by TestRanges().
// Trie2.getVersion(InputStream is, boolean anyEndianOk)
//
try
{
Trie2Writable trie = new Trie2Writable(0, 0);
MemoryStream os = new MemoryStream();
trie.ToTrie2_16().Serialize(os);
MemoryStream @is = new MemoryStream(os.ToArray());
assertEquals(null, 2, Trie2.GetVersion(@is, true));
}
catch (IOException e)
{
Errln(where () + e.ToString());
}
// Equals & hashCode
//
{
Trie2Writable trieWA = new Trie2Writable(0, 0);
Trie2Writable trieWB = new Trie2Writable(0, 0);
Trie2 trieA = trieWA;
Trie2 trieB = trieWB;
assertTrue("", trieA.Equals(trieB));
assertEquals("", trieA, trieB);
assertEquals("", trieA.GetHashCode(), trieB.GetHashCode());
trieWA.Set(500, 2);
assertNotEquals("", trieA, trieB);
// Note that the hash codes do not strictly need to be different,
// but it's highly likely that something is wrong if they are the same.
assertNotEquals("", trieA.GetHashCode(), trieB.GetHashCode());
trieWB.Set(500, 2);
trieA = trieWA.ToTrie2_16();
assertEquals("", trieA, trieB);
assertEquals("", trieA.GetHashCode(), trieB.GetHashCode());
}
//
// Iterator creation
//
{
Trie2Writable trie = new Trie2Writable(17, 0);
IEnumerator <Trie2Range> it;
using (it = trie.GetEnumerator())
{
it.MoveNext();
Trie2Range r = it.Current;
assertEquals("", 0, r.StartCodePoint);
assertEquals("", 0x10ffff, r.EndCodePoint);
assertEquals("", 17, r.Value);
assertEquals("", false, r.IsLeadSurrogate);
it.MoveNext();
r = it.Current;
assertEquals("", 0xd800, r.StartCodePoint);
assertEquals("", 0xdbff, r.EndCodePoint);
assertEquals("", 17, r.Value);
assertEquals("", true, r.IsLeadSurrogate);
int i = 0;
foreach (Trie2Range rr in trie)
{
switch (i)
{
case 0:
assertEquals("", 0, rr.StartCodePoint);
assertEquals("", 0x10ffff, rr.EndCodePoint);
assertEquals("", 17, rr.Value);
assertEquals("", false, rr.IsLeadSurrogate);
break;
case 1:
assertEquals("", 0xd800, rr.StartCodePoint);
assertEquals("", 0xdbff, rr.EndCodePoint);
assertEquals("", 17, rr.Value);
assertEquals("", true, rr.IsLeadSurrogate);
break;
default:
Errln(where () + " Unexpected iteration result");
break;
}
i++;
}
}
}
// Iteration with a value mapping function
//
{
Trie2Writable trie = new Trie2Writable(0xbadfeed, 0);
trie.Set(0x10123, 42);
IValueMapper vm = new Trie2ValueMapper(map: (v) =>
{
if (v == 0xbadfeed)
{
v = 42;
}
return(v);
});
using (IEnumerator <Trie2Range> it2 = trie.GetEnumerator(vm))
{
it2.MoveNext();
Trie2Range r = it2.Current;
assertEquals("", 0, r.StartCodePoint);
assertEquals("", 0x10ffff, r.EndCodePoint);
assertEquals("", 42, r.Value);
assertEquals("", false, r.IsLeadSurrogate);
}
}
// Iteration over a leading surrogate range.
//
{
Trie2Writable trie = new Trie2Writable(0xdefa17, 0);
trie.Set(0x2f810, 10);
using (IEnumerator <Trie2Range> it = trie.GetEnumeratorForLeadSurrogate((char)0xd87e))
{
it.MoveNext();
Trie2Range r = it.Current;
assertEquals("", 0x2f800, r.StartCodePoint);
assertEquals("", 0x2f80f, r.EndCodePoint);
assertEquals("", 0xdefa17, r.Value);
assertEquals("", false, r.IsLeadSurrogate);
it.MoveNext();
r = it.Current;
assertEquals("", 0x2f810, r.StartCodePoint);
assertEquals("", 0x2f810, r.EndCodePoint);
assertEquals("", 10, r.Value);
assertEquals("", false, r.IsLeadSurrogate);
it.MoveNext();
r = it.Current;
assertEquals("", 0x2f811, r.StartCodePoint);
assertEquals("", 0x2fbff, r.EndCodePoint);
assertEquals("", 0xdefa17, r.Value);
assertEquals("", false, r.IsLeadSurrogate);
assertFalse("", it.MoveNext());
}
}
// Iteration over a leading surrogate range with a ValueMapper.
//
{
Trie2Writable trie = new Trie2Writable(0xdefa17, 0);
trie.Set(0x2f810, 10);
IValueMapper m = new Trie2ValueMapper(map: (@in) =>
{
if (@in == 10)
{
@in = 0xdefa17;
}
return(@in);
});
using (IEnumerator <Trie2Range> it = trie.GetEnumeratorForLeadSurrogate((char)0xd87e, m))
{
it.MoveNext();
Trie2Range r = it.Current;
assertEquals("", 0x2f800, r.StartCodePoint);
assertEquals("", 0x2fbff, r.EndCodePoint);
assertEquals("", 0xdefa17, r.Value);
assertEquals("", false, r.IsLeadSurrogate);
assertFalse("", it.MoveNext());
}
}
// Trie2.serialize()
// Test the implementation in Trie2, which is used with Read Only Tries.
//
{
Trie2Writable trie = new Trie2Writable(101, 0);
trie.SetRange(0xf000, 0x3c000, 200, true);
trie.Set(0xffee, 300);
Trie2_16 frozen16 = trie.ToTrie2_16();
Trie2_32 frozen32 = trie.ToTrie2_32();
assertEquals("", trie, frozen16);
assertEquals("", trie, frozen32);
assertEquals("", frozen16, frozen32);
MemoryStream os = new MemoryStream();
try
{
frozen16.Serialize(os);
Trie2 unserialized16 = Trie2.CreateFromSerialized(ByteBuffer.Wrap(os.ToArray()));
assertEquals("", trie, unserialized16);
assertEquals("", typeof(Trie2_16), unserialized16.GetType());
os.Seek(0, SeekOrigin.Begin);
frozen32.Serialize(os);
Trie2 unserialized32 = Trie2.CreateFromSerialized(ByteBuffer.Wrap(os.ToArray()));
assertEquals("", trie, unserialized32);
assertEquals("", typeof(Trie2_32), unserialized32.GetType());
}
catch (IOException e)
{
Errln(where () + " Unexpected exception: " + e);
}
}
}
//
// Check the expected values from a single Trie2.
//
private void trieGettersTest(String testName,
Trie2 trie, // The Trie2 to test.
int[][] checkRanges) // Expected data.
// Tuples of (value, high limit code point)
// High limit is first code point following the range
// with the indicated value.
// (Structures copied from ICU4C tests.)
{
int countCheckRanges = checkRanges.Length;
int initialValue, errorValue;
int value, value2;
int start, limit;
int i, countSpecials;
countSpecials = 0; /*getSpecialValues(checkRanges, countCheckRanges, &initialValue, &errorValue);*/
errorValue = 0x0bad;
initialValue = 0;
if (checkRanges[countSpecials][0] == 0)
{
initialValue = checkRanges[countSpecials][1];
countSpecials++;
}
start = 0;
for (i = countSpecials; i < countCheckRanges; ++i)
{
limit = checkRanges[i][0];
value = checkRanges[i][1];
while (start < limit)
{
value2 = trie.Get(start);
if (value != value2)
{
// The redundant if, outside of the assert, is for speed.
// It makes a significant difference for this test.
assertEquals("wrong value for " + testName + " of " + (start).ToHexString(), value, value2);
}
++start;
}
}
if (!testName.StartsWith("dummy", StringComparison.Ordinal) && !testName.StartsWith("trie1", StringComparison.Ordinal))
{
/* Test values for lead surrogate code units.
* For non-lead-surrogate code units, getFromU16SingleLead() and get()
* should be the same.
*/
for (start = 0xd7ff; start < 0xdc01; ++start)
{
switch (start)
{
case 0xd7ff:
case 0xdc00:
value = trie.Get(start);
break;
case 0xd800:
value = 90;
break;
case 0xd999:
value = 94;
break;
case 0xdbff:
value = 99;
break;
default:
value = initialValue;
break;
}
value2 = trie.GetFromU16SingleLead((char)start);
if (value2 != value)
{
Errln(where () + " testName: " + testName + " getFromU16SingleLead() failed." +
"char, exected, actual = " + (start).ToHexString() + ", " +
(value).ToHexString() + ", " + (value2).ToHexString());
}
}
}
/* test errorValue */
value = trie.Get(-1);
value2 = trie.Get(0x110000);
if (value != errorValue || value2 != errorValue)
{
Errln("trie2.Get() error value test. Expected, actual1, actual2 = " +
errorValue + ", " + value + ", " + value2);
}
// Check that Trie enumeration produces the same contents as simple get()
foreach (Trie2Range range in trie)
{
for (int cp = range.StartCodePoint; cp <= range.EndCodePoint; cp++)
{
if (range.IsLeadSurrogate)
{
assertTrue(testName, cp >= (char)0xd800 && cp < (char)0xdc00);
assertEquals(testName, range.Value, trie.GetFromU16SingleLead((char)cp));
}
else
{
assertEquals(testName, range.Value, trie.Get(cp));
}
}
}
}
public void TestTrie2WritableAPI()
{
//
// Trie2Writable methods. Check that all functions are present and
// nominally working. Not an in-depth test.
//
// Trie2Writable constructor
Trie2 t1 = new Trie2Writable(6, 666);
// Constructor from another Trie2
Trie2 t2 = new Trie2Writable(t1);
assertTrue("", t1.Equals(t2));
// Set / Get
Trie2Writable t1w = new Trie2Writable(10, 666);
t1w.Set(0x4567, 99);
assertEquals("", 10, t1w.Get(0x4566));
assertEquals("", 99, t1w.Get(0x4567));
assertEquals("", 666, t1w.Get(-1));
assertEquals("", 666, t1w.Get(0x110000));
// SetRange
t1w = new Trie2Writable(10, 666);
t1w.SetRange(13 /*start*/, 6666 /*end*/, 7788 /*value*/, false /*overwrite */);
t1w.SetRange(6000, 7000, 9900, true);
assertEquals("", 10, t1w.Get(12));
assertEquals("", 7788, t1w.Get(13));
assertEquals("", 7788, t1w.Get(5999));
assertEquals("", 9900, t1w.Get(6000));
assertEquals("", 9900, t1w.Get(7000));
assertEquals("", 10, t1w.Get(7001));
assertEquals("", 666, t1w.Get(0x110000));
// setRange from a Trie2.Range
// (Ranges are more commonly created by iterating over a Trie2,
// but create one by hand here)
Trie2Range r = new Trie2Range();
r.StartCodePoint = 50;
r.EndCodePoint = 52;
r.Value = 0x12345678;
r.IsLeadSurrogate = false;
t1w = new Trie2Writable(0, 0xbad);
t1w.SetRange(r, true);
assertEquals(null, 0, t1w.Get(49));
assertEquals("", 0x12345678, t1w.Get(50));
assertEquals("", 0x12345678, t1w.Get(52));
assertEquals("", 0, t1w.Get(53));
// setForLeadSurrogateCodeUnit / getFromU16SingleLead
t1w = new Trie2Writable(10, 0xbad);
assertEquals("", 10, t1w.GetFromU16SingleLead((char)0x0d801));
t1w.SetForLeadSurrogateCodeUnit((char)0xd801, 5000);
t1w.Set(0xd801, 6000);
assertEquals("", 5000, t1w.GetFromU16SingleLead((char)0x0d801));
assertEquals("", 6000, t1w.Get(0x0d801));
// get(). Is covered by nearly every other test.
// Trie2_16 getAsFrozen_16()
t1w = new Trie2Writable(10, 666);
t1w.Set(42, 5555);
t1w.Set(0x1ff00, 224);
Trie2_16 t1_16 = t1w.ToTrie2_16();
assertTrue("", t1w.Equals(t1_16));
// alter the writable Trie2 and then re-freeze.
t1w.Set(152, 129);
t1_16 = t1w.ToTrie2_16();
assertTrue("", t1w.Equals(t1_16));
assertEquals("", 129, t1w.Get(152));
// Trie2_32 getAsFrozen_32()
//
t1w = new Trie2Writable(10, 666);
t1w.Set(42, 5555);
t1w.Set(0x1ff00, 224);
Trie2_32 t1_32 = t1w.ToTrie2_32();
assertTrue("", t1w.Equals(t1_32));
// alter the writable Trie2 and then re-freeze.
t1w.Set(152, 129);
assertNotEquals("", t1_32, t1w);
t1_32 = t1w.ToTrie2_32();
assertTrue("", t1w.Equals(t1_32));
assertEquals("", 129, t1w.Get(152));
// serialize(OutputStream os, ValueWidth width)
//
MemoryStream os = new MemoryStream();
t1w = new Trie2Writable(0, 0xbad);
t1w.Set(0x41, 0x100);
t1w.Set(0xc2, 0x200);
t1w.Set(0x404, 0x300);
t1w.Set(0xd903, 0x500);
t1w.Set(0xdd29, 0x600);
t1w.Set(0x1055d3, 0x700);
t1w.SetForLeadSurrogateCodeUnit((char)0xda1a, 0x800);
try
{
// Serialize to 16 bits.
int serializedLen = t1w.ToTrie2_16().Serialize(os);
// Fragile test. Serialized length could change with changes to compaction.
// But it should not change unexpectedly.
assertEquals("", 3508, serializedLen);
Trie2 t1ws16 = Trie2.CreateFromSerialized(ByteBuffer.Wrap(os.ToArray()));
assertEquals("", t1ws16.GetType(), typeof(Trie2_16));
assertEquals("", t1w, t1ws16);
// Serialize to 32 bits
os.Seek(0, SeekOrigin.Begin);
serializedLen = t1w.ToTrie2_32().Serialize(os);
// Fragile test. Serialized length could change with changes to compaction.
// But it should not change unexpectedly.
assertEquals("", 4332, serializedLen);
Trie2 t1ws32 = Trie2.CreateFromSerialized(ByteBuffer.Wrap(os.ToArray()));
assertEquals("", t1ws32.GetType(), typeof(Trie2_32));
assertEquals("", t1w, t1ws32);
}
catch (IOException e)
{
Errln(where () + e.ToString());
}
}
