private unsafe static void SetCJKReferences(string name, ref CodePointIndexer cjkIndexer, ref byte *catTable, ref byte *lv1Table, ref CodePointIndexer lv2Indexer, ref byte *lv2Table)
{
switch (name)
{
case "zh-CHS":
catTable = MSCompatUnicodeTable.cjkCHScategory;
lv1Table = MSCompatUnicodeTable.cjkCHSlv1;
cjkIndexer = MSCompatUnicodeTableUtil.CjkCHS;
break;
case "zh-CHT":
catTable = MSCompatUnicodeTable.cjkCHTcategory;
lv1Table = MSCompatUnicodeTable.cjkCHTlv1;
cjkIndexer = MSCompatUnicodeTableUtil.Cjk;
break;
case "ja":
catTable = MSCompatUnicodeTable.cjkJAcategory;
lv1Table = MSCompatUnicodeTable.cjkJAlv1;
cjkIndexer = MSCompatUnicodeTableUtil.Cjk;
break;
case "ko":
catTable = MSCompatUnicodeTable.cjkKOcategory;
lv1Table = MSCompatUnicodeTable.cjkKOlv1;
lv2Table = MSCompatUnicodeTable.cjkKOlv2;
cjkIndexer = MSCompatUnicodeTableUtil.Cjk;
lv2Indexer = MSCompatUnicodeTableUtil.Cjk;
break;
}
}
static void SetCJKReferences (string name,
ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer lv2Indexer, ref byte* lv2Table)
{
// as a part of mscorlib.dll, this invocation is
// somewhat extraneous (pointers were already assigned).
switch (name)
{
case "zh-CHS":
catTable = cjkCHScategory;
lv1Table = cjkCHSlv1;
cjkIndexer = UUtil.CjkCHS;
break;
case "zh-CHT":
catTable = cjkCHTcategory;
lv1Table = cjkCHTlv1;
cjkIndexer = UUtil.Cjk;
break;
case "ja":
catTable = cjkJAcategory;
lv1Table = cjkJAlv1;
cjkIndexer = UUtil.Cjk;
break;
case "ko":
catTable = cjkKOcategory;
lv1Table = cjkKOlv1;
lv2Table = cjkKOlv2;
cjkIndexer = UUtil.Cjk;
lv2Indexer = UUtil.Cjk;
break;
}
}
public unsafe static void FillCJK(string culture, ref CodePointIndexer cjkIndexer, ref byte *catTable, ref byte *lv1Table, ref CodePointIndexer lv2Indexer, ref byte *lv2Table)
{
object obj = MSCompatUnicodeTable.forLock;
lock (obj)
{
MSCompatUnicodeTable.FillCJKCore(culture, ref cjkIndexer, ref catTable, ref lv1Table, ref lv2Indexer, ref lv2Table);
MSCompatUnicodeTable.SetCJKReferences(culture, ref cjkIndexer, ref catTable, ref lv1Table, ref lv2Indexer, ref lv2Table);
}
}
public static void FillCJK (string name,
ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer cjkLv2Indexer,
ref byte* lv2Table)
{
SetCJKReferences (name, ref cjkIndexer,
ref catTable, ref lv1Table,
ref cjkLv2Indexer, ref lv2Table);
}
static CollationElementTableUtil ()
{
int [] starts = new int [] {
0, 0x4dc0, 0xa000, 0xf900, 0xfb00,
0x1d000, 0x2f800, 0xe0000, 0x110000};
int [] ends = new int [] {
0x3410, 0x4e10, 0xa4d0, 0xfa70, 0x10840,
0x1d800, 0x2fa20, 0xe0200, 0x110000};
Indexer = new CodePointIndexer (starts, ends, 0, 0);
}
static MSCompatUnicodeTableUtil()
{
// FIXME: those ranges could be more compact, but since
// I haven't filled all the table yet, I keep it safer.
int [] ignoreStarts = new int []
{
0, 0xA000, 0xF900
};
int [] ignoreEnds = new int []
{
0x3400, 0xA500, 0x10000
};
int [] catStarts = new int []
{
0, 0x1E00, 0x3000, 0x4E00, 0xAC00, 0xF900
};
int [] catEnds = new int []
{
0x1200, 0x2800, 0x3400, 0xA000, 0xD7B0, 0x10000
};
int [] lv1Starts = new int []
{
0, 0x1E00, 0x3000, 0x4E00, 0xAC00, 0xF900
};
int [] lv1Ends = new int []
{
0x1200, 0x2800, 0x3400, 0xA000, 0xD7B0, 0x10000
};
int [] lv2Starts = new int [] { 0, 0x1E00, 0x3000, 0xFB00 };
int [] lv2Ends = new int [] { 0xF00, 0x2800, 0x3400, 0x10000 };
int [] lv3Starts = new int [] { 0, 0x1E00, 0x3000, 0xFB00 };
int [] lv3Ends = new int [] { 0x1200, 0x2800, 0x3400, 0x10000 };
// int [] widthStarts = new int [] {0, 0x2000, 0x3100, 0xFF00};
// int [] widthEnds = new int [] {0x300, 0x2200, 0x3200, 0x10000};
int [] chsStarts = new int []
{
0x3100, 0x4E00, 0xE800
}; // FIXME: really?
int [] chsEnds = new int []
{
0x3400, 0xA000, 0x10000
};
int [] cjkStarts = new int [] { 0x3100, 0x4E00, 0xF900 };
int [] cjkEnds = new int [] { 0x3400, 0xA000, 0xFB00 };
Ignorable = new CodePointIndexer(ignoreStarts, ignoreEnds, -1, -1);
Category = new CodePointIndexer(catStarts, catEnds, 0, 0);
Level1 = new CodePointIndexer(lv1Starts, lv1Ends, 0, 0);
Level2 = new CodePointIndexer(lv2Starts, lv2Ends, 0, 0);
Level3 = new CodePointIndexer(lv3Starts, lv3Ends, 0, 0);
// WidthCompat = new CodePointIndexer (widthStarts, widthEnds, 0, 0);
CjkCHS = new CodePointIndexer(chsStarts, chsEnds, -1, -1);
Cjk = new CodePointIndexer(cjkStarts, cjkEnds, -1, -1);
}
static CollationElementTableUtil()
{
int [] starts = new int [] {
0, 0x4dc0, 0xa000, 0xf900, 0xfb00,
0x1d000, 0x2f800, 0xe0000, 0x110000
};
int [] ends = new int [] {
0x3410, 0x4e10, 0xa4d0, 0xfa70, 0x10840,
0x1d800, 0x2fa20, 0xe0200, 0x110000
};
Indexer = new CodePointIndexer(starts, ends, 0, 0);
}
public static Array CompressArray(Array source, Type type, CodePointIndexer indexer)
{
int num = 0;
for (int i = 0; i < indexer.ranges.Length; i++)
{
num += indexer.ranges[i].Count;
}
Array array = Array.CreateInstance(type, num);
for (int j = 0; j < indexer.ranges.Length; j++)
{
Array.Copy(source, indexer.ranges[j].Start, array, indexer.ranges[j].IndexStart, indexer.ranges[j].Count);
}
return(array);
}
public static void FillCJK (string culture,
ref CodePointIndexer cjkIndexer,
ref byte* catTable,
ref byte* lv1Table,
ref CodePointIndexer lv2Indexer,
ref byte* lv2Table)
{
lock (forLock) {
FillCJKCore (culture, ref cjkIndexer,
ref catTable, ref lv1Table,
ref lv2Indexer, ref lv2Table);
SetCJKReferences (culture, ref cjkIndexer,
ref catTable, ref lv1Table,
ref lv2Indexer, ref lv2Table);
}
}
public static Array CompressArray (
Array source, Type type, CodePointIndexer indexer)
{
int totalCount = 0;
for (int i = 0; i < indexer.ranges.Length; i++)
totalCount += indexer.ranges [i].Count;
Array ret = Array.CreateInstance (type, totalCount);
for (int i = 0; i < indexer.ranges.Length; i++)
Array.Copy (
source,
indexer.ranges [i].Start,
ret,
indexer.ranges [i].IndexStart,
indexer.ranges [i].Count);
return ret;
}
static MSCompatUnicodeTableUtil ()
{
// FIXME: those ranges could be more compact, but since
// I haven't filled all the table yet, I keep it safer.
int [] ignoreStarts = new int [] {
0, 0xA000, 0xF900};
int [] ignoreEnds = new int [] {
0x3400, 0xA500, 0x10000};
int [] catStarts = new int [] {
0, 0x1E00, 0x3000, 0x4E00, 0xAC00, 0xF900};
int [] catEnds = new int [] {
0x1200, 0x2800, 0x3400, 0xA000, 0xD7B0, 0x10000};
int [] lv1Starts = new int [] {
0, 0x1E00, 0x3000, 0x4E00, 0xAC00, 0xF900};
int [] lv1Ends = new int [] {
0x1200, 0x2800, 0x3400, 0xA000, 0xD7B0, 0x10000};
int [] lv2Starts = new int [] {0, 0x1E00, 0x3000, 0xFB00};
int [] lv2Ends = new int [] {0xF00, 0x2800, 0x3400, 0x10000};
int [] lv3Starts = new int [] {0, 0x1E00, 0x3000, 0xFB00};
int [] lv3Ends = new int [] {0x1200, 0x2800, 0x3400, 0x10000};
// int [] widthStarts = new int [] {0, 0x2000, 0x3100, 0xFF00};
// int [] widthEnds = new int [] {0x300, 0x2200, 0x3200, 0x10000};
int [] chsStarts = new int [] {
0x3100, 0x4E00, 0xE800}; // FIXME: really?
int [] chsEnds = new int [] {
0x3400, 0xA000, 0x10000};
int [] cjkStarts = new int [] {0x3100, 0x4E00, 0xF900};
int [] cjkEnds = new int [] {0x3400, 0xA000, 0xFB00};
Ignorable = new CodePointIndexer (ignoreStarts, ignoreEnds, -1, -1);
Category = new CodePointIndexer (catStarts, catEnds, 0, 0);
Level1 = new CodePointIndexer (lv1Starts, lv1Ends, 0, 0);
Level2 = new CodePointIndexer (lv2Starts, lv2Ends, 0, 0);
Level3 = new CodePointIndexer (lv3Starts, lv3Ends, 0, 0);
// WidthCompat = new CodePointIndexer (widthStarts, widthEnds, 0, 0);
CjkCHS = new CodePointIndexer (chsStarts, chsEnds, -1, -1);
Cjk = new CodePointIndexer (cjkStarts, cjkEnds, -1, -1);
}
public static Array CompressArray(
Array source, Type type, CodePointIndexer indexer)
{
int totalCount = 0;
for (int i = 0; i < indexer.ranges.Length; i++)
{
totalCount += indexer.ranges [i].Count;
}
Array ret = Array.CreateInstance(type, totalCount);
for (int i = 0; i < indexer.ranges.Length; i++)
{
Array.Copy(
source,
indexer.ranges [i].Start,
ret,
indexer.ranges [i].IndexStart,
indexer.ranges [i].Count);
}
return(ret);
}
public static void FillCJK (string name,
ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer cjkLv2Indexer,
ref byte* lv2Table)
{
SetCJKReferences (name, ref cjkIndexer,
ref catTable, ref lv1Table,
ref cjkLv2Indexer, ref lv2Table);
}
public static void FillCJK (string culture,
ref CodePointIndexer cjkIndexer,
ref byte* catTable,
ref byte* lv1Table,
ref CodePointIndexer lv2Indexer,
ref byte* lv2Table)
{
lock (forLock) {
FillCJKCore (culture, ref cjkIndexer,
ref catTable, ref lv1Table,
ref lv2Indexer, ref lv2Table);
SetCJKReferences (culture, ref cjkIndexer,
ref catTable, ref lv1Table,
ref lv2Indexer, ref lv2Table);
}
}
static NormalizationTableUtil()
{
int[] starts = new int[]
{
0,
2320,
6912,
9312,
10624,
11376,
11616,
11920,
42864,
63744
};
int[] ends = new int[]
{
1760,
4608,
9008,
9728,
10976,
11392,
11632,
13312,
42880,
65536
};
int[] starts2 = new int[]
{
144,
2336,
7456,
9312,
9376,
10752,
11616,
11920,
63744
};
int[] ends2 = new int[]
{
1760,
4352,
9008,
9376,
9456,
10976,
11632,
13312,
65536
};
int[] starts3 = new int[]
{
752,
1152,
1424,
2352,
2480,
2608,
2736,
2864,
3008,
3136,
3248,
3392,
3520,
3632,
3760,
3840,
4144,
4944,
5904,
6096,
6304,
6448,
6672,
7616,
8400,
12320,
12432,
43008,
64272,
65056
};
int[] ends3 = new int[]
{
864,
1168,
1872,
2400,
2512,
2640,
2768,
2896,
3024,
3168,
3280,
3408,
3536,
3664,
3792,
4048,
4160,
4960,
5952,
6112,
6320,
6464,
6688,
7632,
8432,
12336,
12448,
43024,
64288,
65072
};
int[] starts4 = new int[]
{
1152,
5136,
5744
};
int[] ends4 = new int[]
{
4224,
5504,
8624
};
int[] starts5 = new int[]
{
0,
2304,
7424,
9472,
12288,
15248,
16400,
19968,
64320
};
int[] ends5 = new int[]
{
1792,
4608,
8960,
9728,
12640,
15264,
16432,
40960,
64336
};
NormalizationTableUtil.Prop = new CodePointIndexer(starts, ends, 0, 0);
NormalizationTableUtil.Map = new CodePointIndexer(starts2, ends2, 0, 0);
NormalizationTableUtil.Combining = new CodePointIndexer(starts3, ends3, 0, 0);
NormalizationTableUtil.Composite = new CodePointIndexer(starts4, ends4, 0, 0);
NormalizationTableUtil.Helper = new CodePointIndexer(starts5, ends5, 0, 0);
}
static void FillCJKCore (string culture,
ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer cjkLv2Indexer, ref byte* lv2Table)
{
if (!IsReady)
return;
string name = null;
switch (culture) {
case "zh-CHS":
name = "cjkCHS";
catTable = cjkCHScategory;
lv1Table = cjkCHSlv1;
break;
case "zh-CHT":
name = "cjkCHT";
catTable = cjkCHTcategory;
lv1Table = cjkCHTlv1;
break;
case "ja":
name = "cjkJA";
catTable = cjkJAcategory;
lv1Table = cjkJAlv1;
break;
case "ko":
name = "cjkKO";
catTable = cjkKOcategory;
lv1Table = cjkKOlv1;
break;
}
if (name == null || lv1Table != null)
return;
byte* raw;
uint idx = 0;
#if USE_MANAGED_RESOURCE
string filename =
String.Format ("collation.{0}.bin", name);
IntPtr ptr = GetResource (filename);
if (ptr == IntPtr.Zero)
return;
raw = (byte*) ((void*) ptr);
idx += ResourceVersionSize;
#elif USE_C_HEADER
int residx = -1;
switch (culture) {
case "zh-CHS": residx = CollationTableIdxCjkCHS; break;
case "zh-CHT": residx = CollationTableIdxCjkCHT; break;
case "ja": residx = CollationTableIdxCjkJA; break;
case "ko": residx = CollationTableIdxCjkKO; break;
}
if (residx < 0)
return;
load_collation_resource (residx, &raw);
#else
int size;
int residx = -1;
switch (culture) {
case "zh-CHS": residx = CollationResourceCJKCHS; break;
case "zh-CHT": residx = CollationResourceCJKCHT; break;
case "ja": residx = CollationResourceCJKJA; break;
case "ko": residx = CollationResourceCJKKO; break;
}
if (residx < 0)
return;
load_collation_resource (corlibPath, residx, &raw, &size);
idx += ResourceVersionSize;
#endif
uint count = UInt32FromBytePtr (raw, idx);
idx += 4;
catTable = (byte*) raw + idx;
lv1Table = (byte*) raw + idx + count;
switch (culture) {
case "zh-CHS":
cjkCHScategory = catTable;
cjkCHSlv1 = lv1Table;
break;
case "zh-CHT":
cjkCHTcategory = catTable;
cjkCHTlv1 = lv1Table;
break;
case "ja":
cjkJAcategory = catTable;
cjkJAlv1 = lv1Table;
break;
case "ko":
cjkKOcategory = catTable;
cjkKOlv1 = lv1Table;
break;
}
if (name != "cjkKO")
return;
#if USE_MANAGED_RESOURCE
ptr = GetResource ("collation.cjkKOlv2.bin");
if (ptr == IntPtr.Zero)
return;
raw = (byte*) ((void*) ptr);
idx = ResourceVersionSize + 4;
#elif USE_C_HEADER
load_collation_resource (CollationTableIdxCjkKOLv2, &raw);
#else
load_collation_resource (corlibPath, CollationResourceCJKKOlv2, &raw, &size);
idx = ResourceVersionSize + 4;
#endif
cjkKOlv2 = raw + idx;
lv2Table = cjkKOlv2;
}
private unsafe static void FillCJKCore(string culture, ref CodePointIndexer cjkIndexer, ref byte *catTable, ref byte *lv1Table, ref CodePointIndexer cjkLv2Indexer, ref byte *lv2Table)
{
if (!MSCompatUnicodeTable.IsReady)
{
return;
}
string text = null;
switch (culture)
{
case "zh-CHS":
text = "cjkCHS";
catTable = MSCompatUnicodeTable.cjkCHScategory;
lv1Table = MSCompatUnicodeTable.cjkCHSlv1;
break;
case "zh-CHT":
text = "cjkCHT";
catTable = MSCompatUnicodeTable.cjkCHTcategory;
lv1Table = MSCompatUnicodeTable.cjkCHTlv1;
break;
case "ja":
text = "cjkJA";
catTable = MSCompatUnicodeTable.cjkJAcategory;
lv1Table = MSCompatUnicodeTable.cjkJAlv1;
break;
case "ko":
text = "cjkKO";
catTable = MSCompatUnicodeTable.cjkKOcategory;
lv1Table = MSCompatUnicodeTable.cjkKOlv1;
break;
}
if (text == null || lv1Table != 0)
{
return;
}
uint num2 = 0u;
string name = string.Format("collation.{0}.bin", text);
IntPtr resource = MSCompatUnicodeTable.GetResource(name);
if (resource == IntPtr.Zero)
{
return;
}
byte *ptr = (byte *)((void *)resource);
num2 += 1u;
uint num3 = MSCompatUnicodeTable.UInt32FromBytePtr(ptr, num2);
num2 += 4u;
catTable = ptr + num2;
lv1Table = ptr + num2 + num3;
switch (culture)
{
case "zh-CHS":
MSCompatUnicodeTable.cjkCHScategory = catTable;
MSCompatUnicodeTable.cjkCHSlv1 = lv1Table;
break;
case "zh-CHT":
MSCompatUnicodeTable.cjkCHTcategory = catTable;
MSCompatUnicodeTable.cjkCHTlv1 = lv1Table;
break;
case "ja":
MSCompatUnicodeTable.cjkJAcategory = catTable;
MSCompatUnicodeTable.cjkJAlv1 = lv1Table;
break;
case "ko":
MSCompatUnicodeTable.cjkKOcategory = catTable;
MSCompatUnicodeTable.cjkKOlv1 = lv1Table;
break;
}
if (text != "cjkKO")
{
return;
}
resource = MSCompatUnicodeTable.GetResource("collation.cjkKOlv2.bin");
if (resource == IntPtr.Zero)
{
return;
}
ptr = (byte *)((void *)resource);
num2 = 5u;
MSCompatUnicodeTable.cjkKOlv2 = ptr + num2;
lv2Table = MSCompatUnicodeTable.cjkKOlv2;
}
ushort [] CompressArray (ushort [] source, CodePointIndexer i)
{
return (ushort []) CodePointIndexer.CompressArray (
source, typeof (ushort), i);
}
unsafe private void SetCJKTable (
CultureInfo culture, ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer lv2Indexer, ref byte* lv2Table)
{
string name = GetNeutralCulture (culture).Name;
Uni.FillCJK (name, ref cjkIndexer, ref catTable,
ref lv1Table, ref lv2Indexer, ref lv2Table);
}
static void SetCJKReferences (string name,
ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer lv2Indexer, ref byte* lv2Table)
{
// as a part of mscorlib.dll, this invocation is
// somewhat extraneous (pointers were already assigned).
switch (name) {
case "zh-CHS":
catTable = cjkCHScategory;
lv1Table = cjkCHSlv1;
cjkIndexer = UUtil.CjkCHS;
break;
case "zh-CHT":
catTable = cjkCHTcategory;
lv1Table = cjkCHTlv1;
cjkIndexer = UUtil.Cjk;
break;
case "ja":
catTable = cjkJAcategory;
lv1Table = cjkJAlv1;
cjkIndexer = UUtil.Cjk;
break;
case "ko":
catTable = cjkKOcategory;
lv1Table = cjkKOlv1;
lv2Table = cjkKOlv2;
cjkIndexer = UUtil.Cjk;
lv2Indexer = UUtil.Cjk;
break;
}
}
private void SerializeUCD()
{
COut = new StreamWriter("normalization-tables.h", true);
// mappedChars
COut.WriteLine("static const guint32 mappedChars [] = {");
CSOut.WriteLine("static readonly int [] mappedCharsArr = new int [] {");
DumpMapArray(mappedChars, mappedCharCount, false);
COut.WriteLine("0};");
CSOut.WriteLine("};");
// charMapIndex
COut.WriteLine("static const guint16 charMapIndex [] = {");
CSOut.WriteLine("static readonly short [] charMapIndexArr = new short [] {");
DumpMapArray(mapIndex, NUtil.MapCount, true);
COut.WriteLine("0};");
CSOut.WriteLine("};");
short [] helperIndexes = new short [0x30000];
// GetPrimaryCompositeHelperIndex ()
int currentHead = 0;
foreach (CharMapping m in mappings)
{
if (mappedChars [m.MapIndex] == currentHead)
{
continue; // has the same head
}
if (!m.IsCanonical)
{
continue;
}
currentHead = mappedChars [m.MapIndex];
helperIndexes [currentHead] = (short)m.MapIndex;
}
helperIndexes = CodePointIndexer.CompressArray(
helperIndexes, typeof(short), NUtil.Helper)
as short [];
COut.WriteLine("static const guint16 helperIndex [] = {");
CSOut.WriteLine("static short [] helperIndexArr = new short [] {");
for (int i = 0; i < helperIndexes.Length; i++)
{
short value = helperIndexes [i];
if (value < 10)
{
CSOut.Write("{0},", value);
}
else
{
CSOut.Write("0x{0:X04},", value);
}
COut.Write("{0},", value);
if (i % 16 == 15)
{
CSOut.WriteLine(" // {0:X04}", NUtil.Helper.ToCodePoint(i - 15));
COut.WriteLine();
}
}
COut.WriteLine("0};");
CSOut.WriteLine("};");
ushort [] mapIndexes = new ushort [char.MaxValue + 1];
// GetPrimaryCompositeFromMapIndex ()
int currentIndex = -1;
foreach (CharMapping m in mappings)
{
if (m.MapIndex == currentIndex)
{
continue;
}
if (!m.IsCanonical)
{
continue;
}
mapIndexes [m.MapIndex] = (ushort)m.CodePoint;
currentIndex = m.MapIndex;
}
mapIndexes = CodePointIndexer.CompressArray(mapIndexes, typeof(ushort), NUtil.Composite) as ushort [];
COut.WriteLine("static const guint16 mapIdxToComposite [] = {");
CSOut.WriteLine("static ushort [] mapIdxToCompositeArr = new ushort [] {");
for (int i = 0; i < mapIndexes.Length; i++)
{
ushort value = (ushort)mapIndexes [i];
if (value < 10)
{
CSOut.Write("{0},", value);
}
else
{
CSOut.Write("0x{0:X04},", value);
}
COut.Write("{0},", value);
if (i % 16 == 15)
{
CSOut.WriteLine(" // {0:X04}", NUtil.Composite.ToCodePoint(i - 15));
COut.WriteLine();
}
}
COut.WriteLine("0};");
CSOut.WriteLine("};");
COut.Close();
}
private void ProcessCombiningClass()
{
TextReader reader = new StreamReader("downloaded/DerivedCombiningClass.txt");
while (reader.Peek() != -1)
{
string line = reader.ReadLine();
lineCount++;
int idx = line.IndexOf('#');
if (idx >= 0)
{
line = line.Substring(0, idx).Trim();
}
if (line.Length == 0)
{
continue;
}
int n = 0;
while (Char.IsDigit(line [n]) || Char.IsLetter(line [n]))
{
n++;
}
int cp = int.Parse(line.Substring(0, n), NumberStyles.HexNumber);
// Windows does not handle surrogate characters.
if (cp >= 0x10000)
{
continue;
}
int cpEnd = -1;
if (line [n] == '.' && line [n + 1] == '.')
{
cpEnd = int.Parse(line.Substring(n + 2, n), NumberStyles.HexNumber);
}
int nameStart = line.IndexOf(';') + 1;
int valueStart = line.IndexOf(';', nameStart) + 1;
string val = valueStart == 0 ? line.Substring(nameStart) :
line.Substring(nameStart, valueStart - nameStart - 1);
SetCombiningProp(cp, cpEnd, short.Parse(val));
}
reader.Close();
byte [] ret = (byte [])CodePointIndexer.CompressArray(
combining, typeof(byte), NUtil.Combining);
COut = new StreamWriter("normalization-tables.h", true);
COut.WriteLine("static const guint8 combiningClass [] = {");
CSOut.WriteLine("public static byte [] combiningClassArr = new byte [] {");
for (int i = 0; i < ret.Length; i++)
{
byte value = ret [i];
if (value < 10)
{
CSOut.Write("{0},", value);
}
else
{
CSOut.Write("0x{0:X02},", value);
}
COut.Write("{0},", value);
if (i % 16 == 15)
{
CSOut.WriteLine(" // {0:X04}", NUtil.Combining.ToCodePoint(i - 15));
COut.WriteLine();
}
}
CSOut.WriteLine("};");
COut.WriteLine("0};");
COut.Close();
}
static void FillCJKCore (string culture,
ref CodePointIndexer cjkIndexer,
ref byte* catTable, ref byte* lv1Table,
ref CodePointIndexer cjkLv2Indexer, ref byte* lv2Table)
{
if (!IsReady)
return;
string name = null;
switch (culture)
{
case "zh-CHS":
name = "cjkCHS";
catTable = cjkCHScategory;
lv1Table = cjkCHSlv1;
break;
case "zh-CHT":
name = "cjkCHT";
catTable = cjkCHTcategory;
lv1Table = cjkCHTlv1;
break;
case "ja":
name = "cjkJA";
catTable = cjkJAcategory;
lv1Table = cjkJAlv1;
break;
case "ko":
name = "cjkKO";
catTable = cjkKOcategory;
lv1Table = cjkKOlv1;
break;
}
if (name == null || lv1Table != null)
return;
byte* raw;
uint idx = 0;
#if USE_MANAGED_RESOURCE
string filename =
String.Format ("collation.{0}.bin", name);
IntPtr ptr = GetResource (filename);
if (ptr == IntPtr.Zero)
return;
raw = (byte*) ((void*) ptr);
idx += ResourceVersionSize;
#elif USE_C_HEADER
int residx = -1;
switch (culture)
{
case "zh-CHS":
residx = CollationTableIdxCjkCHS;
break;
case "zh-CHT":
residx = CollationTableIdxCjkCHT;
break;
case "ja":
residx = CollationTableIdxCjkJA;
break;
case "ko":
residx = CollationTableIdxCjkKO;
break;
}
if (residx < 0)
return;
load_collation_resource (residx, &raw);
#else
int size;
int residx = -1;
switch (culture)
{
case "zh-CHS":
residx = CollationResourceCJKCHS;
break;
case "zh-CHT":
residx = CollationResourceCJKCHT;
break;
case "ja":
residx = CollationResourceCJKJA;
break;
case "ko":
residx = CollationResourceCJKKO;
break;
}
if (residx < 0)
return;
load_collation_resource (corlibPath, residx, &raw, &size);
idx += ResourceVersionSize;
#endif
uint count = UInt32FromBytePtr (raw, idx);
idx += 4;
catTable = (byte*) raw + idx;
lv1Table = (byte*) raw + idx + count;
switch (culture)
{
case "zh-CHS":
cjkCHScategory = catTable;
cjkCHSlv1 = lv1Table;
break;
case "zh-CHT":
cjkCHTcategory = catTable;
cjkCHTlv1 = lv1Table;
break;
case "ja":
cjkJAcategory = catTable;
cjkJAlv1 = lv1Table;
break;
case "ko":
cjkKOcategory = catTable;
cjkKOlv1 = lv1Table;
break;
}
if (name != "cjkKO")
return;
#if USE_MANAGED_RESOURCE
ptr = GetResource ("collation.cjkKOlv2.bin");
if (ptr == IntPtr.Zero)
return;
raw = (byte*) ((void*) ptr);
idx = ResourceVersionSize + 4;
#elif USE_C_HEADER
load_collation_resource (CollationTableIdxCjkKOLv2, &raw);
#else
load_collation_resource (corlibPath, CollationResourceCJKKOlv2, &raw, &size);
idx = ResourceVersionSize + 4;
#endif
cjkKOlv2 = raw + idx;
lv2Table = cjkKOlv2;
}
byte [] CompressArray (byte [] source, CodePointIndexer i)
{
return (byte []) CodePointIndexer.CompressArray (
source, typeof (byte), i);
}
static NormalizationTableUtil()
{
int [] propStarts = new int []
{
0, 0x0910, 0x1B00, 0x2460, 0x2980,
0x2C70, 0x2D60, 0x2E90, 0xA770, 0xF900,
// 0x1D100, 0x2f800, 0x2fa10
};
int [] propEnds = new int []
{
0x06E0, 0x1200, 0x2330, 0x2600, 0x2AE0,
0x2C80, 0x2D70, 0x3400, 0xA780, 0x10000,
// 0x1D800, 0x2f810, 0x2fa20
};
int [] mapStarts = new int []
{
0x90, 0x0920, 0x1D20, 0x2460, 0x24A0, 0x2A00,
0x2D60, 0x2E90, 0xF900,
// 0x1d150, 0x2f800
};
int [] mapEnds = new int []
{
0x06E0, 0x1100, 0x2330, 0x24A0, 0x24F0, 0x2AE0,
0x2D70, 0x3400, 0x10000,
// 0x1d800, 0x2fb00
};
int [] combiningStarts = new int []
{
0x02F0, 0x0480, 0x0590, 0x0930, 0x09B0,
0x0A30, 0x0AB0, 0x0B30, 0x0BC0, 0x0C40,
0x0CB0, 0x0D40, 0x0DC0, 0x0E30, 0x0EB0,
0x0F00, 0x1030, 0x1350, 0x1710, 0x17D0,
0x18A0, 0x1930, 0x1A10, 0x1DC0, 0x20D0,
0x3020, 0x3090, 0xA800, 0xFB10, 0xFE20,
// 0x10A00, 0x1D160, 0x1D240
};
int [] combiningEnds = new int []
{
0x0360, 0x0490, 0x0750, 0x0960, 0x09D0,
0x0A50, 0x0AD0, 0x0B50, 0x0BD0, 0x0C60,
0x0CD0, 0x0D50, 0x0DD0, 0x0E50, 0x0ED0,
0x0FD0, 0x1040, 0x1360, 0x1740, 0x17E0,
0x18B0, 0x1940, 0x1A20, 0x1DD0, 0x20F0,
0x3030, 0x30A0, 0xA810, 0xFB20, 0xFE30,
// 0x10A40, 0x1D1B0, 0x1D250
};
// since mapToCompositeIndex only holds canonical
// mappings, those indexes could be still shorten.
int [] compositeStarts = new int []
{
0x480, 0x1410, 0x1670
};
int [] compositeEnds = new int []
{
0x1080, 0x1580, 0x21B0
};
int [] helperStarts = new int []
{
0, 0x900, 0x1D00, 0x2500, 0x3000, 0x3B90,
0x4010, 0x4E00, 0xFB40,
// 0x1D150, 0x20100, 0x20510,
// 0x20630, 0x20800, 0x20A20, 0x20B60, 0x214E0,
};
int [] helperEnds = new int []
{
0x700, 0x1200, 0x2300, 0x2600, 0x3160, 0x3BA0,
0x4030, 0xA000, 0xFB50,
// 0x1D1C0, 0x20130, 0x20550,
// 0x20640, 0x208E0, 0x20A30, 0x20B70, 0x214F0,
};
Prop = new CodePointIndexer(propStarts, propEnds, 0, 0);
Map = new CodePointIndexer(mapStarts, mapEnds, 0, 0);
Combining = new CodePointIndexer(combiningStarts,
combiningEnds, 0, 0);
Composite = new CodePointIndexer(compositeStarts,
compositeEnds, 0, 0);
Helper = new CodePointIndexer(helperStarts, helperEnds,
0, 0);
}
static NormalizationTableUtil ()
{
int [] propStarts = new int [] {
0, 0x0910, 0x1B00, 0x2460, 0x2980,
0x2C70, 0x2D60, 0x2E90, 0xA770, 0xF900,
// 0x1D100, 0x2f800, 0x2fa10
};
int [] propEnds = new int [] {
0x06E0, 0x1200, 0x2330, 0x2600, 0x2AE0,
0x2C80, 0x2D70, 0x3400, 0xA780, 0x10000,
// 0x1D800, 0x2f810, 0x2fa20
};
int [] mapStarts = new int [] {
0x90, 0x0920, 0x1D20, 0x2460, 0x24A0, 0x2A00,
0x2D60, 0x2E90, 0xF900,
// 0x1d150, 0x2f800
};
int [] mapEnds = new int [] {
0x06E0, 0x1100, 0x2330, 0x24A0, 0x24F0, 0x2AE0,
0x2D70, 0x3400, 0x10000,
// 0x1d800, 0x2fb00
};
int [] combiningStarts = new int [] {
0x02F0, 0x0480, 0x0590, 0x0930, 0x09B0,
0x0A30, 0x0AB0, 0x0B30, 0x0BC0, 0x0C40,
0x0CB0, 0x0D40, 0x0DC0, 0x0E30, 0x0EB0,
0x0F00, 0x1030, 0x1350, 0x1710, 0x17D0,
0x18A0, 0x1930, 0x1A10, 0x1DC0, 0x20D0,
0x3020, 0x3090, 0xA800, 0xFB10, 0xFE20,
// 0x10A00, 0x1D160, 0x1D240
};
int [] combiningEnds = new int [] {
0x0360, 0x0490, 0x0750, 0x0960, 0x09D0,
0x0A50, 0x0AD0, 0x0B50, 0x0BD0, 0x0C60,
0x0CD0, 0x0D50, 0x0DD0, 0x0E50, 0x0ED0,
0x0FD0, 0x1040, 0x1360, 0x1740, 0x17E0,
0x18B0, 0x1940, 0x1A20, 0x1DD0, 0x20F0,
0x3030, 0x30A0, 0xA810, 0xFB20, 0xFE30,
// 0x10A40, 0x1D1B0, 0x1D250
};
// since mapToCompositeIndex only holds canonical
// mappings, those indexes could be still shorten.
int [] compositeStarts = new int [] {
0x480, 0x1410, 0x1670
};
int [] compositeEnds = new int [] {
0x1080, 0x1580, 0x21B0
};
int [] helperStarts = new int [] {
0, 0x900, 0x1D00, 0x2500, 0x3000, 0x3B90,
0x4010, 0x4E00, 0xFB40,
// 0x1D150, 0x20100, 0x20510,
// 0x20630, 0x20800, 0x20A20, 0x20B60, 0x214E0,
};
int [] helperEnds = new int [] {
0x700, 0x1200, 0x2300, 0x2600, 0x3160, 0x3BA0,
0x4030, 0xA000, 0xFB50,
// 0x1D1C0, 0x20130, 0x20550,
// 0x20640, 0x208E0, 0x20A30, 0x20B70, 0x214F0,
};
Prop = new CodePointIndexer (propStarts, propEnds, 0, 0);
Map = new CodePointIndexer (mapStarts, mapEnds, 0, 0);
Combining = new CodePointIndexer (combiningStarts,
combiningEnds, 0, 0);
Composite = new CodePointIndexer (compositeStarts,
compositeEnds, 0, 0);
Helper = new CodePointIndexer (helperStarts, helperEnds,
0, 0);
}
static MSCompatUnicodeTableUtil()
{
int[] starts = new int[]
{
0,
40960,
63744
};
int[] ends = new int[]
{
13312,
42240,
65536
};
int[] starts2 = new int[]
{
0,
7680,
12288,
19968,
44032,
63744
};
int[] ends2 = new int[]
{
4608,
10240,
13312,
40960,
55216,
65536
};
int[] starts3 = new int[]
{
0,
7680,
12288,
19968,
44032,
63744
};
int[] ends3 = new int[]
{
4608,
10240,
13312,
40960,
55216,
65536
};
int[] starts4 = new int[]
{
0,
7680,
12288,
64256
};
int[] ends4 = new int[]
{
3840,
10240,
13312,
65536
};
int[] starts5 = new int[]
{
0,
7680,
12288,
64256
};
int[] ends5 = new int[]
{
4608,
10240,
13312,
65536
};
int[] starts6 = new int[]
{
12544,
19968,
59392
};
int[] ends6 = new int[]
{
13312,
40960,
65536
};
int[] starts7 = new int[]
{
12544,
19968,
63744
};
int[] ends7 = new int[]
{
13312,
40960,
64256
};
MSCompatUnicodeTableUtil.Ignorable = new CodePointIndexer(starts, ends, -1, -1);
MSCompatUnicodeTableUtil.Category = new CodePointIndexer(starts2, ends2, 0, 0);
MSCompatUnicodeTableUtil.Level1 = new CodePointIndexer(starts3, ends3, 0, 0);
MSCompatUnicodeTableUtil.Level2 = new CodePointIndexer(starts4, ends4, 0, 0);
MSCompatUnicodeTableUtil.Level3 = new CodePointIndexer(starts5, ends5, 0, 0);
MSCompatUnicodeTableUtil.CjkCHS = new CodePointIndexer(starts6, ends6, -1, -1);
MSCompatUnicodeTableUtil.Cjk = new CodePointIndexer(starts7, ends7, -1, -1);
}