// Can only read file of Model1 format for now
public void ReadFromBinary(BinaryReader br)
{
int version = br.ReadInt32();
if (version != 1)
{
throw new Exception();
}
var converter = new Utility.VietConverter();
// Read number of bits for each number type
m_NumBitPerAccStringIndex = br.ReadInt32();
m_NumBitPerAccCountIndex = br.ReadInt32();
m_NumBitPerHashEntryCount = br.ReadInt32();
m_NumBitPerAccStringLength = br.ReadInt32();
// Read look up table of accent codes
m_NumAccString = br.ReadInt32();
var asciiBytes = new List <byte>(m_NumAccString * 2);
m_PredictionAsciiIndex = new int[m_NumAccString + 1];
int index = 0;
for (int i = 0; i < m_NumAccString; i++)
{
byte stringLength = br.ReadByte();
m_PredictionAsciiIndex[i] = index;
index += stringLength;
asciiBytes.AddRange(br.ReadBytes(stringLength));
}
m_PredictionAsciiIndex[m_NumAccString] = index;
m_PredictionAsciiBytes = asciiBytes.ToArray();
// Read look up table of accent code counts
int numAccCodeCount = br.ReadInt32();
m_PredictionCounts = new int[numAccCodeCount];
for (int i = 0; i < numAccCodeCount; i++)
{
m_PredictionCounts[i] = br.ReadInt32();
}
// Deserialize hash function
int hashFunctionBytesLength = br.ReadInt32();
byte[] hashFunctionBytes = br.ReadBytes(hashFunctionBytesLength);
var hashFunctionStream = new MemoryStream(hashFunctionBytes);
var formatter = new BinaryFormatter();
m_HashFunction = (MinPerfectHash)formatter.Deserialize(hashFunctionStream);
// Read bytes for hash codes
int hashSize = (int)m_HashFunction.N;
var hashTableByteList = new List <byte>((int)m_HashFunction.N * 4);
m_HashTableByteIndex = new int[hashSize + 1];
index = 0;
for (int i = 0; i < hashSize; i++)
{
byte[] countByte = br.ReadBytes(1);
BitArray baCount = new BitArray(countByte);
int count = 0;
for (int ib = 0; ib < m_NumBitPerHashEntryCount; ib++)
{
count += (baCount[ib] ? 1 : 0) << (m_NumBitPerHashEntryCount - ib - 1);
}
if (count == 0)
{
continue;
}
br.BaseStream.Position = br.BaseStream.Position - 1;
int numBitPerElement = m_NumBitPerAccStringIndex + m_NumBitPerAccCountIndex;
int totalNumBits = m_NumBitPerHashEntryCount + count * numBitPerElement;
int numBytes = (int)Math.Ceiling(totalNumBits / 8.0);
m_HashTableByteIndex[i] = index;
index += numBytes;
hashTableByteList.AddRange(br.ReadBytes(numBytes));
}
m_HashTableByteIndex[hashSize] = index;
m_HashTableBytes = hashTableByteList.ToArray();
}
public void WriteToBinary(BinaryWriter bw)
{
m_HashFunction = PerfectHash.MPH.MinPerfectHash.Create(new RawStringKeySource(m_Map.Keys.ToList()), c: 1.0);
m_HashTable = Enumerable.Repeat <Dictionary <string, int> >(null, (int)m_HashFunction.N).ToList();
foreach (string rawString in m_Map.Keys)
{
int hashCode = (int)m_HashFunction.Search(Encoding.UTF8.GetBytes(rawString));
m_HashTable[hashCode] = m_Map[rawString];
}
var converter = new Utility.VietConverter();
var uniqueAccentStrings = new HashSet <string>();
var uniqueAccentCodeCount = new HashSet <int>();
int maxHashEntryCount = 0;
foreach (Dictionary <string, int> accCodeToCount in m_Map.Values)
{
foreach (string accString in accCodeToCount.Keys)
{
uniqueAccentStrings.Add(accString);
uniqueAccentCodeCount.Add(accCodeToCount[accString]);
}
maxHashEntryCount = Math.Max(maxHashEntryCount, accCodeToCount.Count);
}
int numBitPerAccStringIndex = GetRepresentativeBitCount(uniqueAccentStrings.Count);
int numBitPerAccCountIndex = GetRepresentativeBitCount(uniqueAccentCodeCount.Count);
int numBitPerHashEntryCount = GetRepresentativeBitCount(maxHashEntryCount);
bw.Write(Version); // Write version number
bw.Write(numBitPerAccStringIndex); // How many bits to represent an accent code
bw.Write(numBitPerAccCountIndex); // How many bits to represent an accent code's count
bw.Write(numBitPerHashEntryCount); // How many bits to represent the count of elements in each hash entry
var accStringList = new List <string>();
var accStringToIndex = new Dictionary <string, int>();
int index = 0;
foreach (string accString in uniqueAccentStrings)
{
accStringList.Add(accString);
accStringToIndex.Add(accString, index);
index++;
}
int numBitPerAccStringLength = GetRepresentativeBitCount(accStringList.Max(s => s.Length));
bw.Write(numBitPerAccStringLength);
// Write ascii accent codes lookup table
bw.Write(accStringList.Count);
foreach (string accString in accStringList)
{
string asciiString = String.Join(String.Empty, accString.Select(c => (char)converter.AccentToAsciiMap[c]));
byte[] asciiBytes = System.Text.Encoding.ASCII.GetBytes(asciiString);
bw.Write((byte)asciiBytes.Length);
bw.Write(asciiBytes);
}
var accCodeCountList = new List <int>();
var accCodeCountToIndex = new Dictionary <int, int>();
index = 0;
foreach (int accCodeCount in uniqueAccentCodeCount)
{
accCodeCountList.Add(accCodeCount);
accCodeCountToIndex.Add(accCodeCount, index);
index++;
}
// Write unique accent code counts lookup table
bw.Write(accCodeCountList.Count);
foreach (int accCodeCount in accCodeCountList)
{
bw.Write(accCodeCount);
}
// Serialize hash function
var hashFunctionStream = new MemoryStream();
var formatter = new BinaryFormatter();
formatter.Serialize(hashFunctionStream, m_HashFunction);
byte[] hashFunctionBytes = hashFunctionStream.ToArray();
bw.Write(hashFunctionBytes.Length);
bw.Write(hashFunctionBytes);
var bitList = new List <bool>();
foreach (var kvp in m_HashTable)
{
// Write number of entries, using the minimal number of bits
int count = kvp != null ? kvp.Count : 0;
for (int ib = numBitPerHashEntryCount - 1; ib >= 0; ib--)
{
bitList.Add(((count >> ib) & 1) == 1);
}
if (kvp != null)
{
foreach (string accString in kvp.Keys)
{
// Write accent code's index in the lookup table, using the minimal number of bits
for (int ib = numBitPerAccStringIndex - 1; ib >= 0; ib--)
{
bitList.Add(((accStringToIndex[accString] >> ib) & 1) == 1);
}
// Write accent code count's index in the lookup table, using the minimal number of bits
for (int ib = numBitPerAccCountIndex - 1; ib >= 0; ib--)
{
bitList.Add(((accCodeCountToIndex[kvp[accString]] >> ib) & 1) == 1);
}
}
}
// Write from queue to disk byte-aligned
while (bitList.Count % 8 != 0)
{
bitList.Add(false);
}
BitArray ba = new BitArray(bitList.ToArray());
byte[] bytes = new byte[bitList.Count / 8];
ba.CopyTo(bytes, 0);
bw.Write(bytes);
bitList.Clear();
}
}