public override void Allocate()
{
TimerPool.GetTimer(this, "Load LM").Start();
this.LogInfo("Loading n-gram language model from: " + location);
// create the log file if specified
if (ngramLogFile != null)
{
logFile = new StreamWriter(ngramLogFile);
}
BinaryLoader loader;
if (location.Path == null || location.Path.Equals("file"))
{
try
{
loader = new BinaryLoader(new FileInfo(location.Path));
}
catch (Exception ex)
{
loader = new BinaryLoader(new FileInfo(location.Path));
}
}
else
{
loader = new BinaryLoader(location);
}
loader.verifyHeader();
counts = loader.readCounts();
if (MaxDepth <= 0 || MaxDepth > counts.Length)
{
MaxDepth = counts.Length;
}
if (MaxDepth > 1)
{
quant = loader.readQuant(MaxDepth);
}
unigrams = loader.readUnigrams(counts[0]);
if (MaxDepth > 1)
{
trie = new NgramTrie(counts, quant.getProbBoSize(), quant.getProbSize());
loader.readTrieByteArr(trie.getMem());
}
//string words can be read here
words = loader.readWords(counts[0]);
BuildUnigramIDMap();
ngramProbCache = new LRUCache <WordSequence, Float>(ngramCacheSize);
loader.close();
TimerPool.GetTimer(this, "Load LM").Stop();
}
public NgramTrieQuant readQuant(int order)
{
int quantTypeInt = Utilities.ReadLittleEndianInt(inStream);
if (quantTypeInt < 0 || quantTypeInt >= 2 /* QuantType has 2 Enums with int value 0 and 1*/)
{
throw new Error("Unknown quantatization type: " + quantTypeInt);
}
NgramTrieQuant.QuantType quantType = (NgramTrieQuant.QuantType)quantTypeInt;
NgramTrieQuant quant = new NgramTrieQuant(order, quantType);
//reading tables
for (int i = 2; i <= order; i++)
{
quant.setTable(readFloatArr(quant.getProbTableLen()), i, true);
if (i < order)
{
quant.setTable(readFloatArr(quant.getBackoffTableLen()), i, false);
}
}
return(quant);
}
