public override void Allocate() { TimerPool.GetTimer(this, "Load LM").Start(); this.LogInfo("Loading n-gram language model from: " + location); // create the log file if specified if (ngramLogFile != null) { logFile = new StreamWriter(ngramLogFile); } BinaryLoader loader; if (location.Path == null || location.Path.Equals("file")) { try { loader = new BinaryLoader(new FileInfo(location.Path)); } catch (Exception ex) { loader = new BinaryLoader(new FileInfo(location.Path)); } } else { loader = new BinaryLoader(location); } loader.verifyHeader(); counts = loader.readCounts(); if (MaxDepth <= 0 || MaxDepth > counts.Length) { MaxDepth = counts.Length; } if (MaxDepth > 1) { quant = loader.readQuant(MaxDepth); } unigrams = loader.readUnigrams(counts[0]); if (MaxDepth > 1) { trie = new NgramTrie(counts, quant.getProbBoSize(), quant.getProbSize()); loader.readTrieByteArr(trie.getMem()); } //string words can be read here words = loader.readWords(counts[0]); BuildUnigramIDMap(); ngramProbCache = new LRUCache <WordSequence, Float>(ngramCacheSize); loader.close(); TimerPool.GetTimer(this, "Load LM").Stop(); }
public NgramTrieQuant readQuant(int order) { int quantTypeInt = Utilities.ReadLittleEndianInt(inStream); if (quantTypeInt < 0 || quantTypeInt >= 2 /* QuantType has 2 Enums with int value 0 and 1*/) { throw new Error("Unknown quantatization type: " + quantTypeInt); } NgramTrieQuant.QuantType quantType = (NgramTrieQuant.QuantType)quantTypeInt; NgramTrieQuant quant = new NgramTrieQuant(order, quantType); //reading tables for (int i = 2; i <= order; i++) { quant.setTable(readFloatArr(quant.getProbTableLen()), i, true); if (i < order) { quant.setTable(readFloatArr(quant.getBackoffTableLen()), i, false); } } return(quant); }