/// <summary> /// Sets the InputStream to read cepstral data from. /// </summary> /// <param name="_is">The InputStream to read cepstral data from.</param> /// <param name="bigEndian">true if the InputStream data is in big-endian, false otherwise.</param> public void SetInputStream(Stream _is, Boolean bigEndian) { _bigEndian = bigEndian; if (_binary) { _binaryStream = new BufferedStream(_is, 8192); if (_bigEndian) { _numPoints = _binaryStream.ReadInt(); this.LogInfo("BigEndian"); } else { _numPoints = Utilities.ReadLittleEndianInt(_binaryStream); this.LogInfo("LittleEndian"); } this.LogInfo("Frames: " + _numPoints / _cepstrumLength); } else { _est = new ExtendedStreamTokenizer(_is, false); _numPoints = _est.GetInt("num_frames"); _est.ExpectString("frames"); } _curPoint = -1; _firstSampleNumber = 0; }
protected internal override GrammarNode createGrammar() { ExtendedStreamTokenizer extendedStreamTokenizer = new ExtendedStreamTokenizer(this.path, true); GrammarNode grammarNode = this.createGrammarNode("<sil>"); GrammarNode grammarNode2 = this.createGrammarNode(false); GrammarNode grammarNode3 = this.createGrammarNode("<sil>"); grammarNode3.setFinalNode(true); LinkedList linkedList = new LinkedList(); while (!extendedStreamTokenizer.isEOF()) { string @string; while ((@string = extendedStreamTokenizer.getString()) != null) { GrammarNode grammarNode4 = this.createGrammarNode(@string); linkedList.add(grammarNode4); } } grammarNode.add(grammarNode2, 0f); float logProbability = this.logMath.linearToLog((double)1f / (double)linkedList.size()); Iterator iterator = linkedList.iterator(); while (iterator.hasNext()) { GrammarNode grammarNode5 = (GrammarNode)iterator.next(); grammarNode2.add(grammarNode5, logProbability); grammarNode5.add(grammarNode3, 0f); if (this.isLooping) { grammarNode5.add(grammarNode2, 0f); } } return(grammarNode); }
/** * Reads the FST file in the given path, and creates the nodes in the FST file. * * @param path the path of the FST file to read * @return the highest ID of all nodes * @throws java.io.IOException */ private int CreateNodes(String path) { ExtendedStreamTokenizer tok = new ExtendedStreamTokenizer(path, true); int maxNodeId = 0; while (!tok.IsEOF()) { tok.Skipwhite(); String token = tok.GetString(); if (token == null) { break; } else if (token.Equals("T")) { tok.GetInt("src id"); // toss source node int id = tok.GetInt("dest id"); // dest node numb if (id > maxNodeId) { maxNodeId = id; } String word1 = tok.GetString(); // get word if (word1 == null) { continue; } String word2 = tok.GetString(); // get word tok.GetString(); // toss probability String nodeName = "G" + id; GrammarNode node = _nodes.Get(nodeName); if (node == null) { if (word2.Equals(",")) { node = CreateGrammarNode(id, false); } else { node = CreateGrammarNode(id, word2); } _nodes.Put(nodeName, node); } else { if (!word2.Equals(",")) { /* * if (!word2.equals(getWord(node))) { * System.out.println(node + ": " + word2 + ' ' + getWord(node)); } */ Debug.Assert(word2.Equals(GetWord(node))); } } } } tok.Close(); return(maxNodeId); }
private int createNodes(string text) { ExtendedStreamTokenizer extendedStreamTokenizer = new ExtendedStreamTokenizer(text, true); int num = 0; while (!extendedStreamTokenizer.isEOF()) { extendedStreamTokenizer.skipwhite(); string @string = extendedStreamTokenizer.getString(); if (@string == null) { break; } if (String.instancehelper_equals(@string, "T")) { extendedStreamTokenizer.getInt("src id"); int @int = extendedStreamTokenizer.getInt("dest id"); if (@int > num) { num = @int; } if (extendedStreamTokenizer.getString() != null) { string string2 = extendedStreamTokenizer.getString(); extendedStreamTokenizer.getString(); string text2 = new StringBuilder().append("G").append(@int).toString(); GrammarNode grammarNode = (GrammarNode)this.nodes.get(text2); if (grammarNode == null) { if (String.instancehelper_equals(string2, ",")) { grammarNode = this.createGrammarNode(@int, false); } else { grammarNode = this.createGrammarNode(@int, string2); } this.nodes.put(text2, grammarNode); } else if (!String.instancehelper_equals(string2, ",") && !FSTGrammar.assertionsDisabled && !String.instancehelper_equals(string2, this.getWord(grammarNode))) { throw new AssertionError(); } } } } extendedStreamTokenizer.close(); return(num); }
protected GrammarNode CreateGrammar() { ExtendedStreamTokenizer tok = null; if (File.Exists(_path)) { tok = new ExtendedStreamTokenizer(_path, true); } else { tok = new ExtendedStreamTokenizer((StreamReader)null, true); } GrammarNode initialNode = CreateGrammarNode("<sil>"); GrammarNode branchNode = CreateGrammarNode(false); GrammarNode finalNode = CreateGrammarNode("<sil>"); finalNode.SetFinalNode(true); List <GrammarNode> wordGrammarNodes = new List <GrammarNode>(); while (!tok.IsEOF()) { string word; while ((word = tok.GetString()) != null) { wordGrammarNodes.Add(CreateGrammarNode(word)); } } // now connect all the GrammarNodes together initialNode.Add(branchNode, LogMath.LogOne); if (wordGrammarNodes.Count != 0) { float branchScore = _logMath.LinearToLog(1.0 / wordGrammarNodes.Count); foreach (GrammarNode wordNode in wordGrammarNodes) { branchNode.Add(wordNode, branchScore); wordNode.Add(finalNode, LogMath.LogOne); if (_isLooping) { wordNode.Add(branchNode, LogMath.LogOne); } } } return(initialNode); }
/* * Reads Text and converts it into a list of tokens */ public void setText(string text) { string word; try { ExtendedStreamTokenizer tok = new ExtendedStreamTokenizer( new java.io.StringReader(text), true); tokens.Clear(); while (!tok.isEOF()) { while ((word = tok.getString()) != null) { word = word.toLowerCase(); tokens.Add(word); } } } catch (Exception e) { e.printStackTrace(); } }
/** * /// Loads the sphinx3 density file, a set of density arrays are created and * /// placed in the given pool. * /// * /// @param useCDUnits * /// if true, loads also the context dependent units * /// @param inputStream * /// the open input stream to use * /// @param path * /// the path to a density file * /// @throws FileNotFoundException * /// if a file cannot be found * /// @throws IOException * /// if an error occurs while loading the data */ protected void LoadHMMPool(Boolean useCDUnits, Stream inputStream, string path) { var est = new ExtendedStreamTokenizer(inputStream, '#', false); this.LogInfo("Loading HMM file from: " + path); est.ExpectString(ModelVersion); var numBase = est.GetInt("numBase"); est.ExpectString("n_base"); var numTri = est.GetInt("numTri"); est.ExpectString("n_tri"); var numStateMap = est.GetInt("numStateMap"); est.ExpectString("n_state_map"); var numTiedState = est.GetInt("numTiedState"); est.ExpectString("n_tied_state"); var numContextIndependentTiedState = est .GetInt("numContextIndependentTiedState"); est.ExpectString("n_tied_ci_state"); var numTiedTransitionMatrices = est.GetInt("numTiedTransitionMatrices"); est.ExpectString("n_tied_tmat"); var numStatePerHMM = numStateMap / (numTri + numBase); Debug.Assert(numTiedState == MixtureWeightsPool.StatesNum); Debug.Assert(numTiedTransitionMatrices == MatrixPool.Size); // Load the base phones for (var i = 0; i < numBase; i++) { var name = est.GetString(); var left = est.GetString(); var right = est.GetString(); var position = est.GetString(); var attribute = est.GetString(); var tmat = est.GetInt("tmat"); var stid = new int[numStatePerHMM - 1]; for (var j = 0; j < numStatePerHMM - 1; j++) { stid[j] = est.GetInt("j"); Debug.Assert(stid[j] >= 0 && stid[j] < numContextIndependentTiedState); } est.ExpectString("N"); Debug.Assert(left.Equals("-")); Debug.Assert(right.Equals("-")); Debug.Assert(position.Equals("-")); Debug.Assert(tmat < numTiedTransitionMatrices); var unit = _unitManager.GetUnit(name, attribute.Equals(Filler)); ContextIndependentUnits.Put(unit.Name, unit); //this.LogInfo("Loaded " + unit.ToString()); // The first filler if (unit.IsFiller && unit.Name.Equals(SilenceCiphone)) { unit = UnitManager.Silence; } var transitionMatrix = MatrixPool.Get(tmat); var ss = GetSenoneSequence(stid); IHMM hmm = new SenoneHMM(unit, ss, transitionMatrix, GetHMMPosition(position)); HmmManager.Put(hmm); } if (HmmManager.Get(HMMPosition.Undefined, UnitManager.Silence) == null) { throw new IOException("Could not find SIL unit in acoustic model"); } // Load the context dependent phones. If the useCDUnits // property is false, the CD phones will not be created, but // the values still need to be read in from the file. var lastUnitName = ""; Unit lastUnit = null; int[] lastStid = null; SenoneSequence lastSenoneSequence = null; for (var i = 0; i < numTri; i++) { var name = est.GetString(); var left = est.GetString(); var right = est.GetString(); var position = est.GetString(); var attribute = est.GetString(); var tmat = est.GetInt("tmat"); var stid = new int[numStatePerHMM - 1]; for (var j = 0; j < numStatePerHMM - 1; j++) { stid[j] = est.GetInt("j"); Debug.Assert(stid[j] >= numContextIndependentTiedState && stid[j] < numTiedState); } est.ExpectString("N"); Debug.Assert(!left.Equals("-")); Debug.Assert(!right.Equals("-")); Debug.Assert(!position.Equals("-")); Debug.Assert(attribute.Equals("n/a")); Debug.Assert(tmat < numTiedTransitionMatrices); if (useCDUnits) { Unit unit; var unitName = (name + ' ' + left + ' ' + right); if (unitName.Equals(lastUnitName)) { unit = lastUnit; } else { var leftContext = new Unit[1]; leftContext[0] = ContextIndependentUnits.Get(left); var rightContext = new Unit[1]; rightContext[0] = ContextIndependentUnits.Get(right); Context context = LeftRightContext.Get(leftContext, rightContext); unit = _unitManager.GetUnit(name, false, context); } lastUnitName = unitName; lastUnit = unit; //this.LogInfo("Loaded " + unit.ToString()); var transitionMatrix = MatrixPool.Get(tmat); var ss = lastSenoneSequence; if (ss == null || !SameSenoneSequence(stid, lastStid)) { ss = GetSenoneSequence(stid); } lastSenoneSequence = ss; lastStid = stid; IHMM hmm = new SenoneHMM(unit, ss, transitionMatrix, GetHMMPosition(position)); HmmManager.Put(hmm); } } est.Close(); }
private void GetSenoneToCIPhone() { var inputStream = GetDataStream(Path.Combine(Location.Path, Model)); if (inputStream == null) { throw new IOException("can't find modelDef " + Model); } var est = new ExtendedStreamTokenizer(inputStream, '#', false); this.LogInfo("Loading HMM file from: " + Model); est.ExpectString(ModelVersion); _numBase = est.GetInt("numBase"); est.ExpectString("n_base"); var numTri = est.GetInt("numTri"); est.ExpectString("n_tri"); var numStateMap = est.GetInt("numStateMap"); est.ExpectString("n_state_map"); var numTiedState = est.GetInt("numTiedState"); est.ExpectString("n_tied_state"); Senone2Ci = new int[numTiedState]; est.GetInt("numContextIndependentTiedState"); est.ExpectString("n_tied_ci_state"); var numTiedTransitionMatrices = est.GetInt("numTiedTransitionMatrices"); est.ExpectString("n_tied_tmat"); var numStatePerHMM = numStateMap / (numTri + _numBase); Debug.Assert(numTiedState == MixtureWeightsPool.StatesNum); Debug.Assert(numTiedTransitionMatrices == MatrixPool.Size); // Load the base phones for (var i = 0; i < _numBase + numTri; i++) { //TODO name this magic const somehow for (var j = 0; j < 5; j++) { est.GetString(); } var tmat = est.GetInt("tmat"); for (var j = 0; j < numStatePerHMM - 1; j++) { Senone2Ci[est.GetInt("j")] = tmat; } est.ExpectString("N"); Debug.Assert(tmat < numTiedTransitionMatrices); } est.Close(); }
protected internal override GrammarNode createGrammar() { GrammarNode grammarNode = null; int num = this.createNodes(this.path); num++; GrammarNode grammarNode2 = this.createGrammarNode(num, "<sil>"); grammarNode2.setFinalNode(true); num = this.expandWordNodes(num); ExtendedStreamTokenizer extendedStreamTokenizer = new ExtendedStreamTokenizer(this.path, true); while (!extendedStreamTokenizer.isEOF()) { extendedStreamTokenizer.skipwhite(); string @string = extendedStreamTokenizer.getString(); if (@string == null) { break; } if (String.instancehelper_equals(@string, "I")) { if (!FSTGrammar.assertionsDisabled && grammarNode != null) { throw new AssertionError(); } int @int = extendedStreamTokenizer.getInt("initial ID"); string text = new StringBuilder().append("G").append(@int).toString(); grammarNode = this.createGrammarNode(@int, "<sil>"); this.nodes.put(text, grammarNode); if (this.addInitialSilenceNode) { num++; GrammarNode grammarNode3 = this.createGrammarNode(num, "<sil>"); grammarNode.add(grammarNode3, 0f); grammarNode3.add(grammarNode, 0f); } } else if (String.instancehelper_equals(@string, "T")) { int @int = extendedStreamTokenizer.getInt("this id"); int int2 = extendedStreamTokenizer.getInt("next id"); GrammarNode grammarNode3 = this.get(@int); GrammarNode grammarNode4 = this.get(int2); if (this.hasEndNode(grammarNode3)) { grammarNode3 = this.getEndNode(grammarNode3); } float num2 = 0f; string string2 = extendedStreamTokenizer.getString(); if (string2 == null || String.instancehelper_equals(string2, ",")) { if (string2 != null && String.instancehelper_equals(string2, ",")) { extendedStreamTokenizer.getString(); num2 = extendedStreamTokenizer.getFloat("probability"); } if (this.hasEndNode(grammarNode4)) { grammarNode4 = this.getEndNode(grammarNode4); } } else { string string3 = extendedStreamTokenizer.getString(); num2 = extendedStreamTokenizer.getFloat("probability"); if (String.instancehelper_equals(string3, "<unknown>")) { continue; } if (!FSTGrammar.assertionsDisabled && !this.hasWord(grammarNode4)) { throw new AssertionError(); } } grammarNode3.add(grammarNode4, this.convertProbability(num2)); } else if (String.instancehelper_equals(@string, "F")) { int @int = extendedStreamTokenizer.getInt("this id"); float @float = extendedStreamTokenizer.getFloat("probability"); GrammarNode grammarNode3 = this.get(@int); GrammarNode grammarNode4 = grammarNode2; if (this.hasEndNode(grammarNode3)) { grammarNode3 = this.getEndNode(grammarNode3); } grammarNode3.add(grammarNode4, this.convertProbability(@float)); } } extendedStreamTokenizer.close(); if (!FSTGrammar.assertionsDisabled && grammarNode == null) { throw new AssertionError(); } return(grammarNode); }
private void loadPhoneList(PropertySheet propertySheet, bool flag, InputStream inputStream, string text) { int num = 0; int num2 = 1; int num3 = 1; ExtendedStreamTokenizer extendedStreamTokenizer = new ExtendedStreamTokenizer(inputStream, 35, false); this.meansPool = new Pool("means"); this.variancePool = new Pool("variances"); this.matrixPool = new Pool("transitionmatrices"); this.senonePool = new Pool("senones"); float @float = propertySheet.getFloat("MixtureComponentScoreFloor"); float float2 = propertySheet.getFloat("mixtureWeightFloor"); float num4 = 0f; float float3 = propertySheet.getFloat("varianceFloor"); this.logger.info("Loading phone list file from: "); this.logger.info(text); string expecting = "0.1"; extendedStreamTokenizer.expectString("version"); extendedStreamTokenizer.expectString(expecting); extendedStreamTokenizer.expectString("same_sized_models"); int num5 = String.instancehelper_equals(extendedStreamTokenizer.getString(), "yes") ? 1 : 0; if (num5 != 0) { extendedStreamTokenizer.expectString("n_state"); num = extendedStreamTokenizer.getInt("numBase"); } extendedStreamTokenizer.expectString("tmat_skip"); int num6 = String.instancehelper_equals(extendedStreamTokenizer.getString(), "yes") ? 1 : 0; int num7 = 0; int num8 = 0; for (;;) { string @string = extendedStreamTokenizer.getString(); if (extendedStreamTokenizer.isEOF()) { break; } int num9 = num; if (num5 == 0) { num9 = extendedStreamTokenizer.getInt("ModelSize"); } this.phoneList.put(@string, Integer.valueOf(num9)); this.logger.fine(new StringBuilder().append("Phone: ").append(@string).append(" size: ").append(num9).toString()); int[] array = new int[num9]; string rep = "-"; int i = 0; while (i < num9) { array[i] = num7; i++; num7++; } Unit unit = this.unitManager.getUnit(@string, String.instancehelper_equals(@string, "SIL")); this.contextIndependentUnits.put(unit.getName(), unit); if (this.logger.isLoggable(Level.FINE)) { this.logger.fine(new StringBuilder().append("Loaded ").append(unit).append(" with ").append(num9).append(" states").toString()); } this.addModelToDensityPool(this.meansPool, array, num2, num3); this.addModelToDensityPool(this.variancePool, array, num2, num3); this.addModelToTransitionMatrixPool(this.matrixPool, num8, array.Length, num4, num6 != 0); this.addModelToSenonePool(this.senonePool, array, @float, float3); float[][] transitionMatrix = (float[][])this.matrixPool.get(num8); SenoneSequence senoneSequence = this.getSenoneSequence(array); SenoneHMM hmm = new SenoneHMM(unit, senoneSequence, transitionMatrix, HMMPosition.lookup(rep)); this.hmmManager.put(hmm); num8++; } this.mixtureWeights = this.initMixtureWeights(num7, num2, num3, float2); extendedStreamTokenizer.close(); }
protected internal virtual void loadHMMPool(bool useCDUnits, InputStream inputStream) { ExtendedStreamTokenizer extendedStreamTokenizer = new ExtendedStreamTokenizer(inputStream, 35, false); this.logger.fine(new StringBuilder().append("Loading HMM file from: ").append(this.location).toString()); extendedStreamTokenizer.expectString("0.3"); int @int = extendedStreamTokenizer.getInt("numBase"); extendedStreamTokenizer.expectString("n_base"); int int2 = extendedStreamTokenizer.getInt("numTri"); extendedStreamTokenizer.expectString("n_tri"); int int3 = extendedStreamTokenizer.getInt("numStateMap"); extendedStreamTokenizer.expectString("n_state_map"); int int4 = extendedStreamTokenizer.getInt("numTiedState"); extendedStreamTokenizer.expectString("n_tied_state"); int int5 = extendedStreamTokenizer.getInt("numContextIndependentTiedState"); extendedStreamTokenizer.expectString("n_tied_ci_state"); int int6 = extendedStreamTokenizer.getInt("numTiedTransitionMatrices"); extendedStreamTokenizer.expectString("n_tied_tmat"); int num = int3; int num2 = int2 + @int; int num3 = (num2 != -1) ? (num / num2) : (-num); if (!Sphinx3Loader.assertionsDisabled && int4 != this.mixtureWeights.getStatesNum()) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && int6 != this.transitionsPool.size()) { throw new AssertionError(); } for (int i = 0; i < @int; i++) { string @string = extendedStreamTokenizer.getString(); string string2 = extendedStreamTokenizer.getString(); string string3 = extendedStreamTokenizer.getString(); string string4 = extendedStreamTokenizer.getString(); string string5 = extendedStreamTokenizer.getString(); int int7 = extendedStreamTokenizer.getInt("tmat"); int[] array = new int[num3 - 1]; for (int j = 0; j < num3 - 1; j++) { array[j] = extendedStreamTokenizer.getInt("j"); if (!Sphinx3Loader.assertionsDisabled && (array[j] < 0 || array[j] >= int5)) { throw new AssertionError(); } } extendedStreamTokenizer.expectString("N"); if (!Sphinx3Loader.assertionsDisabled && !String.instancehelper_equals(string2, "-")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && !String.instancehelper_equals(string3, "-")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && !String.instancehelper_equals(string4, "-")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && int7 >= int6) { throw new AssertionError(); } Unit unit = this.unitManager.getUnit(@string, String.instancehelper_equals(string5, "filler")); this.contextIndependentUnits.put(unit.getName(), unit); if (this.logger.isLoggable(Level.FINE)) { this.logger.fine(new StringBuilder().append("Loaded ").append(unit).toString()); } if (unit.isFiller() && String.instancehelper_equals(unit.getName(), "SIL")) { unit = UnitManager.__SILENCE; } float[][] transitionMatrix = (float[][])this.transitionsPool.get(int7); SenoneSequence senoneSequence = this.getSenoneSequence(array); SenoneHMM hmm = new SenoneHMM(unit, senoneSequence, transitionMatrix, HMMPosition.lookup(string4)); this.hmmManager.put(hmm); } if (this.hmmManager.get(HMMPosition.__UNDEFINED, UnitManager.__SILENCE) == null) { string text = "Could not find SIL unit in acoustic model"; throw new IOException(text); } string text2 = ""; Unit unit2 = null; int[] ssid = null; SenoneSequence senoneSequence2 = null; for (int k = 0; k < int2; k++) { string string5 = extendedStreamTokenizer.getString(); string string6 = extendedStreamTokenizer.getString(); string string7 = extendedStreamTokenizer.getString(); string string8 = extendedStreamTokenizer.getString(); string string9 = extendedStreamTokenizer.getString(); int int8 = extendedStreamTokenizer.getInt("tmat"); int[] array2 = new int[num3 - 1]; for (int l = 0; l < num3 - 1; l++) { array2[l] = extendedStreamTokenizer.getInt("j"); if (!Sphinx3Loader.assertionsDisabled && (array2[l] < int5 || array2[l] >= int4)) { throw new AssertionError(); } } extendedStreamTokenizer.expectString("N"); if (!Sphinx3Loader.assertionsDisabled && String.instancehelper_equals(string6, "-")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && String.instancehelper_equals(string7, "-")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && String.instancehelper_equals(string8, "-")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && !String.instancehelper_equals(string9, "n/a")) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && int8 >= int6) { throw new AssertionError(); } if (useCDUnits) { string text3 = new StringBuilder().append(string5).append(' ').append(string6).append(' ').append(string7).toString(); Unit unit3; if (String.instancehelper_equals(text3, text2)) { unit3 = unit2; } else { LeftRightContext context = LeftRightContext.get(new Unit[] { (Unit)this.contextIndependentUnits.get(string6) }, new Unit[] { (Unit)this.contextIndependentUnits.get(string7) }); unit3 = this.unitManager.getUnit(string5, false, context); } text2 = text3; unit2 = unit3; if (this.logger.isLoggable(Level.FINE)) { this.logger.fine(new StringBuilder().append("Loaded ").append(unit3).toString()); } float[][] transitionMatrix2 = (float[][])this.transitionsPool.get(int8); SenoneSequence senoneSequence3 = senoneSequence2; if (senoneSequence3 == null || !this.sameSenoneSequence(array2, ssid)) { senoneSequence3 = this.getSenoneSequence(array2); } senoneSequence2 = senoneSequence3; ssid = array2; SenoneHMM hmm2 = new SenoneHMM(unit3, senoneSequence3, transitionMatrix2, HMMPosition.lookup(string8)); this.hmmManager.put(hmm2); } } extendedStreamTokenizer.close(); }
private void getSenoneToCIPhone() { InputStream dataStream = this.getDataStream("mdef"); if (dataStream == null) { string text = "can't find model definition"; throw new IOException(text); } ExtendedStreamTokenizer extendedStreamTokenizer = new ExtendedStreamTokenizer(dataStream, 35, false); this.logger.fine(new StringBuilder().append("Loading HMM file from ").append(this.location).toString()); extendedStreamTokenizer.expectString("0.3"); this.numBase = extendedStreamTokenizer.getInt("numBase"); extendedStreamTokenizer.expectString("n_base"); int @int = extendedStreamTokenizer.getInt("numTri"); extendedStreamTokenizer.expectString("n_tri"); int int2 = extendedStreamTokenizer.getInt("numStateMap"); extendedStreamTokenizer.expectString("n_state_map"); int int3 = extendedStreamTokenizer.getInt("numTiedState"); extendedStreamTokenizer.expectString("n_tied_state"); this.senone2ci = new int[int3]; extendedStreamTokenizer.getInt("numContextIndependentTiedState"); extendedStreamTokenizer.expectString("n_tied_ci_state"); int int4 = extendedStreamTokenizer.getInt("numTiedTransitionMatrices"); extendedStreamTokenizer.expectString("n_tied_tmat"); int num = int2; int num2 = @int + this.numBase; int num3 = (num2 != -1) ? (num / num2) : (-num); if (!Sphinx3Loader.assertionsDisabled && int3 != this.mixtureWeights.getStatesNum()) { throw new AssertionError(); } if (!Sphinx3Loader.assertionsDisabled && int4 != this.transitionsPool.size()) { throw new AssertionError(); } for (int i = 0; i < this.numBase + @int; i++) { int j; for (j = 0; j < 5; j++) { extendedStreamTokenizer.getString(); } j = extendedStreamTokenizer.getInt("tmat"); for (int k = 0; k < num3 - 1; k++) { this.senone2ci[extendedStreamTokenizer.getInt("j")] = j; } extendedStreamTokenizer.expectString("N"); if (!Sphinx3Loader.assertionsDisabled && j >= int4) { throw new AssertionError(); } } extendedStreamTokenizer.close(); }
/// <summary> /// Creates the grammar. /// </summary> /// <returns>The initial node for the grammar.</returns> protected override GrammarNode CreateGrammar() { GrammarNode initialNode = null; GrammarNode finalNode = null; // first pass create the FST nodes int maxNodeId = CreateNodes(_path); // create the final node: finalNode = CreateGrammarNode(++maxNodeId, IDictionary.SilenceSpelling); finalNode.SetFinalNode(true); // replace each word node with a pair of nodes, which // consists of the word node and a new dummy end node, which is // for adding null or backoff transitions maxNodeId = ExpandWordNodes(maxNodeId); ExtendedStreamTokenizer tok = new ExtendedStreamTokenizer(_path, true); // Second pass, add all of the arcs while (!tok.IsEOF()) { String token; tok.Skipwhite(); token = tok.GetString(); // System.out.println(token); if (token == null) { break; } else if (token.Equals("I")) { Debug.Assert(initialNode == null); int initialID = tok.GetInt("initial ID"); String nodeName = "G" + initialID; // TODO: FlatLinguist requires the initial grammar node // to contain a single silence. We'll do that for now, // but once the FlatLinguist is fixed, this should be // returned to its former method of creating an empty // initial grammar node // initialNode = createGrammarNode(initialID, false); initialNode = CreateGrammarNode(initialID, IDictionary.SilenceSpelling); _nodes.Put(nodeName, initialNode); // optionally add a silence node if (_addInitialSilenceNode) { GrammarNode silenceNode = CreateGrammarNode(++maxNodeId, IDictionary.SilenceSpelling); initialNode.Add(silenceNode, LogMath.LogOne); silenceNode.Add(initialNode, LogMath.LogOne); } } else if (token.Equals("T")) { int thisID = tok.GetInt("this id"); int nextID = tok.GetInt("next id"); GrammarNode thisNode = Get(thisID); GrammarNode nextNode = Get(nextID); // if the source node is an FSTGrammarNode, we want // to join the endNode to the destination node if (HasEndNode(thisNode)) { thisNode = GetEndNode(thisNode); } float lnProb = 0f; // negative natural log String output = tok.GetString(); if (output == null || output.Equals(",")) { // these are epsilon (meaning backoff) transitions if (output != null && output.Equals(",")) { tok.GetString(); // skip the word lnProb = tok.GetFloat("probability"); } // if the destination node has been expanded // we actually want to add the backoff transition // to the endNode if (HasEndNode(nextNode)) { nextNode = GetEndNode(nextNode); } } else { String word = tok.GetString(); // skip words lnProb = tok.GetFloat("probability"); if (_ignoreUnknownTransitions && word.Equals("<unknown>")) { continue; } /* * System.out.println(nextNode + ": " + output); */ Debug.Assert(HasWord(nextNode)); } thisNode.Add(nextNode, ConvertProbability(lnProb)); } else if (token.Equals("F")) { int thisID = tok.GetInt("this id"); float lnProb = tok.GetFloat("probability"); GrammarNode thisNode = Get(thisID); GrammarNode nextNode = finalNode; if (HasEndNode(thisNode)) { thisNode = GetEndNode(thisNode); } thisNode.Add(nextNode, ConvertProbability(lnProb)); } } tok.Close(); Debug.Assert(initialNode != null); return(initialNode); }