/**
* /// Expands the given hmm state tree
*
* /// @param parent the parent of the tree
* /// @param tree the tree to expand
* /// @return the final state in the tree
*/
private HMMStateState ExpandHMMTree(UnitState parent, HMMStateState tree)
{
HMMStateState retState = tree;
foreach (HmmStateArc arc in tree.HmmState.GetSuccessors())
{
HMMStateState newState;
if (arc.HmmState.IsEmitting)
{
newState = new HMMStateState(parent, arc.HmmState);
}
else
{
newState = new NonEmittingHMMState(parent, arc.HmmState);
}
SentenceHMMState existingState = GetExistingState(newState);
float logProb = arc.LogProbability;
if (existingState != null)
{
AttachState(tree, existingState, LogOne, logProb);
}
else
{
AttachState(tree, newState, LogOne, logProb);
AddStateToCache(newState);
retState = ExpandHMMTree(parent, newState);
}
}
return(retState);
}
protected void AttachState(SentenceHMMState prevState, SentenceHMMState nextState, float logLanguageProbability, float logInsertionProbability)
{
var arc = new SentenceHMMStateArc
(nextState, logLanguageProbability, logInsertionProbability);
prevState.Connect(arc);
}
/**
* /// Attaches the given unit to the given tail, expanding the unit if necessary. If an identical unit is already
* /// attached, then this path is folded into the existing path.
*
* /// @param parent the parent state
* /// @param tail the place to attach the unit to
* /// @param units the set of units
* /// @param which the index into the set of units
* /// @param leftContext the left context for the unit
* /// @param rightContext the right context for the unit
* /// @return the tail of the added unit (or null if the path was folded onto an already expanded path.
*/
private SentenceHMMState AttachUnit(PronunciationState parent,
SentenceHMMState tail, Unit[] units, int which,
UnitContext leftContext, UnitContext rightContext)
{
Unit[] lc = GetLC(leftContext, units, which);
Unit[] rc = GetRC(units, which, rightContext);
UnitContext actualRightContext = UnitContext.Get(rc);
LeftRightContext context = LeftRightContext.Get(lc, rc);
Unit cdUnit = _parent.UnitManager.GetUnit(units[which].Name, units[which].IsFiller, context);
UnitState unitState = new ExtendedUnitState(parent, which, cdUnit);
float logInsertionProbability;
if (unitState.Unit.IsSilence)
{
logInsertionProbability = _parent.LogSilenceInsertionProbability;
}
else if (unitState.Unit.IsFiller)
{
logInsertionProbability = _parent._logFillerInsertionProbability;
}
else if (unitState.GetWhich() == 0)
{
logInsertionProbability = _parent._logWordInsertionProbability;
}
else
{
logInsertionProbability = _parent._logUnitInsertionProbability;
}
// check to see if this state already exists, if so
// branch to it and we are done, otherwise, branch to
// the new state and expand it.
SentenceHMMState existingState = GetExistingState(unitState);
if (existingState != null)
{
AttachState(tail, existingState, LogOne, logInsertionProbability);
// T(" Folding " + existingState);
return(null);
}
else
{
AttachState(tail, unitState, LogOne, logInsertionProbability);
AddStateToCache(unitState);
// T(" Attaching " + unitState);
tail = ExpandUnit(unitState);
// if we are attaching the last state of a word, then
// we add it to the exitPoints table. the exit points
// table is indexed by a ContextPair, consisting of
// the exiting left context and the right context.
if (unitState.IsLast())
{
UnitContext nextLeftContext = GenerateNextLeftContext(leftContext, units[which]);
ContextPair cp = ContextPair.Get(nextLeftContext, actualRightContext);
// T(" Adding to exitPoints " + cp);
AddExitPoint(cp, tail);
}
return(tail);
}
}
/**
* /// Creates a SentenceHMMState
*
* /// @param name the name of the current SentenceHMMState
* /// @param parent the parent of the current instance
* /// @param which the index of the current instance
*/
protected SentenceHMMState(String name, SentenceHMMState parent, int which)
: this()
{
this.Name = name + which;
this._parent = parent;
SetWhich(which);
SetProcessed(false);
SetColor(Color.Red);
}
/**
* /// Gets a SentenceHMMStateArc. The arc is drawn from a pool of arcs.
*
* /// @param nextState the next state
* /// @param logLanguageProbability the log language probability
* /// @param logInsertionProbability the log insertion probability
*/
protected SentenceHMMStateArc GetArc(SentenceHMMState nextState, float logLanguageProbability, float logInsertionProbability)
{
var arc = new SentenceHMMStateArc(nextState,
logLanguageProbability * _languageWeight,
logInsertionProbability);
var pooledArc = ArcPool.cache(arc);
ActualArcs.Value = ArcPool.Misses;
TotalArcs.Value = ArcPool.Hits + ArcPool.Misses;
return(pooledArc == null ? arc : pooledArc);
}
/// <summary>
/// Adds an exit point to this gstate
/// </summary>
/// <param name="cp">the context tag for the state</param>
/// <param name="state">the state associated with the tag</param>
private void AddExitPoint(ContextPair cp, SentenceHMMState state)
{
List <ISearchState> list = _exitPoints.Get(cp);
if (list == null)
{
list = new List <ISearchState>();
_exitPoints.Put(cp, list);
}
list.Add(state);
}
/**
* /// Creates a SentenceHMMStateArc
*
* /// @param nextState the next state
* /// @param logLanguageProbability the log language probability
* /// @param logInsertionProbability the log insertion probability
*/
public SentenceHMMStateArc(SentenceHMMState nextState,
float logLanguageProbability,
float logInsertionProbability)
{
_nextState = nextState;
LanguageProbability = logLanguageProbability;
InsertionProbability = logInsertionProbability;
_hashCode = 111 + nextState.GetHashCode() +
17 * Float.FloatToIntBits(logLanguageProbability) +
23 * Float.FloatToIntBits(logInsertionProbability);
}
/**
* /// Attaches one SentenceHMMState as a child to another, the transition has the given probability
*
* /// @param prevState the parent state
* /// @param nextState the child state
* /// @param logLanguageProbablity the language probability of transition in the LogMath log domain
* /// @param logInsertionProbablity insertion probability of transition in the LogMath log domain
*/
protected void AttachState(SentenceHMMState prevState,
SentenceHMMState nextState,
float logLanguageProbablity,
float logInsertionProbablity)
{
prevState.Connect(_parent.GetArc(nextState,
logLanguageProbablity, logInsertionProbablity));
if (_parent._showCompilationProgress && _parent._totalStateCounter++ % 1000 == 0)
{
this.LogInfo(".");
}
}
/// <summary>
/// connect all the states in the source list to the states in the destination list
/// </summary>
/// <param name="sourceList">the set of source states</param>
/// <param name="destList">the set of destination states.</param>
/// <param name="logLangProb"></param>
private void Connect(List <ISearchState> sourceList, List <ISearchState> destList, float logLangProb)
{
foreach (ISearchState source in sourceList)
{
SentenceHMMState sourceState = (SentenceHMMState)source;
foreach (ISearchState dest in destList)
{
SentenceHMMState destState = (SentenceHMMState)dest;
sourceState.Connect(_parent.GetArc(destState, logLangProb, LogOne));
_exitConnections++;
}
}
}
/**
* /// Expands the unit into a set of HMMStates. If the unit is a silence unit add an optional loopback to the
* /// tail.
*
* /// @param unit the unit to expand
* /// @return the head of the hmm tree
*/
protected SentenceHMMState ExpandUnit(UnitState unit)
{
SentenceHMMState tail = GetHMMStates(unit);
// if the unit is a silence unit add a loop back from the
// tail silence unit
if (unit.Unit.IsSilence)
{
// add the loopback, but don't expand it // anymore
AttachState(tail, unit, LogOne, _parent.LogSilenceInsertionProbability);
}
return(tail);
}
/**
* /// Expand the pronunciation given the left context
*
* /// @param leftContext the left context
* /// @param pronunciation the pronunciation to expand
* /// @param which unique ID for this pronunciation
*/
// Each GState maintains a list of entry points. This list of
// entry points is used when connecting up the end states of
// one GState to the beginning states in another GState. The
// entry points are tagged by a ContextPair which represents
// the left context upon entering the state (the left context
// of the initial units of the state), and the right context
// of the previous states (corresponding to the starting
// contexts for this state).
//
// When expanding a pronunciation, the following steps are
// taken:
// 1) Get the starting context for the pronunciation.
// This is the set of units that correspond to the start
// of the pronunciation.
//
// 2) Create a new PronunciationState for the
// pronunciation.
//
// 3) Add the PronunciationState to the entry point table
// (a hash table keyed by the ContextPair(LeftContext,
// StartingContext).
//
// 4) Generate the set of context dependent units, using
// the left and right context of the GState as necessary.
// Note that there will be fan out at the end of the
// pronunciation to allow for units with all of the
// various right contexts. The point where the fan-out
// occurs is the (length of the pronunciation - the max
// right context size).
//
// 5) Attach each cd unit to the tree
//
// 6) Expand each cd unit into the set of HMM states
//
// 7) Attach the optional and looping back silence cd
// unit
//
// 8) Collect the leaf states of the tree and add them to
// the exitStates list.
private void ExpandPronunciation(UnitContext leftContext, Pronunciation pronunciation, int which)
{
UnitContext startingContext = GetStartingContext(pronunciation);
// Add the pronunciation state to the entry point list
// (based upon its left and right context)
string pname = "P(" + pronunciation.Word + '[' + leftContext
+ ',' + startingContext + "])-G" + GetNode().ID;
PronunciationState ps = new PronunciationState(pname, pronunciation, which);
_parent.T(" Expanding " + ps.Pronunciation + " for lc " + leftContext);
ContextPair cp = ContextPair.Get(leftContext, startingContext);
var epList = _entryPoints.Get(cp);
if (epList == null)
{
throw new Error("No EP list for context pair " + cp);
}
else
{
epList.Add(ps);
}
Unit[] units = pronunciation.Units;
int fanOutPoint = units.Length - GetRightContextSize();
if (fanOutPoint < 0)
{
fanOutPoint = 0;
}
SentenceHMMState tail = ps;
for (int i = 0; tail != null && i < fanOutPoint; i++)
{
tail = AttachUnit(ps, tail, units, i, leftContext, UnitContext.Empty);
}
SentenceHMMState branchTail = tail;
foreach (UnitContext finalRightContext in _rightContexts)
{
tail = branchTail;
for (int i = fanOutPoint; tail != null && i < units.Length; i++)
{
tail = AttachUnit(ps, tail, units, i, leftContext, finalRightContext);
}
}
}
/**
* /// Visit all of the states starting at start with the given vistor
*
* /// @param visitor the state visitor
* /// @param start the place to start the search
* /// @param sorted if true, states are sorted before visited
* /// @return true if the visiting was terminated before all nodes were visited
*/
public static Boolean VisitStates(ISentenceHMMStateVisitor visitor, SentenceHMMState start, Boolean sorted)
{
IEnumerable <SentenceHMMState> states = CollectStates(start);
if (sorted)
{
// sort the states by stateNumber
var sortedStates = new SortedSet <SentenceHMMState>(new FirstComparer());
sortedStates.AddAll(states);
states = sortedStates;
}
foreach (var state in states)
{
if (visitor.Visit(state))
{
return(true);
}
}
return(false);
}
/** Constructs a phone loop search graph. */
public PhoneLoopSearchGraph(CIPhoneLoop parent)
{
_parent = parent;
ExistingStates = new Dictionary <string, ISearchState>();
FirstState = new UnknownWordState();
SentenceHMMState branchState = new BranchOutState(FirstState);
AttachState(FirstState, branchState, _parent.LogOne, _parent.LogOne);
SentenceHMMState lastState = new LoopBackState(FirstState);
lastState.SetFinalState(true);
AttachState(lastState, branchState, _parent.LogOne, _parent.LogOne);
for (var i = _parent.Model.GetContextIndependentUnitIterator(); i.MoveNext();)
{
var unitState = new UnitState(i.Current, HMMPosition.Undefined);
var debug = unitState.ToString();
// attach unit state to the branch out state
AttachState(branchState, unitState, _parent.LogOne, _parent.LogPhoneInsertionProbability);
var hmm = _parent.Model.LookupNearestHMM(unitState.Unit, unitState.GetPosition(), false);
var initialState = hmm.GetInitialState();
var hmmTree = new HMMStateState(unitState, initialState);
AddStateToCache(hmmTree);
// attach first HMM state to the unit state
AttachState(unitState, hmmTree, _parent.LogOne, _parent.LogOne);
// expand the HMM tree
var finalState = ExpandHMMTree(unitState, hmmTree);
// attach final state of HMM tree to the loopback state
AttachState(finalState, lastState, _parent.LogOne, _parent.LogOne);
}
}
/// <summary>
/// Collect all states starting from the given start state
/// </summary>
/// <param name="start">the state to start the search from</param>
/// <returns>set of collected state</returns>
public static HashSet <SentenceHMMState> CollectStates(SentenceHMMState start)
{
var visitedStates = new HashSet <SentenceHMMState>();
var queue = new List <SentenceHMMState> {
start
};
while (queue.Count != 0)
{
var state = queue[0];
queue.RemoveAt(0);
visitedStates.Add(state);
var successors = ((ISearchState)state).GetSuccessors();
foreach (var arc in successors)
{
var nextState = (SentenceHMMState)arc.State;
if (!visitedStates.Contains(nextState) && !queue.Contains(nextState))
{
queue.Add(nextState);
}
}
}
return(visitedStates);
}
/**
* /// Creates a NonEmittingHMMState
*
* /// @param parent the parent of this state
* /// @param hmmState the hmmState associated with this state
*/
public NonEmittingHMMState(SentenceHMMState parent, IHMMState hmmState)
: base(parent, hmmState)
{
}
/**
* /// Adds the given state to the cache of states
*
* /// @param state the state to add
*/
private void AddStateToCache(SentenceHMMState state)
{
_existingStates.Put(state.Signature, state);
}
/**
* /// Checks to see if a state that matches the given state already exists
*
* /// @param state the state to check
* /// @return true if a state with an identical signature already exists.
*/
private SentenceHMMState GetExistingState(SentenceHMMState state)
{
return(_existingStates.Get(state.Signature));
}
/**
* /// Adds the given state to the cache of states
*
* /// @param state the state to add
*/
protected void AddStateToCache(SentenceHMMState state)
{
ExistingStates.Add(state.Signature, state);
}
/**
* /// Creates a HMMStateState
*
* /// @param parent the parent of this state
* /// @param hmmState the hmmState associated with this state
*/
public HMMStateState(SentenceHMMState parent, IHMMState hmmState)
: base("S", parent, hmmState.State)
{
_hmmState = hmmState;
_isEmitting = hmmState.IsEmitting;
}
/// <summary>
/// Compiles the grammar into a sentence HMM. A GrammarJob is created for the
/// initial grammar node and added to the GrammarJob queue. While there are
/// jobs left on the grammar job queue, a job is removed from the queue and
/// the associated grammar node is expanded and attached to the tails.
/// GrammarJobs for the successors are added to the grammar job queue.
/// </summary>
/// <returns></returns>
protected HashSet <SentenceHMMState> CompileGrammar()
{
InitialGrammarState = Grammar.InitialNode;
NodeStateMap = new HashMap <GrammarNode, GState>();
// create in declaration section (22.12.2014)
ArcPool = new Cache <SentenceHMMStateArc>();
var gstateList = new List <GState>();
TimerPool.GetTimer(this, "Compile").Start();
// get the nodes from the grammar and create states
// for them. Add the non-empty gstates to the gstate list.
TimerPool.GetTimer(this, "Create States").Start();
foreach (var grammarNode in Grammar.GrammarNodes)
{
var gstate = CreateGState(grammarNode);
gstateList.Add(gstate);
}
TimerPool.GetTimer(this, "Create States").Stop();
AddStartingPath();
// ensures an initial path to the start state
// Prep all the gstates, by gathering all of the contexts up
// this allows each gstate to know about its surrounding contexts
TimerPool.GetTimer(this, "Collect Contexts").Start();
foreach (var gstate in gstateList)
{
gstate.CollectContexts();
}
TimerPool.GetTimer(this, "Collect Contexts").Stop();
// now all gstates know all about their contexts, we can expand them fully
TimerPool.GetTimer(this, "Expand States").Start();
foreach (var gstate in gstateList)
{
gstate.Expand();
}
TimerPool.GetTimer(this, "Expand States").Stop();
// now that all states are expanded fully, we can connect all the states up
TimerPool.GetTimer(this, "Connect Nodes").Start();
foreach (var gstate in gstateList)
{
gstate.Connect();
}
TimerPool.GetTimer(this, "Connect Nodes").Stop();
var initialState = FindStartingState();
// add an out-of-grammar branch if configured to do so
if (AddOutOfGrammarBranch)
{
var phoneLoop = new CIPhoneLoop(PhoneLoopAcousticModel, LogPhoneInsertionProbability);
var firstBranchState = (SentenceHMMState)phoneLoop.GetSearchGraph().InitialState;
initialState.Connect(GetArc(firstBranchState, LogOne, LogOutOfGrammarBranchProbability));
}
_searchGraph = new FlatSearchGraph(initialState);
TimerPool.GetTimer(this, "Compile").Stop();
// Now that we are all done, dump out some interesting
// information about the process
if (_dumpGStates)
{
foreach (var grammarNode in Grammar.GrammarNodes)
{
var gstate = GetGState(grammarNode);
gstate.DumpInfo();
}
}
NodeStateMap = null;
ArcPool = null;
return(SentenceHMMState.CollectStates(initialState));
}
public bool Visit(SentenceHMMState state)
{
state.SetProcessed(false);
return(false);
}
/**
* /// Searches the set of arcs for an arc that points to a state with an identical value
*
* /// @param state the state to search for
* /// @return the arc or null if none could be found.
*/
public SentenceHMMStateArc FindArc(SentenceHMMState state)
{
return(arcs.Get(state.GetValueSignature()));
}
public LoopBackState(SentenceHMMState parent)
: base("CIPhonesLoopBackState", parent, 0)
{
}
public bool Visit(SentenceHMMState state)
{
state.Dump();
return(false);
}
public BranchOutState(SentenceHMMState parent)
: base("BranchOutState", parent, 0)
{
}