/**
* /// Sets the best scoring token for this active list
*
* /// @param token the best scoring token
*/
override public void SetBestToken(Token token)
{
_bestToken = token;
}
/// <summary>
/// Determine if the given token should be expanded.
/// </summary>
/// <param name="t">The token to test.</param>
/// <returns><code>true</code> if the token should be expanded</returns>
protected Boolean AllowExpansion(Token t)
{
return(true); // currently disabled
}
/**
* /// Collects the next set of emitting tokens from a token and accumulates them in the active or result lists
*
* /// @param token the token to collect successors from be immediately expanded are placed. Null if we should always
* /// expand all nodes.
*/
protected virtual void CollectSuccessorTokens(Token token)
{
// tokenTracker.add(token);
// tokenTypeTracker.add(token);
// If this is a final state, add it to the final list
if (token.IsFinal)
{
ResultList.Add(GetResultListPredecessor(token));
return;
}
// if this is a non-emitting token and we've already
// visited the same state during this frame, then we
// are in a grammar loop, so we don't continue to expand.
// This check only works properly if we have kept all of the
// tokens (instead of skipping the non-word tokens).
// Note that certain linguists will never generate grammar loops
// (lextree linguist for example). For these cases, it is perfectly
// fine to disable this check by setting keepAllTokens to false
if (!token.IsEmitting && (KeepAllTokens && IsVisited(token)))
{
return;
}
var state = token.SearchState;
var arcs = state.GetSuccessors();
//this.LogDebug("Total Arcs: {0}", arcs.Length);
var predecessor = GetResultListPredecessor(token);
// For each successor
// calculate the entry score for the token based upon the
// predecessor token score and the transition probabilities
// if the score is better than the best score encountered for
// the SearchState and frame then create a new token, add
// it to the lattice and the SearchState.
// If the token is an emitting token add it to the list,
// otherwise recursively collect the new tokens successors.
foreach (var arc in arcs)
{
var nextState = arc.State;
//this.LogDebug("NextState is of type: {0}",nextState.GetType());
if (_checkStateOrder)
{
CheckStateOrder(state, nextState);
}
// We're actually multiplying the variables, but since
// these come in log(), multiply gets converted to add
var logEntryScore = token.Score + arc.GetProbability();//TODO: CHECK
var bestToken = GetBestToken(nextState);
//
if (bestToken == null)
{
var newBestToken = new Token(predecessor, nextState, logEntryScore, arc.InsertionProbability, arc.LanguageProbability, CurrentFrameNumber);
TokensCreated.Value++;
SetBestToken(newBestToken, nextState);
ActiveListAdd(newBestToken);
}
else if (bestToken.Score < logEntryScore)
{
// System.out.println("Updating " + bestToken + " with " +
// newBestToken);
var oldPredecessor = bestToken.Predecessor;
bestToken.Update(predecessor, nextState, logEntryScore, arc.InsertionProbability, arc.LanguageProbability, CurrentFrameNumber);
if (BuildWordLattice && nextState is IWordSearchState)
{
LoserManager.AddAlternatePredecessor(bestToken, oldPredecessor);
}
}
else if (BuildWordLattice && nextState is IWordSearchState)
{
if (predecessor != null)
{
LoserManager.AddAlternatePredecessor(bestToken, predecessor);
}
}
}
}
protected void ActiveListAdd(Token token)
{
_activeListManager.Add(token);
}
/**
* /// Sets the best token for a given state
*
* /// @param token the best token
* /// @param state the state
*/
protected void SetBestToken(Token token, ISearchState state)
{
BestTokenMap.Add(state, token);
}