public bool AddState(State newState, ParsingOperation op)
{
var addedToStates = false;
var addedToCompletedViterbi = false;
State oldState;
var oldOrNewState = false;
var isCompleted = newState.IsCompleted();
//step 1: check in states dictionary
//if does not exist, add. otherwise, merge.
if (!StatesDictionary.TryGetValue(newState, out oldState))
{
StatesDictionary.Add(newState, newState);
newState.EndColumn = this;
States.Add(newState);
addedToStates = true;
}
else
{
if (op != ParsingOperation.Scan)
{
//the state is already present, modulo the stack. we need to share stacks:
Merge(oldState, newState); //merging into the old state.
oldOrNewState = true;
//inner probability of completed, but not predicted, item changes upon merge
//(see stolcke 1995)
if (oldState.Node != null && op == ParsingOperation.Complete)
{
oldState.Node.LogProbability =
Grammar.GetProbabilitySumOfTwoLogProbabilities(oldState.Node.LogProbability,
newState.Node.LogProbability);
if (oldState.Node.LogProbability < 0)
{
throw new Exception(
string.Format(
"case2: impossible probability resulted from summation of two log probabilities {0},",
oldState.Node.LogProbability));
}
}
}
else
{
throw new Exception("scan - identical states - not supposed to happen?!?");
}
}
//step 2: if completed, check in completed dictionary.
//if does not exist, add. otherwise, replace with old state.
if (isCompleted)
{
var stateTocheck = oldOrNewState ? oldState : newState;
State oldStateViterbi;
if (!StateViterbiCompletedDictionary.TryGetValue(stateTocheck, out oldStateViterbi))
{
StateViterbiCompletedDictionary.Add(stateTocheck, stateTocheck);
addedToCompletedViterbi = true;
}
else
{
if (stateTocheck.Node.LogProbability < oldStateViterbi.Node.LogProbability)
{
oldStateViterbi.Node = stateTocheck.Node;
if (oldStateViterbi.Node.LogProbability < 0)
{
throw new LogException(
string.Format(
"columnarrr! NODE log probability lower than 0: {0}, reductor state: {1}",
oldStateViterbi.Node.LogProbability, oldStateViterbi));
}
if (debug)
{
Console.WriteLine("state {0} has higher probability than state {1}. ",
stateTocheck.StateNumber,
oldStateViterbi.StateNumber);
}
}
}
}
var finalAdd = false;
//step 3: determine whether the state needs to be added to the agendas.
if (!isCompleted && addedToStates)
{
ActionableNonCompleteStates.Enqueue(newState);
var term = newState.NextProductionTerm();
//check if the next nonterminal leads to an expansion of null production, if yes, insert it to the
//completed rules.
//initially just check if the next nonterminal is nullable production
if (grammar.nullableProductions.ContainsKey(term))
{
//spontaneous dot shift.
EpsilonComplete(newState);
}
finalAdd = true;
}
else if (isCompleted && addedToCompletedViterbi)
{
//note: it is possible that a new state has been added to the state dictionary
//but it is not added to the viterbi completed state dictionary.
//in such a case, do not push the item to the agenda.
//with the same LHS side
if (!ActionableCompleteStates.ContainsKey(newState))
{
ActionableCompleteStates[newState] = new Queue <State>();
}
ActionableCompleteStates[newState].Enqueue(newState);
finalAdd = true;
}
return(finalAdd);
}