public void AddChildren(Node v, Node w = null)
{
if (Children == null)
Children = new List<Node>();
Children.Add(v);
if (w != null)
{
Children.Insert(0, w);
}
}
public State(Rule r, int dotIndex, Column c, Node n)
{
Rule = r;
DotIndex = dotIndex;
StartColumn = c;
EndColumn = null;
Node = n;
StateNumber = stateCounter;
stateCounter += 1;
LogProbability = -1;
}
//public static int GetDerivationTreeLength(Node n)
//{
// //rule number 0 is a scan operation of some POS / word.
// if (n.RuleNumber == Grammar.SCAN_RULE_NUMBER)
// return 0;
// var selfLength = RequiredBitsGivenLogProbability(n.LogProbability);
// if (selfLength < 0)
// {
// Console.WriteLine("rule number {0}, prob {1}", n.RuleNumber, n.LogProbability);
// }
// if (n.Children == null)
// return selfLength;
// var sumOfChildrenLengths = n.Children.Sum(child => GetDerivationTreeLength(child));
// return selfLength + sumOfChildrenLengths;
//}
public static void CollectRuleUsages(Node n, Dictionary<int, int> ruleCounts, int sentenceCount)
{
if (n.Children != null)
{
foreach (var child in n.Children)
CollectRuleUsages(child, ruleCounts, sentenceCount);
}
if (n.RuleNumber != 0) //SCAN_RULE_NUMBER = 0.
{
if (!ruleCounts.ContainsKey(n.RuleNumber)) ruleCounts[n.RuleNumber] = 0;
ruleCounts[n.RuleNumber] += sentenceCount;
//add +1 to the count of the rule, multiplied by the number of times the sentence appears in the text (sentenceCount).
}
}
private void Scan(Column col, State state, string term, string token)
{
//if there is nonempty stack arriving to this part of speech term, stop here - the derivation is wrong.
//SEFI - note: this is a restriction on the form of your grammar.
//consider removing it to see the conequences.
//if (Grammar.isPOS(term))
{
if (!state.NextProductionTerm().IsStackEmpty()) return;
}
var v = new Node(term, col.Index - 1, col.Index)
{
AssociatedTerminal = token,
LogProbability = 0.0f,
Bits = 1
};
var y = State.MakeNode(state, col.Index, v);
var newState = new State(state.Rule, state.DotIndex + 1, state.StartColumn, y);
col.AddState(newState, ParsingOperation.Scan);
if (Debug)
Console.WriteLine("{0} & {1} & {2} & Scanned from State {3}, word: {4}\\\\", newState.StateNumber,
newState, col.Index,
state.StateNumber, token);
if (newState.Node.LogProbability < 0)
{
throw new LogException(string.Format("scanarrrr! NODE log probability lower than 0: {0}, state: {1}",
newState.Node.LogProbability, newState));
}
}
public static Node MakeNode(State predecessorState, int endIndex, Node reductor)
{
Node y;
var nextDotIndex = predecessorState.DotIndex + 1;
var nodeName = RuleWithDotNotation(predecessorState.Rule, nextDotIndex);
if (nextDotIndex == 1 && predecessorState.Rule.Production.Length > 1)
{
y = reductor;
if (predecessorState.Node == null)
{
y.LogProbability = predecessorState.LogProbability + reductor.LogProbability;
y.Bits = RequiredBitsGivenLogProbability(predecessorState.LogProbability) + reductor.Bits;
if (predecessorState.LogProbability < 0 || reductor.LogProbability < 0)
{
throw new Exception(
string.Format("first case y NODE log probability lower than 0: {0} < , {1} , {2}",
y.LogProbability, predecessorState.LogProbability, reductor.LogProbability));
}
}
else
throw new Exception("arrived in a clause that should not be possible. make_node");
}
else
{
y = new Node(nodeName, predecessorState.StartColumn.Index, endIndex);
if (!y.HasChildren())
y.AddChildren(reductor, predecessorState.Node);
if (predecessorState.Node == null)
{
y.LogProbability = predecessorState.LogProbability + reductor.LogProbability;
y.Bits = RequiredBitsGivenLogProbability(predecessorState.LogProbability) + reductor.Bits;
if (predecessorState.LogProbability < 0 || reductor.LogProbability < 0)
{
throw new Exception(
string.Format("second case y NODE log probability lower than 0: {0} = , {1} + {2}",
y.LogProbability, predecessorState.LogProbability, reductor.LogProbability));
}
}
else
{
y.LogProbability = predecessorState.Node.LogProbability + reductor.LogProbability;
y.Bits = predecessorState.Node.Bits + reductor.Bits;
if (predecessorState.Node.LogProbability < 0 || reductor.LogProbability < 0)
{
throw new Exception(
string.Format("third case y NODE log probability lower than 0: {0} = , {1} + {2}",
y.LogProbability, predecessorState.Node.LogProbability, reductor.LogProbability));
}
}
y.RuleNumber = predecessorState.Rule.Number;
}
return y;
}