private CompiledNode CompileNode(UnCompiledNode <T> nodeIn, int tailLength)
{
long node;
if (DedupHash != null && (DoShareNonSingletonNodes || nodeIn.NumArcs <= 1) && tailLength <= ShareMaxTailLength)
{
if (nodeIn.NumArcs == 0)
{
node = Fst.AddNode(nodeIn);
}
else
{
node = DedupHash.Add(nodeIn);
}
}
else
{
node = Fst.AddNode(nodeIn);
}
Debug.Assert(node != -2);
nodeIn.Clear();
CompiledNode fn = new CompiledNode();
fn.Node = node;
return(fn);
}
private CompiledNode CompileNode(UnCompiledNode <T> nodeIn, int tailLength)
{
long node;
if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.NumArcs <= 1) && tailLength <= shareMaxTailLength)
{
if (nodeIn.NumArcs == 0)
{
node = fst.AddNode(nodeIn);
}
else
{
node = dedupHash.Add(nodeIn);
}
}
else
{
node = fst.AddNode(nodeIn);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(node != -2);
}
nodeIn.Clear();
return(new CompiledNode {
Node = node
});
}
private void DoFreezeTail(int prefixLenPlus1)
{
if (FreezeTail_Renamed != null)
{
// Custom plugin:
FreezeTail_Renamed.Freeze(Frontier, prefixLenPlus1, LastInput);
}
else
{
//System.out.println(" compileTail " + prefixLenPlus1);
int downTo = Math.Max(1, prefixLenPlus1);
for (int idx = LastInput.Length; idx >= downTo; idx--)
{
bool doPrune = false;
bool doCompile = false;
UnCompiledNode <T> node = Frontier[idx];
UnCompiledNode <T> parent = Frontier[idx - 1];
if (node.InputCount < MinSuffixCount1)
{
doPrune = true;
doCompile = true;
}
else if (idx > prefixLenPlus1)
{
// prune if parent's inputCount is less than suffixMinCount2
if (parent.InputCount < MinSuffixCount2 || (MinSuffixCount2 == 1 && parent.InputCount == 1 && idx > 1))
{
// my parent, about to be compiled, doesn't make the cut, so
// I'm definitely pruned
// if minSuffixCount2 is 1, we keep only up
// until the 'distinguished edge', ie we keep only the
// 'divergent' part of the FST. if my parent, about to be
// compiled, has inputCount 1 then we are already past the
// distinguished edge. NOTE: this only works if
// the FST outputs are not "compressible" (simple
// ords ARE compressible).
doPrune = true;
}
else
{
// my parent, about to be compiled, does make the cut, so
// I'm definitely not pruned
doPrune = false;
}
doCompile = true;
}
else
{
// if pruning is disabled (count is 0) we can always
// compile current node
doCompile = MinSuffixCount2 == 0;
}
//System.out.println(" label=" + ((char) lastInput.ints[lastInput.offset+idx-1]) + " idx=" + idx + " inputCount=" + frontier[idx].inputCount + " doCompile=" + doCompile + " doPrune=" + doPrune);
if (node.InputCount < MinSuffixCount2 || (MinSuffixCount2 == 1 && node.InputCount == 1 && idx > 1))
{
// drop all arcs
for (int arcIdx = 0; arcIdx < node.NumArcs; arcIdx++)
{
UnCompiledNode <T> target = (UnCompiledNode <T>)node.Arcs[arcIdx].Target;
target.Clear();
}
node.NumArcs = 0;
}
if (doPrune)
{
// this node doesn't make it -- deref it
node.Clear();
parent.DeleteLast(LastInput.Ints[LastInput.Offset + idx - 1], node);
}
else
{
if (MinSuffixCount2 != 0)
{
CompileAllTargets(node, LastInput.Length - idx);
}
T nextFinalOutput = node.Output;
// We "fake" the node as being final if it has no
// outgoing arcs; in theory we could leave it
// as non-final (the FST can represent this), but
// FSTEnum, Util, etc., have trouble w/ non-final
// dead-end states:
bool isFinal = node.IsFinal || node.NumArcs == 0;
if (doCompile)
{
// this node makes it and we now compile it. first,
// compile any targets that were previously
// undecided:
parent.ReplaceLast(LastInput.Ints[LastInput.Offset + idx - 1], CompileNode(node, 1 + LastInput.Length - idx), nextFinalOutput, isFinal);
}
else
{
// replaceLast just to install
// nextFinalOutput/isFinal onto the arc
parent.ReplaceLast(LastInput.Ints[LastInput.Offset + idx - 1], node, nextFinalOutput, isFinal);
// this node will stay in play for now, since we are
// undecided on whether to prune it. later, it
// will be either compiled or pruned, so we must
// allocate a new node:
Frontier[idx] = new UnCompiledNode <T>(this, idx);
}
}
}
}
}