private CompiledNode compileNode(UnCompiledNode <T> nodeIn, int tailLength)
{
long node;
long bytesPosStart = bytes.getPosition();
//TODO: deduphash
node = fst.addNode(this, nodeIn);
Debug.Assert(node != -2);
long bytesPosEnd = bytes.getPosition();
if (bytesPosEnd != bytesPosStart)
{
// The FST added a new node:
Debug.Assert(bytesPosEnd > bytesPosStart);
lastFrozenNode = node;
}
nodeIn.clear();
CompiledNode fn = new CompiledNode();
fn.node = node;
return(fn);
}
private void freezeTail(int prefixLenPlus1)
{
int downTo = Math.Max(1, prefixLenPlus1);
for (int idx = lastInput.length; idx >= downTo; idx--)
{
bool doPrune = false;
bool doCompile = false;
UnCompiledNode <T> node = frontier[idx];
UnCompiledNode <T> parent = frontier[idx - 1];
if (node.inputCount < minSuffixCount1)
{
doPrune = true;
doCompile = true;
}
else if (idx > prefixLenPlus1)
{
// prune if parent's inputCount is less than suffixMinCount2
if (parent.inputCount < minSuffixCount2 || (minSuffixCount2 == 1 && parent.inputCount == 1 && idx > 1))
{
// my parent, about to be compiled, doesn't make the cut, so
// I'm definitely pruned
// if minSuffixCount2 is 1, we keep only up
// until the 'distinguished edge', ie we keep only the
// 'divergent' part of the FST. if my parent, about to be
// compiled, has inputCount 1 then we are already past the
// distinguished edge. NOTE: this only works if
// the FST outputs are not "compressible" (simple
// ords ARE compressible).
doPrune = true;
}
else
{
// my parent, about to be compiled, does make the cut, so
// I'm definitely not pruned
doPrune = false;
}
doCompile = true;
}
else
{
// if pruning is disabled (count is 0) we can always
// compile current node
doCompile = minSuffixCount2 == 0;
}
if (node.inputCount < minSuffixCount2 || (minSuffixCount2 == 1 && node.inputCount == 1 && idx > 1))
{
// drop all arcs
for (int arcIdx = 0; arcIdx < node.numArcs; arcIdx++)
{
UnCompiledNode <T> target = (UnCompiledNode <T>)node.arcs[arcIdx].target;
target.clear();
}
node.numArcs = 0;
}
if (doPrune)
{
// this node doesn't make it -- deref it
node.clear();
parent.deleteLast(lastInput.ints[idx - 1], node);
}
else
{
if (minSuffixCount2 != 0)
{
//TODO: minSuffixCount2 is always 0 for now.
//compileAllTargets(node, lastInput.length()-idx);
}
T nextFinalOutput = node.output;
// We "fake" the node as being final if it has no
// outgoing arcs; in theory we could leave it
// as non-final (the FST can represent this), but
// FSTEnum, Util, etc., have trouble w/ non-final
// dead-end states:
bool isFinal = node.isFinal || node.numArcs == 0;
if (doCompile)
{
// this node makes it and we now compile it. first,
// compile any targets that were previously
// undecided:
parent.replaceLast(lastInput.ints[idx - 1],
compileNode(node, 1 + lastInput.length - idx),
nextFinalOutput,
isFinal);
}
else
{
// replaceLast just to install
// nextFinalOutput/isFinal onto the arc
parent.replaceLast(lastInput.ints[idx - 1],
node,
nextFinalOutput,
isFinal);
// this node will stay in play for now, since we are
// undecided on whether to prune it. later, it
// will be either compiled or pruned, so we must
// allocate a new node:
frontier[idx] = new UnCompiledNode <T>(this, idx);
}
}
}
}