private Fst CreateUnsorted()
{
var fst = new Fst(new TropicalSemiring());
var s1 = new State(0f);
var s2 = new State(0f);
var s3 = new State(0f);
// State 0
fst.AddState(s1);
s1.AddArc(new Arc(1, 4, 0f, s2));
s1.AddArc(new Arc(3, 5, 0f, s2));
s1.AddArc(new Arc(2, 3, 0f, s2));
s1.AddArc(new Arc(5, 2, 0f, s3));
s1.AddArc(new Arc(4, 1, 0f, s3));
// State 1
fst.AddState(s2);
s2.AddArc(new Arc(2, 3, 0f, s2));
s2.AddArc(new Arc(3, 1, 0f, s3));
s2.AddArc(new Arc(1, 2, 0f, s3));
// State 2 (final)
fst.AddState(s3);
return(fst);
}
private Fst CreatePo()
{
var ts = new TropicalSemiring();
var fst = new Fst(ts);
var s1 = new State(ts.Zero);
var s2 = new State(ts.Zero);
var s3 = new State(ts.Zero);
var s4 = new State(2f);
// State 0
fst.AddState(s1);
s1.AddArc(new Arc(5, 5, 1f, s2));
s1.AddArc(new Arc(4, 4, 3f, s2));
fst.SetStart(s1);
// State 1
fst.AddState(s2);
s2.AddArc(new Arc(3, 3, 7f, s2));
s2.AddArc(new Arc(2, 2, 5f, s3));
// State 2
fst.AddState(s3);
s3.AddArc(new Arc(1, 1, 9f, s4));
// State 3
fst.AddState(s4);
return(fst);
}
/**
* /// Get a filter to use for avoiding multiple epsilon paths in the resulting
* /// Fst
* ///
* /// See: M. Mohri, "Weighted automata algorithms", Handbook of Weighted
* /// Automata. Springer, pp. 213-250, 2009.
* ///
* /// @param syms the gilter's input/output symbols
* /// @param semiring the semiring to use in the operation
* /// @return the filter
*/
public static Fst GetFilter(String[] syms, Semiring semiring)
{
Fst filter = new Fst(semiring);
if (syms == null)
{
return(filter); //empty one
}
int e1Index = syms == null ? 0 : syms.Length;
int e2Index = syms == null ? 1 : syms.Length + 1;
filter.Isyms = syms;
filter.Osyms = syms;
// State 0
State s0 = new State(syms.Length + 3);
s0.FinalWeight = semiring.One;
State s1 = new State(syms.Length);
s1.FinalWeight = semiring.One;
State s2 = new State(syms.Length);
s2.FinalWeight = semiring.One;
filter.AddState(s0);
s0.AddArc(new Arc(e2Index, e1Index, semiring.One, s0));
s0.AddArc(new Arc(e1Index, e1Index, semiring.One, s1));
s0.AddArc(new Arc(e2Index, e2Index, semiring.One, s2));
for (int i = 1; i < syms.Length; i++)
{
s0.AddArc(new Arc(i, i, semiring.One, s0));
}
filter.SetStart(s0);
// State 1
filter.AddState(s1);
s1.AddArc(new Arc(e1Index, e1Index, semiring.One, s1));
for (int i = 1; i < syms.Length; i++)
{
s1.AddArc(new Arc(i, i, semiring.One, s0));
}
// State 2
filter.AddState(s2);
s2.AddArc(new Arc(e2Index, e2Index, semiring.One, s2));
for (int i = 1; i < syms.Length; i++)
{
s2.AddArc(new Arc(i, i, semiring.One, s0));
}
return(filter);
}
/**
* /// Extends an Fst to a single final state.
* ///
* /// It adds a new final state with a 0.0 (Semiring's 1) final wight and
* /// connects the current final states to it using epsilon transitions with
* /// weight equal to the original final state's weight.
* ///
* /// @param fst the Fst to extend
*/
public static void Apply(Fst fst)
{
Semiring semiring = fst.Semiring;
List <State> fStates = new List <State>();
int numStates = fst.GetNumStates();
for (int i = 0; i < numStates; i++)
{
State s = fst.GetState(i);
if (s.FinalWeight != semiring.Zero)
{
fStates.Add(s);
}
}
// Add a new single final
State newFinal = new State(semiring.One);
fst.AddState(newFinal);
foreach (State s in fStates)
{
// add epsilon transition from the old final to the new one
s.AddArc(new Arc(0, 0, s.FinalWeight, newFinal));
// set old state's weight to zero
s.FinalWeight = semiring.Zero;
}
}
/**
* /// Reverses an fst
* ///
* /// @param fst the fst to reverse
* /// @return the reversed fst
*/
public static Fst Get(Fst fst)
{
if (fst.Semiring == null)
{
return(null);
}
ExtendFinal.Apply(fst);
Semiring semiring = fst.Semiring;
Fst res = new Fst(fst.GetNumStates());
res.Semiring = semiring;
res.Isyms = fst.Osyms;
res.Osyms = fst.Isyms;
State[] stateMap = new State[fst.GetNumStates()];
int numStates = fst.GetNumStates();
for (int i = 0; i < numStates; i++)
{
State _is = fst.GetState(i);
State s = new State(semiring.Zero);
res.AddState(s);
stateMap[_is.GetId()] = s;
if (_is.FinalWeight != semiring.Zero)
{
res.SetStart(s);
}
}
stateMap[fst.Start.GetId()].FinalWeight = semiring.One;
for (int i = 0; i < numStates; i++)
{
State olds = fst.GetState(i);
State news = stateMap[olds.GetId()];
int numArcs = olds.GetNumArcs();
for (int j = 0; j < numArcs; j++)
{
Arc olda = olds.GetArc(j);
State next = stateMap[olda.NextState.GetId()];
Arc newa = new Arc(olda.Ilabel, olda.Olabel,
semiring.Reverse(olda.Weight), news);
next.AddArc(newa);
}
}
ExtendFinal.Undo(fst);
return(res);
}
/**
* /// Removes epsilon transitions from an fst.
* ///
* /// It return a new epsilon-free fst and does not modify the original fst
* ///
* /// @param fst the fst to remove epsilon transitions from
* /// @return the epsilon-free fst
*/
public static Fst Get(Fst fst)
{
if (fst == null)
{
return(null);
}
if (fst.Semiring == null)
{
return(null);
}
var semiring = fst.Semiring;
var res = new Fst(semiring);
var cl = new Dictionary <State, float> [fst.GetNumStates()];
var oldToNewStateMap = new State[fst.GetNumStates()];
var newToOldStateMap = new State[fst.GetNumStates()];
var numStates = fst.GetNumStates();
for (var i = 0; i < numStates; i++)
{
var s = fst.GetState(i);
// Add non-epsilon arcs
var newState = new State(s.FinalWeight);
res.AddState(newState);
oldToNewStateMap[s.GetId()] = newState;
newToOldStateMap[newState.GetId()] = s;
if (newState.GetId() == fst.Start.GetId())
{
res.SetStart(newState);
}
}
for (var i = 0; i < numStates; i++)
{
var s = fst.GetState(i);
// Add non-epsilon arcs
var newState = oldToNewStateMap[s.GetId()];
var numArcs = s.GetNumArcs();
for (var j = 0; j < numArcs; j++)
{
var a = s.GetArc(j);
if ((a.Ilabel != 0) || (a.Olabel != 0))
{
newState.AddArc(new Arc(a.Ilabel, a.Olabel, a.Weight, oldToNewStateMap[a.NextState
.GetId()]));
}
}
// Compute e-Closure
if (cl[s.GetId()] == null)
{
CalcClosure(fst, s, cl, semiring);
}
}
// augment fst with arcs generated from epsilon moves.
numStates = res.GetNumStates();
for (var i = 0; i < numStates; i++)
{
var s = res.GetState(i);
var oldState = newToOldStateMap[s.GetId()];
if (cl[oldState.GetId()] != null)
{
foreach (var pathFinalState in cl[oldState.GetId()].Keys)
{
var s1 = pathFinalState;
if (s1.FinalWeight != semiring.Zero)
{
s.FinalWeight = semiring.Plus(s.FinalWeight,
semiring.Times(GetPathWeight(oldState, s1, cl),
s1.FinalWeight));
}
var numArcs = s1.GetNumArcs();
for (var j = 0; j < numArcs; j++)
{
var a = s1.GetArc(j);
if ((a.Ilabel != 0) || (a.Olabel != 0))
{
var newArc = new Arc(a.Ilabel, a.Olabel,
semiring.Times(a.Weight,
GetPathWeight(oldState, s1, cl)),
oldToNewStateMap[a.NextState.GetId()]);
s.AddArc(newArc);
}
}
}
}
}
res.Isyms = fst.Isyms;
res.Osyms = fst.Osyms;
Connect.Apply(res);
return(res);
}
/**
* /// Determinizes an fst. The result will be an equivalent fst that has the
* /// property that no state has two transitions with the same input label. For
* /// this algorithm, epsilon transitions are treated as regular symbols.
* ///
* /// @param fst the fst to determinize
* /// @return the determinized fst
*/
public static Fst Get(Fst fst)
{
if (fst.Semiring == null)
{
// semiring not provided
return(null);
}
// initialize the queue and new fst
Semiring semiring = fst.Semiring;
Fst res = new Fst(semiring);
res.Isyms = fst.Isyms;
res.Osyms = fst.Osyms;
// stores the queue (item in index 0 is next)
Queue <List <Pair <State, float> > > queue = new Queue <List <Pair <State, float> > >();
Dictionary <String, State> stateMapper = new Dictionary <String, State>();
State s = new State(semiring.Zero);
string stateString = "(" + fst.Start + "," + semiring.One + ")";
queue.Enqueue(new List <Pair <State, float> >());
queue.Peek().Add(new Pair <State, float>(fst.Start, semiring.One));
res.AddState(s);
stateMapper.Add(stateString, s);
res.SetStart(s);
while (queue.Count != 0)
{
List <Pair <State, float> > p = queue.Dequeue();
State pnew = GetStateLabel(p, stateMapper);
//queueRemoveAt(0);
List <int> labels = GetUniqueLabels(fst, p);
foreach (int label in labels)
{
float wnew = semiring.Zero;
// calc w'
foreach (Pair <State, float> ps in p)
{
State old = ps.GetLeft();
float u = ps.GetRight();
int numArcs = old.GetNumArcs();
for (int j = 0; j < numArcs; j++)
{
Arc arc = old.GetArc(j);
if (label == arc.Ilabel)
{
wnew = semiring.Plus(wnew,
semiring.Times(u, arc.Weight));
}
}
}
// calc new states
// keep residual weights to variable forQueue
List <Pair <State, float> > forQueue = new List <Pair <State, float> >();
foreach (Pair <State, float> ps in p)
{
State old = ps.GetLeft();
float u = ps.GetRight();
float wnewRevert = semiring.Divide(semiring.One, wnew);
int numArcs = old.GetNumArcs();
for (int j = 0; j < numArcs; j++)
{
Arc arc = old.GetArc(j);
if (label == arc.Ilabel)
{
State oldstate = arc.NextState;
Pair <State, float> pair = GetPair(forQueue,
oldstate, semiring.Zero);
pair.SetRight(semiring.Plus(
pair.GetRight(),
semiring.Times(wnewRevert,
semiring.Times(u, arc.Weight))));
}
}
}
// build new state's id and new elements for queue
string qnewid = "";
foreach (Pair <State, float> ps in forQueue)
{
State old = ps.GetLeft();
float unew = ps.GetRight();
if (!qnewid.Equals(""))
{
qnewid = qnewid + ",";
}
qnewid = qnewid + "(" + old + "," + unew + ")";
}
if (stateMapper.Get(qnewid) == null)
{
State qnew = new State(semiring.Zero);
res.AddState(qnew);
stateMapper.Add(qnewid, qnew);
// update new state's weight
float fw = qnew.FinalWeight;
foreach (Pair <State, float> ps in forQueue)
{
fw = semiring.Plus(fw, semiring.Times(ps.GetLeft().FinalWeight, ps.GetRight()));
}
qnew.FinalWeight = fw;
queue.Enqueue(forQueue);
}
pnew.AddArc(new Arc(label, label, wnew, stateMapper[qnewid]));
}
}
return(res);
}
/**
* /// Computes the composition of two Fsts. Assuming no epsilon transitions.
* ///
* /// Input Fsts are not modified.
* ///
* /// @param fst1 the first Fst
* /// @param fst2 the second Fst
* /// @param semiring the semiring to use in the operation
* /// @param sorted
* /// @return the composed Fst
*/
public static Fst compose(Fst fst1, Fst fst2, Semiring semiring, Boolean sorted)
{
if (!Arrays.AreEqual(fst1.Osyms, fst2.Isyms))
{
// symboltables do not match
return(null);
}
Fst res = new Fst(semiring);
Dictionary <Pair <State, State>, State> stateMap = new Dictionary <Pair <State, State>, State>();
Queue <Pair <State, State> > queue = new Queue <Pair <State, State> >();
State s1 = fst1.Start;
State s2 = fst2.Start;
if ((s1 == null) || (s2 == null))
{
Logger.LogInfo <Compose>("Cannot find initial state.");
return(null);
}
Pair <State, State> p = new Pair <State, State>(s1, s2);
State s = new State(semiring.Times(s1.FinalWeight,
s2.FinalWeight));
res.AddState(s);
res.SetStart(s);
if (stateMap.ContainsKey(p))
{
stateMap[p] = s;
}
else
{
stateMap.Add(p, s);
}
queue.Enqueue(p);
while (queue.Count != 0)
{
p = queue.Dequeue();
s1 = p.GetLeft();
s2 = p.GetRight();
s = stateMap[p];
int numArcs1 = s1.GetNumArcs();
int numArcs2 = s2.GetNumArcs();
for (int i = 0; i < numArcs1; i++)
{
Arc a1 = s1.GetArc(i);
for (int j = 0; j < numArcs2; j++)
{
Arc a2 = s2.GetArc(j);
if (sorted && a1.Olabel < a2.Ilabel)
{
break;
}
if (a1.Olabel == a2.Ilabel)
{
State nextState1 = a1.NextState;
State nextState2 = a2.NextState;
Pair <State, State> nextPair = new Pair <State, State>(
nextState1, nextState2);
State nextState = stateMap.Get(nextPair);
if (nextState == null)
{
nextState = new State(semiring.Times(
nextState1.FinalWeight,
nextState2.FinalWeight));
res.AddState(nextState);
if (stateMap.ContainsKey(nextPair))
{
stateMap[nextPair] = nextState;
}
else
{
stateMap.Add(nextPair, nextState);
}
queue.Enqueue(nextPair);
}
Arc a = new Arc(a1.Ilabel, a2.Olabel,
semiring.Times(a1.Weight, a2.Weight),
nextState);
s.AddArc(a);
}
}
}
}
res.Isyms = fst1.Isyms;
res.Osyms = fst2.Osyms;
return(res);
}
/**
* /// Calculates the n-best shortest path from the initial to the final state.
* ///
* /// @param fst
* /// the fst to calculate the nbest shortest paths
* /// @param n
* /// number of best paths to return
* /// @param determinize
* /// if true the input fst will bwe determinized prior the
* /// operation
* /// @return an fst containing the n-best shortest paths
*/
public static Fst Get(Fst fst, int n, Boolean determinize)
{
if (fst == null)
{
return(null);
}
if (fst.Semiring == null)
{
return(null);
}
var fstdet = fst;
if (determinize)
{
fstdet = Determinize.Get(fst);
}
var semiring = fstdet.Semiring;
var res = new Fst(semiring);
res.Isyms = fstdet.Isyms;
res.Osyms = fstdet.Osyms;
var d = ShortestDistance(fstdet);
ExtendFinal.Apply(fstdet);
var r = new int[fstdet.GetNumStates()];
var queue = new PriorityQueue <Pair <State, float> >(new CustomComparer(d, semiring));
var previous = new HashMap <Pair <State, float>, Pair <State, float> >(fst.GetNumStates());
var stateMap = new HashMap <Pair <State, float>, State>(fst.GetNumStates());
var start = fstdet.Start;
var item = new Pair <State, float>(start, semiring.One);
queue.Add(item);
previous.Put(item, null);
while (queue.Count != 0)
{
var pair = queue.Remove();
var p = pair.GetLeft();
var c = pair.GetRight();
var s = new State(p.FinalWeight);
res.AddState(s);
stateMap.Put(pair, s);
if (previous[pair] == null)
{
// this is the start state
res.SetStart(s);
}
else
{
// add the incoming arc from previous to current
var previouState = stateMap.Get(previous.Get(pair));
var previousOldState = previous.Get(pair).GetLeft();
for (var j = 0; j < previousOldState.GetNumArcs(); j++)
{
var a = previousOldState.GetArc(j);
if (a.NextState.Equals(p))
{
previouState.AddArc(new Arc(a.Ilabel, a.Olabel, a.Weight, s));
}
}
}
var stateIndex = p.GetId();
r[stateIndex]++;
if ((r[stateIndex] == n) && (p.FinalWeight != semiring.Zero))
{
break;
}
if (r[stateIndex] <= n)
{
for (var j = 0; j < p.GetNumArcs(); j++)
{
var a = p.GetArc(j);
var cnew = semiring.Times(c, a.Weight);
var next = new Pair <State, float>(a.NextState, cnew);
previous.Put(next, pair);
queue.Add(next);
}
}
}
return(res);
}
