public RootAllomorphTrie(Func<Annotation<ShapeNode>, bool> filter)
{
_fsa = new Fst<Shape, ShapeNode> {Filter = filter, UseUnification = true};
_fsa.StartState = _fsa.CreateState();
_filter = filter;
_allomorphs = new Dictionary<string, RootAllomorph>();
}
public void Compose()
{
var featSys = new FeatureSystem
{
new StringFeature("value")
};
var fst1 = new Fst<AnnotatedStringData, int>(_operations);
fst1.StartState = fst1.CreateState();
State<AnnotatedStringData, int> s1 = fst1.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("value").EqualTo("a").Value, FeatureStruct.New(featSys).Feature("value").EqualTo("x").Value, fst1.CreateAcceptingState());
s1.Arcs.Add(FeatureStruct.New(featSys).Feature("value").EqualTo("b").Value, FeatureStruct.New(featSys).Feature("value").EqualTo("y").Value, s1);
var fst2 = new Fst<AnnotatedStringData, int>(_operations);
fst2.StartState = fst2.CreateAcceptingState();
fst2.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("value").EqualTo("x").Value, null, fst2.StartState);
fst2.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("value").EqualTo("y").Value, FeatureStruct.New(featSys).Feature("value").EqualTo("z").Value, fst2.StartState);
Fst<AnnotatedStringData, int> composedFsa = fst1.Compose(fst2);
var writer = new StringWriter();
composedFsa.ToGraphViz(writer);
Assert.That(writer.ToString().Replace("\r\n", "\n"), Is.EqualTo(@"digraph G {
0 [shape=""diamond"", color=""green""];
0 -> 1 [label=""[value:\""a\""],1:ε""];
1 [shape=""circle"", color=""red"", peripheries=""2""];
1 -> 1 [label=""[value:\""b\""],1:([value:\""z\""],∪)""];
}
".Replace("\r\n", "\n")));
}
public static Fst Plus(this Fst fst)
{
var initial = new[] { NewState(fst.States) };
var transitions = fst.Transitions
.Concat(fst.Initial.Select(i => (initial[0], string.Empty, string.Empty, i)))
.Concat(fst.Final.Select(f => (f, string.Empty, string.Empty, initial[0])));
return(new Fst(
fst.States.Concat(initial),
initial,
fst.Final,
transitions));
}
public static void augment(int label, Fst fst, Semiring semiring)
{
string[] isyms = fst.getIsyms();
string[] osyms = fst.getOsyms();
int ilabel = isyms.Length;
int iLabel = isyms.Length + 1;
int oLabel = osyms.Length;
int olabel = osyms.Length + 1;
int numStates = fst.getNumStates();
for (int i = 0; i < numStates; i++)
{
State state = fst.getState(i);
int num = (!(fst is ImmutableFst)) ? state.getNumArcs() : (state.getNumArcs() - 1);
for (int j = 0; j < num; j++)
{
Arc arc = state.getArc(j);
if (label == 1 && arc.getOlabel() == 0)
{
arc.setOlabel(olabel);
}
else if (label == 0 && arc.getIlabel() == 0)
{
arc.setIlabel(ilabel);
}
}
if (label == 0)
{
if (fst is ImmutableFst)
{
state.setArc(num, new Arc(iLabel, 0, semiring.one(), state));
}
else
{
state.addArc(new Arc(iLabel, 0, semiring.one(), state));
}
}
else if (label == 1)
{
if (fst is ImmutableFst)
{
state.setArc(num, new Arc(0, oLabel, semiring.one(), state));
}
else
{
state.addArc(new Arc(0, oLabel, semiring.one(), state));
}
}
}
}
internal override State <TData, TOffset> GenerateNfa(Fst <TData, TOffset> fsa, State <TData, TOffset> startState, out bool hasVariables)
{
hasVariables = false;
ArcPriorityType priorityType = IsGreedy ? ArcPriorityType.High : ArcPriorityType.Low;
State <TData, TOffset> endState;
State <TData, TOffset> currentState = startState;
var startStates = new List <State <TData, TOffset> >();
if (MinOccur == 0)
{
endState = startState.Arcs.Add(fsa.CreateState(), priorityType);
endState = base.GenerateNfa(fsa, endState, out hasVariables);
startStates.Add(currentState);
}
else
{
endState = startState;
for (int i = 0; i < MinOccur; i++)
{
currentState = endState;
endState = base.GenerateNfa(fsa, currentState, out hasVariables);
}
}
if (MaxOccur == Infinite)
{
endState.Arcs.Add(currentState, priorityType);
}
else
{
int numCopies = MaxOccur - MinOccur;
if (MinOccur == 0)
{
numCopies--;
}
for (int i = 1; i <= numCopies; i++)
{
startStates.Add(endState);
endState = endState.Arcs.Add(fsa.CreateState(), priorityType);
endState = base.GenerateNfa(fsa, endState, out hasVariables);
}
}
foreach (State <TData, TOffset> state in startStates)
{
state.Arcs.Add(endState);
}
return(endState);
}
public static void main(params string[] args)
{
JAXBContext jaxbcontext = JAXBContext.newInstance(new Class[]
{
ClassLiteral <SequiturImport.FSA> .Value
});
Unmarshaller unmarshaller = jaxbcontext.createUnmarshaller();
Unmarshaller unmarshaller2 = unmarshaller;
SequiturImport.FSA fsa = (SequiturImport.FSA)unmarshaller2.unmarshal(new File(args[0]));
Fst fst = fsa.toFst();
fst.saveModel(args[1]);
[email protected](new StringBuilder().append("The Sequitur G2P XML-formatted FST ").append(args[0]).append(" has been converted to Sphinx' OpenFst binary format in the file ").append(args[1]).toString());
}
/**
* /// Calculates the shortest distances from each state to the final
* ///
* /// @param fst
* /// the fst to calculate the shortest distances
* /// @return the array containing the shortest distances
*/
public static float[] ShortestDistance(Fst fst)
{
var reversed = Reverse.Get(fst);
var d = new float[reversed.GetNumStates()];
var r = new float[reversed.GetNumStates()];
var semiring = reversed.Semiring;
Arrays.Fill(d, semiring.Zero);
Arrays.Fill(r, semiring.Zero);
var queue = new List <State>();//TODO: Find LinkedHashSet Implementation
queue.Add(reversed.Start);
d[reversed.Start.GetId()] = semiring.One;
r[reversed.Start.GetId()] = semiring.One;
while (!queue.IsEmpty())
{
var q = queue.First();
queue.Remove(q);
var rnew = r[q.GetId()];
r[q.GetId()] = semiring.Zero;
for (var i = 0; i < q.GetNumArcs(); i++)
{
var a = q.GetArc(i);
var nextState = a.NextState;
var dnext = d[a.NextState.GetId()];
var dnextnew = semiring.Plus(dnext,
semiring.Times(rnew, a.Weight));
if (dnext != dnextnew)
{
d[a.NextState.GetId()] = dnextnew;
r[a.NextState.GetId()] = semiring.Plus(r[a.NextState.GetId()], semiring.Times(rnew,
a.Weight));
if (!queue.Contains(nextState))
{
queue.Add(nextState);
}
}
}
}
return(d);
}
public static Fst Concat(this Fst first, Fst second)
{
second = second.Remap(first.States);
var transitions = first.Transitions
.Concat(second.Transitions)
.Concat(first.Final
.SelectMany(f1 =>
second.Initial.Select(i2 => (f1, string.Empty, string.Empty, i2))));
return(new Fst(
first.States.Concat(second.States),
first.Initial,
second.Final,
transitions));
}
internal override State <TData, TOffset> GenerateNfa(Fst <TData, TOffset> fsa, State <TData, TOffset> startState, out bool hasVariables)
{
if (IsLeaf)
{
hasVariables = false;
return(startState);
}
if (_name != null)
{
startState = fsa.CreateTag(startState, fsa.CreateState(), _name, true);
}
startState = base.GenerateNfa(fsa, startState, out hasVariables);
startState = _name != null?fsa.CreateTag(startState, fsa.CreateState(), _name, false) : startState.Arcs.Add(fsa.CreateState());
return(startState);
}
private void Compile(Pattern <TData, TOffset> pattern)
{
_fsa = new Fst <TData, TOffset>(EqualityComparer <TOffset> .Default)
{
Direction = _settings.Direction,
Filter = _settings.Filter,
UseUnification = _settings.MatchingMethod == MatchingMethod.Unification
};
_fsa.StartState = _fsa.CreateState();
int nextPriority = 0;
bool hasVariables = GeneratePatternNfa(_fsa.StartState, pattern, null,
new Func <Match <TData, TOffset>, bool> [0], ref nextPriority);
#if FST_GRAPHS
using (var writer = new System.IO.StreamWriter(string.Format("{0}-nfa.dot",
_settings.Direction == Direction.LeftToRight ? "ltor" : "rtol")))
_fsa.ToGraphViz(writer);
#endif
if (!_settings.Nondeterministic && !hasVariables && !_settings.AllSubmatches)
{
_fsa = _fsa.Determinize();
#if FST_GRAPHS
using (var writer = new System.IO.StreamWriter(string.Format("{0}-dfa.dot",
_settings.Direction == Direction.LeftToRight ? "ltor" : "rtol")))
_fsa.ToGraphViz(writer);
#endif
_fsa.Minimize();
#if FST_GRAPHS
using (var writer = new System.IO.StreamWriter(string.Format("{0}-mindfa.dot",
_settings.Direction == Direction.LeftToRight ? "ltor" : "rtol")))
_fsa.ToGraphViz(writer);
#endif
}
else
{
_fsa = _fsa.EpsilonRemoval();
#if FST_GRAPHS
using (var writer = new System.IO.StreamWriter(string.Format("{0}-ernfa.dot",
_settings.Direction == Direction.LeftToRight ? "ltor" : "rtol")))
_fsa.ToGraphViz(writer);
#endif
}
_fsa.IgnoreVariables = !hasVariables;
_fsa.Freeze();
}
public void ComposeEpsilon_Test()
{
URL url = new URL(Helper.FilesDirectory + "/fst/algorithms/composeeps/A.fst.txt");
String path = url.File.DirectoryName + "/A";
Fst fstA = Convert.ImportFst(path, new TropicalSemiring());
path = url.File.DirectoryName + "/B";
Fst fstB = Convert.ImportFst(path, new TropicalSemiring());
path = Path.Combine(url.File.DirectoryName, "fstcomposeeps");
Fst fstC = Convert.ImportFst(path, new TropicalSemiring());
Fst fstComposed = Compose.Get(fstA, fstB, new TropicalSemiring());
Assert.AreEqual(fstC, fstComposed);
}
public void Project_Test()
{
Console.WriteLine("Testing Project...");
// Project on Input label
Fst fst = CreateFst();
Fst p = CreatePi();
Project.Apply(fst, ProjectType.Input);
Assert.IsTrue(fst.Equals(p));
// Project on Output label
fst = CreateFst();
p = CreatePo();
Project.Apply(fst, ProjectType.Output);
Assert.IsTrue(fst.Equals(p));
Console.WriteLine("Testing Project Completed!\n");
}
public static Fst get(Fst fst1, Fst fst2, Semiring semiring)
{
if (fst1 == null || fst2 == null)
{
return(null);
}
if (!Arrays.equals(fst1.getOsyms(), fst2.getIsyms()))
{
return(null);
}
Fst filter = Compose.getFilter(fst1.getOsyms(), semiring);
Compose.augment(1, fst1, semiring);
Compose.augment(0, fst2, semiring);
Fst fst3 = Compose.compose(fst1, filter, semiring, false);
return(Compose.compose(fst3, fst2, semiring, false));
}
public static Fst get(Fst fst)
{
if (fst.getSemiring() == null)
{
return(null);
}
ExtendFinal.apply(fst);
Semiring semiring = fst.getSemiring();
Fst fst2 = new Fst(fst.getNumStates());
fst2.setSemiring(semiring);
fst2.setIsyms(fst.getOsyms());
fst2.setOsyms(fst.getIsyms());
State[] array = new State[fst.getNumStates()];
int numStates = fst.getNumStates();
for (int i = 0; i < numStates; i++)
{
State state = fst.getState(i);
State state2 = new State(semiring.zero());
fst2.addState(state2);
array[state.getId()] = state2;
if (state.getFinalWeight() != semiring.zero())
{
fst2.setStart(state2);
}
}
array[fst.getStart().getId()].setFinalWeight(semiring.one());
for (int i = 0; i < numStates; i++)
{
State state = fst.getState(i);
State state2 = array[state.getId()];
int numArcs = state.getNumArcs();
for (int j = 0; j < numArcs; j++)
{
Arc arc = state.getArc(j);
State state3 = array[arc.getNextState().getId()];
Arc arc2 = new Arc(arc.getIlabel(), arc.getOlabel(), semiring.reverse(arc.getWeight()), state2);
state3.addArc(arc2);
}
}
ExtendFinal.undo(fst);
return(fst2);
}
protected TraversalMethodBase(Fst <TData, TOffset> fst, TData data, VariableBindings varBindings, bool startAnchor, bool endAnchor, bool useDefaults)
{
_fst = fst;
_data = data;
_varBindings = varBindings;
_startAnchor = startAnchor;
_endAnchor = endAnchor;
_useDefaults = useDefaults;
_annotations = new List <Annotation <TOffset> >();
// insertion sort
foreach (Annotation <TOffset> topAnn in _data.Annotations.GetNodes(_fst.Direction))
{
foreach (Annotation <TOffset> ann in topAnn.GetNodesDepthFirst(_fst.Direction))
{
if (!_fst.Filter(ann))
{
continue;
}
int i = _annotations.Count - 1;
while (i >= 0 && CompareAnnotations(_annotations[i], ann) > 0)
{
if (i + 1 == _annotations.Count)
{
_annotations.Add(_annotations[i]);
}
else
{
_annotations[i + 1] = _annotations[i];
}
i--;
}
if (i + 1 == _annotations.Count)
{
_annotations.Add(ann);
}
else
{
_annotations[i + 1] = ann;
}
}
}
_cachedInstances = new Queue <TInst>();
}
private static List <int> GetUniqueLabels(Fst fst, List <Pair <State, float> > pa)
{
List <int> res = new List <int>();
foreach (Pair <State, float> p in pa)
{
State s = p.GetLeft();
int numArcs = s.GetNumArcs();
for (int j = 0; j < numArcs; j++)
{
Arc arc = s.GetArc(j);
if (!res.Contains(arc.Ilabel))
{
res.Add(arc.Ilabel);
}
}
}
return(res);
}
internal virtual State <TData, TOffset> GenerateNfa(Fst <TData, TOffset> fsa, State <TData, TOffset> startState, out bool hasVariables)
{
hasVariables = false;
if (IsLeaf)
{
return(startState);
}
foreach (PatternNode <TData, TOffset> child in Children.GetNodes(fsa.Direction))
{
bool childHasVariables;
startState = child.GenerateNfa(fsa, startState, out childHasVariables);
if (childHasVariables)
{
hasVariables = true;
}
}
return(startState);
}
/**
* /// The depth first search recursion
*/
private static State DepthFirstSearchNext(Fst fst, State start,
List <List <State> > paths, List <Arc>[] exploredArcs, HashSet <State> accessible)
{
int lastPathIndex = paths.Count - 1;
List <Arc> currentExploredArcs = exploredArcs[start.GetId()];
paths[lastPathIndex].Add(start);
if (start.GetNumArcs() != 0)
{
int arcCount = 0;
int numArcs = start.GetNumArcs();
for (int j = 0; j < numArcs; j++)
{
Arc arc = start.GetArc(j);
if ((currentExploredArcs == null) ||
!currentExploredArcs.Contains(arc))
{
lastPathIndex = paths.Count - 1;
if (arcCount++ > 0)
{
DuplicatePath(lastPathIndex, fst.Start, start,
paths);
lastPathIndex = paths.Count - 1;
paths[lastPathIndex].Add(start);
}
State next = arc.NextState;
AddExploredArc(start.GetId(), arc, exploredArcs);
// detect self loops
if (next.GetId() != start.GetId())
{
DepthFirstSearchNext(fst, next, paths, exploredArcs, accessible);
}
}
}
}
lastPathIndex = paths.Count - 1;
accessible.Add(start);
return(start);
}
// Removes the epsilon transitions by preserving the transducer's language
public static Fst EpsilonFree(this Fst fst)
{
var epsilonClosureOf = fst.Transitions
.Where(t => (string.IsNullOrEmpty($"{t.In}{t.Out}")))
.Select(t => (t.From, t.To))
.ToHashSet()
.TransitiveClosure()
.Union(fst.States.Select(s => (From: s, To: s)))
.GroupBy(p => p.Item1, p => p.Item2)
.ToDictionary(g => g.Key, g => g.ToHashSet());
var transitions = fst.Transitions
.Where(tr => !(string.IsNullOrEmpty(tr.In) && string.IsNullOrEmpty(tr.Out)))
.SelectMany(tr => epsilonClosureOf[tr.To].Select(to => (tr.From, tr.In, tr.Out, to)));
return(new Fst(
fst.States,
fst.Initial.SelectMany(s => epsilonClosureOf[s]),
fst.Final,
transitions));
}
private static ArrayList getUniqueLabels(Fst fst, ArrayList arrayList)
{
ArrayList arrayList2 = new ArrayList();
Iterator iterator = arrayList.iterator();
while (iterator.hasNext())
{
Pair pair = (Pair)iterator.next();
State state = (State)pair.getLeft();
int numArcs = state.getNumArcs();
for (int i = 0; i < numArcs; i++)
{
Arc arc = state.getArc(i);
if (!arrayList2.contains(Integer.valueOf(arc.getIlabel())))
{
arrayList2.add(Integer.valueOf(arc.getIlabel()));
}
}
}
return(arrayList2);
}
public static float[] shortestDistance(Fst fst)
{
Fst fst2 = Reverse.get(fst);
float[] array = new float[fst2.getNumStates()];
float[] array2 = new float[fst2.getNumStates()];
Semiring semiring = fst2.getSemiring();
Arrays.fill(array, semiring.zero());
Arrays.fill(array2, semiring.zero());
LinkedHashSet linkedHashSet = new LinkedHashSet();
linkedHashSet.add(fst2.getStart());
array[fst2.getStart().getId()] = semiring.one();
array2[fst2.getStart().getId()] = semiring.one();
while (!linkedHashSet.isEmpty())
{
State state = (State)linkedHashSet.iterator().next();
linkedHashSet.remove(state);
float f = array2[state.getId()];
array2[state.getId()] = semiring.zero();
for (int i = 0; i < state.getNumArcs(); i++)
{
Arc arc = state.getArc(i);
State nextState = arc.getNextState();
float num = array[arc.getNextState().getId()];
float num2 = semiring.plus(num, semiring.times(f, arc.getWeight()));
if (num != num2)
{
array[arc.getNextState().getId()] = num2;
array2[arc.getNextState().getId()] = semiring.plus(array2[arc.getNextState().getId()], semiring.times(f, arc.getWeight()));
if (!linkedHashSet.contains(nextState))
{
linkedHashSet.add(nextState);
}
}
}
}
return(array);
}
public static Fst getFilter(string[] syms, Semiring semiring)
{
Fst fst = new Fst(semiring);
int num = syms.Length;
int num2 = syms.Length + 1;
fst.setIsyms(syms);
fst.setOsyms(syms);
State state = new State(syms.Length + 3);
state.setFinalWeight(semiring.one());
State state2 = new State(syms.Length);
state2.setFinalWeight(semiring.one());
State state3 = new State(syms.Length);
state3.setFinalWeight(semiring.one());
fst.addState(state);
state.addArc(new Arc(num2, num, semiring.one(), state));
state.addArc(new Arc(num, num, semiring.one(), state2));
state.addArc(new Arc(num2, num2, semiring.one(), state3));
for (int i = 1; i < syms.Length; i++)
{
state.addArc(new Arc(i, i, semiring.one(), state));
}
fst.setStart(state);
fst.addState(state2);
state2.addArc(new Arc(num, num, semiring.one(), state2));
for (int i = 1; i < syms.Length; i++)
{
state2.addArc(new Arc(i, i, semiring.one(), state));
}
fst.addState(state3);
state3.addArc(new Arc(num2, num2, semiring.one(), state3));
for (int i = 1; i < syms.Length; i++)
{
state3.addArc(new Arc(i, i, semiring.one(), state));
}
return(fst);
}
internal override State <TData, TOffset> GenerateNfa(Fst <TData, TOffset> fsa, State <TData, TOffset> startState, out bool hasVariables)
{
hasVariables = false;
if (IsLeaf)
{
return(startState);
}
State <TData, TOffset> endState = fsa.CreateState();
foreach (PatternNode <TData, TOffset> node in Children)
{
bool childHasVariables;
State <TData, TOffset> nodeEndState = node.GenerateNfa(fsa, startState, out childHasVariables);
if (childHasVariables)
{
hasVariables = true;
}
nodeEndState.Arcs.Add(endState);
}
return(endState);
}
private static State depthFirstSearchNext(Fst fst, State state, ArrayList arrayList, ArrayList[] array, HashSet hashSet)
{
int num = arrayList.size() - 1;
ArrayList arrayList2 = array[state.getId()];
((ArrayList)arrayList.get(num)).add(state);
if (state.getNumArcs() != 0)
{
int num2 = 0;
int numArcs = state.getNumArcs();
for (int i = 0; i < numArcs; i++)
{
Arc arc = state.getArc(i);
if (arrayList2 == null || !arrayList2.contains(arc))
{
num = arrayList.size() - 1;
int num3 = num2;
num2++;
if (num3 > 0)
{
Connect.duplicatePath(num, fst.getStart(), state, arrayList);
num = arrayList.size() - 1;
((ArrayList)arrayList.get(num)).add(state);
}
State nextState = arc.getNextState();
Connect.addExploredArc(state.getId(), arc, array);
if (nextState.getId() != state.getId())
{
Connect.depthFirstSearchNext(fst, nextState, arrayList, array, hashSet);
}
}
}
}
int num4 = arrayList.size() - 1;
hashSet.add(state);
return(state);
}
private static void depthFirstSearch(Fst fst, HashSet hashSet, ArrayList arrayList, ArrayList[] array, HashSet hashSet2)
{
State start = fst.getStart();
State state = start;
do
{
if (!hashSet.contains(start))
{
state = Connect.depthFirstSearchNext(fst, start, arrayList, array, hashSet);
}
}while (start.getId() != state.getId());
int numStates = fst.getNumStates();
for (int i = 0; i < numStates; i++)
{
State state2 = fst.getState(i);
if (state2.getFinalWeight() != fst.getSemiring().zero())
{
Connect.calcCoAccessible(fst, state2, arrayList, hashSet2);
}
}
}
public virtual ArrayList phoneticize(ArrayList entry, int nbest)
{
Fst fst = this.entryToFSA(entry);
Semiring semiring = fst.getSemiring();
Compose.augment(1, fst, semiring);
ArcSort.apply(fst, new OLabelCompare());
Fst fst2 = Compose.compose(fst, this.epsilonFilter, semiring, true);
ArcSort.apply(fst2, new OLabelCompare());
fst2 = Compose.compose(fst2, this.g2pmodel, semiring, true);
Project.apply(fst2, ProjectType.__OUTPUT);
if (nbest == 1)
{
fst2 = NShortestPaths.get(fst2, 1, false);
}
else
{
fst2 = NShortestPaths.get(fst2, nbest * 10, false);
}
fst2 = RmEpsilon.get(fst2);
return(this.findAllPaths(fst2, nbest, this.skipSeqs, this.tie));
}
private static void DoFormat
(
object arg
)
{
try
{
RecordTask taskInfo = (RecordTask)arg;
using (IrbisConnection client = new IrbisConnection())
{
client.ParseConnectionString(ConnectionString);
client.Connect();
MarcRecord targetRecord = Fst.Execute
(
client,
taskInfo.Record
);
if (targetRecord != null)
{
targetRecord.SetSubField(463, 'w', taskInfo.CurrentIssue);
targetRecord.SetSubField(463, 'c', taskInfo.Magazine.Title);
lock (SyncRoot)
{
RecordCount++;
//Buffer.Append(targetRecord);
client.WriteRecord(targetRecord, false, true);
}
}
}
}
catch (Exception ex)
{
WriteLogLine("EXCEPTION: {0}", ex);
}
}
public void Determinize()
{
var featSys = new FeatureSystem
{
new StringFeature("A"),
new StringFeature("B"),
new StringFeature("C"),
new StringFeature("D"),
new StringFeature("E"),
new StringFeature("F")
};
var nfst = new Fst <StringData, int>(_operations);
nfst.StartState = nfst.CreateState();
State <StringData, int> s1 = nfst.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("C").EqualTo("true").Value, nfst.CreateState());
State <StringData, int> sa = s1.Arcs.Add(FeatureStruct.New(featSys).Feature("D").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("E").EqualTo("true").Value, nfst.CreateAcceptingState());
State <StringData, int> s2 = nfst.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, nfst.CreateState());
State <StringData, int> s3 = s2.Arcs.Add(FeatureStruct.New(featSys).Value, FeatureStruct.New(featSys).Value, nfst.CreateState());
s3.Arcs.Add(FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, sa);
var writer = new System.IO.StreamWriter(string.Format("c:\\ltor-nfst.dot"));
nfst.ToGraphViz(writer);
writer.Close();
Fst <StringData, int> dfst;
Assert.That(nfst.TryDeterminize(out dfst), Is.True);
writer = new System.IO.StreamWriter(string.Format("c:\\ltor-dfst.dot"));
dfst.ToGraphViz(writer);
writer.Close();
}
/**
* /// Initialization of a depth first search recursion
*/
private static void DepthFirstSearch(Fst fst, HashSet <State> accessible, List <List <State> > paths, List <Arc>[] exploredArcs, HashSet <State> coaccessible)
{
State currentState = fst.Start;
State nextState = currentState;
do
{
if (!accessible.Contains(currentState))
{
nextState = DepthFirstSearchNext(fst, currentState, paths, exploredArcs,
accessible);
}
} while (currentState.GetId() != nextState.GetId());
int numStates = fst.GetNumStates();
for (int i = 0; i < numStates; i++)
{
State s = fst.GetState(i);
if (s.FinalWeight != fst.Semiring.Zero)
{
CalcCoAccessible(fst, s, paths, coaccessible);
}
}
}
public void Determinize()
{
var featSys = new FeatureSystem
{
new StringFeature("A"),
new StringFeature("B"),
new StringFeature("C"),
new StringFeature("D"),
new StringFeature("E"),
new StringFeature("F")
};
var nfst = new Fst<AnnotatedStringData, int>(_operations);
nfst.StartState = nfst.CreateState();
State<AnnotatedStringData, int> s1 = nfst.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("C").EqualTo("true").Value, nfst.CreateState());
State<AnnotatedStringData, int> sa = s1.Arcs.Add(FeatureStruct.New(featSys).Feature("D").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("E").EqualTo("true").Value, nfst.CreateAcceptingState());
State<AnnotatedStringData, int> s2 = nfst.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, nfst.CreateState());
State<AnnotatedStringData, int> s3 = s2.Arcs.Add(FeatureStruct.New(featSys).Value, FeatureStruct.New(featSys).Value, nfst.CreateState());
s3.Arcs.Add(FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, sa);
Fst<AnnotatedStringData, int> dfst;
Assert.That(nfst.TryDeterminize(out dfst), Is.True);
}
/**
* /// Removes epsilon transitions from an fst.
* ///
* /// It return a new epsilon-free fst and does not modify the original fst
* ///
* /// @param fst the fst to remove epsilon transitions from
* /// @return the epsilon-free fst
*/
public static Fst Get(Fst fst)
{
if (fst == null)
{
return(null);
}
if (fst.Semiring == null)
{
return(null);
}
var semiring = fst.Semiring;
var res = new Fst(semiring);
var cl = new Dictionary <State, float> [fst.GetNumStates()];
var oldToNewStateMap = new State[fst.GetNumStates()];
var newToOldStateMap = new State[fst.GetNumStates()];
var numStates = fst.GetNumStates();
for (var i = 0; i < numStates; i++)
{
var s = fst.GetState(i);
// Add non-epsilon arcs
var newState = new State(s.FinalWeight);
res.AddState(newState);
oldToNewStateMap[s.GetId()] = newState;
newToOldStateMap[newState.GetId()] = s;
if (newState.GetId() == fst.Start.GetId())
{
res.SetStart(newState);
}
}
for (var i = 0; i < numStates; i++)
{
var s = fst.GetState(i);
// Add non-epsilon arcs
var newState = oldToNewStateMap[s.GetId()];
var numArcs = s.GetNumArcs();
for (var j = 0; j < numArcs; j++)
{
var a = s.GetArc(j);
if ((a.Ilabel != 0) || (a.Olabel != 0))
{
newState.AddArc(new Arc(a.Ilabel, a.Olabel, a.Weight, oldToNewStateMap[a.NextState
.GetId()]));
}
}
// Compute e-Closure
if (cl[s.GetId()] == null)
{
CalcClosure(fst, s, cl, semiring);
}
}
// augment fst with arcs generated from epsilon moves.
numStates = res.GetNumStates();
for (var i = 0; i < numStates; i++)
{
var s = res.GetState(i);
var oldState = newToOldStateMap[s.GetId()];
if (cl[oldState.GetId()] != null)
{
foreach (var pathFinalState in cl[oldState.GetId()].Keys)
{
var s1 = pathFinalState;
if (s1.FinalWeight != semiring.Zero)
{
s.FinalWeight = semiring.Plus(s.FinalWeight,
semiring.Times(GetPathWeight(oldState, s1, cl),
s1.FinalWeight));
}
var numArcs = s1.GetNumArcs();
for (var j = 0; j < numArcs; j++)
{
var a = s1.GetArc(j);
if ((a.Ilabel != 0) || (a.Olabel != 0))
{
var newArc = new Arc(a.Ilabel, a.Olabel,
semiring.Times(a.Weight,
GetPathWeight(oldState, s1, cl)),
oldToNewStateMap[a.NextState.GetId()]);
s.AddArc(newArc);
}
}
}
}
}
res.Isyms = fst.Isyms;
res.Osyms = fst.Osyms;
Connect.Apply(res);
return(res);
}
public NondeterministicFsaTraversalMethod(Fst <TData, TOffset> fst, TData data, VariableBindings varBindings, bool startAnchor, bool endAnchor, bool useDefaults)
: base(fst, data, varBindings, startAnchor, endAnchor, useDefaults)
{
}
public static Fst compose(Fst fst1, Fst fst2, Semiring semiring, bool sorted)
{
if (!Arrays.equals(fst1.getOsyms(), fst2.getIsyms()))
{
return(null);
}
Fst fst3 = new Fst(semiring);
HashMap hashMap = new HashMap();
LinkedList linkedList = new LinkedList();
State state = fst1.getStart();
State state2 = fst2.getStart();
if (state == null || state2 == null)
{
java.lang.System.err.println("Cannot find initial state.");
return(null);
}
Pair pair = new Pair(state, state2);
State state3 = new State(semiring.times(state.getFinalWeight(), state2.getFinalWeight()));
fst3.addState(state3);
fst3.setStart(state3);
hashMap.put(pair, state3);
linkedList.add(pair);
while (!linkedList.isEmpty())
{
pair = (Pair)linkedList.remove();
state = (State)pair.getLeft();
state2 = (State)pair.getRight();
state3 = (State)hashMap.get(pair);
int numArcs = state.getNumArcs();
int numArcs2 = state2.getNumArcs();
for (int i = 0; i < numArcs; i++)
{
Arc arc = state.getArc(i);
for (int j = 0; j < numArcs2; j++)
{
Arc arc2 = state2.getArc(j);
if (sorted && arc.getOlabel() < arc2.getIlabel())
{
break;
}
if (arc.getOlabel() == arc2.getIlabel())
{
State nextState = arc.getNextState();
State nextState2 = arc2.getNextState();
Pair pair2 = new Pair(nextState, nextState2);
State state4 = (State)hashMap.get(pair2);
if (state4 == null)
{
state4 = new State(semiring.times(nextState.getFinalWeight(), nextState2.getFinalWeight()));
fst3.addState(state4);
hashMap.put(pair2, state4);
linkedList.add(pair2);
}
Arc arc3 = new Arc(arc.getIlabel(), arc2.getOlabel(), semiring.times(arc.getWeight(), arc2.getWeight()), state4);
state3.addArc(arc3);
}
}
}
}
fst3.setIsyms(fst1.getIsyms());
fst3.setOsyms(fst2.getOsyms());
return(fst3);
}
public void IsDeterminizable()
{
var featSys = new FeatureSystem
{
new StringFeature("A"),
new StringFeature("B"),
new StringFeature("C"),
new StringFeature("D"),
new StringFeature("E"),
new StringFeature("F")
};
var fst = new Fst<AnnotatedStringData, int>(_operations);
fst.StartState = fst.CreateState();
State<AnnotatedStringData, int> s1 = fst.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("E").EqualTo("true").Value, fst.CreateState());
State<AnnotatedStringData, int> s2 = fst.StartState.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("F").EqualTo("true").Value, fst.CreateState());
State<AnnotatedStringData, int> s3 = s1.Arcs.Add(FeatureStruct.New(featSys).Feature("B").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("D").EqualTo("true").Value, fst.CreateState());
s2.Arcs.Add(FeatureStruct.New(featSys).Feature("C").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("D").EqualTo("true").Value, s3);
State<AnnotatedStringData, int> s4 = s3.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, fst.CreateAcceptingState());
Assert.That(fst.IsDeterminizable, Is.True);
s4.Arcs.Add(FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, FeatureStruct.New(featSys).Feature("A").EqualTo("true").Value, s4);
Assert.That(fst.IsDeterminizable, Is.False);
}
public void Transduce()
{
var fst = new Fst<AnnotatedStringData, int>(_operations) {UseUnification = false};
fst.StartState = fst.CreateAcceptingState();
fst.StartState.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("nas-", "nas?").Value, fst.StartState);
fst.StartState.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("nas+").Symbol("cor+", "cor-").Value, fst.StartState);
State<AnnotatedStringData, int> s1 = fst.StartState.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cor?").Symbol("nas+").Value, FeatureStruct.New(PhoneticFeatSys).Symbol("cor-").Value, fst.CreateState());
s1.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cor-").Value, fst.StartState);
State<AnnotatedStringData, int> s2 = fst.StartState.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cor?").Symbol("nas+").Value, FeatureStruct.New(PhoneticFeatSys).Symbol("cor+").Value, fst.CreateAcceptingState());
s2.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cor?").Symbol("nas+").Value, FeatureStruct.New(PhoneticFeatSys).Symbol("cor+").Value, s2);
s2.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("nas-", "nas?").Symbol("cor+", "cor?").Value, fst.StartState);
s2.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("nas+").Symbol("cor+").Value, fst.StartState);
s2.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cor?").Symbol("nas+").Value, FeatureStruct.New(PhoneticFeatSys).Symbol("cor-").Value, s1);
Fst<AnnotatedStringData, int> dfst = fst.Determinize();
AnnotatedStringData data = CreateStringData("caNp");
FstResult<AnnotatedStringData, int> result;
Assert.That(dfst.Transduce(data, data.Annotations.First, null, true, true, true, out result), Is.True);
Assert.That(result.Output.String, Is.EqualTo("camp"));
data = CreateStringData("caN");
Assert.That(dfst.Transduce(data, data.Annotations.First, null, true, true, true, out result), Is.True);
Assert.That(result.Output.String, Is.EqualTo("can"));
data = CreateStringData("carp");
Assert.That(dfst.Transduce(data, data.Annotations.First, null, true, true, true, out result), Is.True);
Assert.That(result.Output.String, Is.EqualTo("carp"));
fst = new Fst<AnnotatedStringData, int>(_operations) {UseUnification = false};
fst.StartState = fst.CreateAcceptingState();
s1 = fst.StartState.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cons+").Value, fst.CreateState())
.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cons-").Value, fst.CreateState());
s2 = s1.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("nas+").Value, null, fst.CreateState());
State<AnnotatedStringData, int> s3 = s1.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("voice-").Value, fst.CreateState());
s3.Arcs.Add(null, FeatureStruct.New(PhoneticFeatSys).Symbol(Bdry).Feature("strRep").EqualTo(".").Value, s2);
s3.Arcs.Add(null, FeatureStruct.New(PhoneticFeatSys).Symbol(Bdry).Feature("strRep").EqualTo("+").Value, fst.CreateState())
.Arcs.Add(null, FeatureStruct.New(PhoneticFeatSys).Symbol(Bdry).Feature("strRep").EqualTo(".").Value, s2);
s2.Arcs.Add(FeatureStruct.New(PhoneticFeatSys).Symbol("cons+").Value, fst.CreateAcceptingState());
dfst = fst.Determinize();
data = CreateStringData("camp");
Assert.That(dfst.Transduce(data, data.Annotations.First, null, true, true, true, out result), Is.True);
Assert.That(result.Output.String, Is.EqualTo("cap"));
data = CreateStringData("casp");
IEnumerable<FstResult<AnnotatedStringData, int>> results;
Assert.That(dfst.Transduce(data, data.Annotations.First, null, true, true, true, out results), Is.True);
FstResult<AnnotatedStringData, int>[] resultsArray = results.ToArray();
Assert.That(resultsArray.Length, Is.EqualTo(2));
Assert.That(resultsArray.Select(r => r.Output.String), Is.EquivalentTo(new [] {"cas+.p", "cas.p"}));
}
