private static bool Accepts(this INfa <char> nfa, string input, IState state, Dictionary <Tuple <IState, string>, bool> cache)
{
if (input == string.Empty && nfa.IsAccepting(state))
{
return(true);
}
var key = Tuple.Create(state, input);
if (!cache.ContainsKey(key))
{
cache[key] = false; // temporarily mark the (state, string) pair as non-accepting to avoid cycles
// check epsilon transitions
var result = nfa.EpsilonTransitions(state).Any(nextState => nfa.Accepts(input, nextState, cache));
// check transitions using the first letter of input
if (input.Length > 0)
{
var firstLetter = input[0];
var remainingInput = input.Substring(1);
var transitions = nfa.Transitions(state).GetValueOrDefault(firstLetter);
if (transitions != null)
{
result = result || transitions.Any(nextState => nfa.Accepts(remainingInput, nextState, cache));
}
}
cache[key] = result;
}
return(cache.GetValueOrDefault(key));
}
private static void VerifyNfaIsNotNullAndHasValidStartAndEndStates(INfa nfa)
{
Assert.IsNotNull(nfa);
Assert.IsNotNull(nfa.Start);
Assert.IsNotNull(nfa.End);
Assert.AreEqual(0, nfa.End.Transitions.Count());
}
private static Util.HashableHashSet <IState> EpsilonClosure(INfa <Symbol> nfa, Util.HashableHashSet <IState> states)
{
Util.HashableHashSet <IState> closure = new Util.HashableHashSet <IState>();
Queue <IState> q = new Queue <IState>();
foreach (IState state in states)
{
closure.Add(state);
q.Enqueue(state);
}
while (q.Count != 0)
{
IState state = q.Dequeue();
foreach (IState neighbour in nfa.EpsilonTransitions(state))
{
if (!closure.Contains(neighbour))
{
closure.Add(neighbour);
q.Enqueue(neighbour);
}
}
}
return(closure);
}
private static INfa KleenePlus(INfa nfa)
{
var end = new NfaState();
nfa.End.AddTransistion(new NullNfaTransition(end));
nfa.End.AddTransistion(new NullNfaTransition(nfa.Start));
return(new Nfa(nfa.Start, end));
}
public static ConcreteNfa <Symbol> CreateFromNfa(INfa <Symbol> nfa)
{
var stateIds = new Dictionary <IState, int>();
var newNfa = new ConcreteNfa <Symbol>(new ValueState <int>(0));
newNfa.ConstructStates(stateIds, nfa, nfa.StartingState());
return(newNfa);
}
private static IDfa <bool, TLabel> RegexToDfa(Regex <TLabel> regex)
{
INfa <TLabel> nfa = RegexToNfaConverter <TLabel> .Convert(regex);
IDfa <bool, TLabel> dfa = NfaToDfaConverter <TLabel> .Convert(nfa);
return(DfaMinimizer <bool, TLabel> .Minimize(dfa));
}
public IDfaState Transform(INfa nfa)
{
var processOnceQueue = new ProcessOnceQueue<NfaClosure>();
var set = SharedPools.Default<HashSet<INfaState>>().AllocateAndClear();
foreach (var state in nfa.Start.Closure())
set.Add(state);
var start = new NfaClosure(set, nfa.Start.Equals(nfa.End));
processOnceQueue.Enqueue(start);
while (processOnceQueue.Count > 0)
{
var nfaClosure = processOnceQueue.Dequeue();
var transitions = SharedPools
.Default<Dictionary<ITerminal, HashSet<INfaState>>>()
.AllocateAndClear();
foreach (var state in nfaClosure.Closure)
{
for (var t = 0; t < state.Transitions.Count; t++)
{
var transition = state.Transitions[t];
switch (transition.TransitionType)
{
case NfaTransitionType.Terminal:
var terminalTransition = transition as TerminalNfaTransition;
var terminal = terminalTransition.Terminal;
if (!transitions.ContainsKey(terminalTransition.Terminal))
transitions[terminal] = SharedPools.Default<HashSet<INfaState>>().AllocateAndClear();
transitions[terminal].Add(transition.Target);
break;
}
}
}
foreach (var terminal in transitions.Keys)
{
var targetStates = transitions[terminal];
var closure = Closure(targetStates, nfa.End);
closure = processOnceQueue.EnqueueOrGetExisting(closure);
nfaClosure.State.AddTransition(
new DfaTransition(terminal, closure.State));
SharedPools.Default<HashSet<INfaState>>().Free(targetStates);
}
SharedPools
.Default<HashSet<INfaState>>()
.Free(nfaClosure.Closure);
SharedPools
.Default<Dictionary<ITerminal, HashSet<INfaState>>>()
.ClearAndFree(transitions);
}
return start.State;
}
private static INfa Optional(INfa nfa)
{
var start = new NfaState();
var end = new NfaState();
start.AddTransistion(new NullNfaTransition(nfa.Start));
start.AddTransistion(new NullNfaTransition(end));
nfa.End.AddTransistion(new NullNfaTransition(end));
return(new Nfa(start, end));
}
public Nfa BuildConcat(INfa concat1, INfa concat2)
{
var shiftedConcat2 = new ShiftedNfa(concat2, concat1.Size - 1);
var nfa = new Nfa(concat1.Size + concat2.Size - 1);
nfa.FillStates(concat1);
nfa.FillStates(shiftedConcat2);
return(nfa);
}
public INfa Union(INfa nfa)
{
var newStart = new NfaState();
var newEnd = new NfaState();
newStart.AddTransistion(new NullNfaTransition(Start));
newStart.AddTransistion(new NullNfaTransition(nfa.Start));
End.AddTransistion(new NullNfaTransition(newEnd));
nfa.End.AddTransistion(new NullNfaTransition(newEnd));
return new Nfa(newStart, newEnd);
}
public INfa Union(INfa nfa)
{
var newStart = new NfaState();
var newEnd = new NfaState();
newStart.AddTransistion(new NullNfaTransition(Start));
newStart.AddTransistion(new NullNfaTransition(nfa.Start));
End.AddTransistion(new NullNfaTransition(newEnd));
nfa.End.AddTransistion(new NullNfaTransition(newEnd));
return(new Nfa(newStart, newEnd));
}
public Nfa BuildStar(INfa inner)
{
var shiftedInner = new ShiftedNfa(inner, 1);
var nfa = new Nfa(shiftedInner.Size + 2);
nfa.FillStates(shiftedInner);
nfa.AddEpsilonTransition(nfa.Initial, shiftedInner.Initial);
nfa.AddEpsilonTransition(nfa.Initial, nfa.Final);
nfa.AddEpsilonTransition(shiftedInner.Final, nfa.Final);
nfa.AddEpsilonTransition(shiftedInner.Final, shiftedInner.Initial);
return(nfa);
}
private static INfa Union(INfa first, INfa second)
{
var start = new NfaState();
start.AddTransistion(new NullNfaTransition(first.Start));
start.AddTransistion(new NullNfaTransition(second.Start));
var end = new NfaState();
var endTransition = new NullNfaTransition(end);
first.End.AddTransistion(endTransition);
second.End.AddTransistion(endTransition);
return(new Nfa(start, end));
}
private static INfa KleeneStar(INfa nfa)
{
var start = new NfaState();
var nullToNfaStart = new NullNfaTransition(nfa.Start);
start.AddTransistion(nullToNfaStart);
nfa.End.AddTransistion(nullToNfaStart);
var end = new NfaState();
var nullToNewEnd = new NullNfaTransition(end);
start.AddTransistion(nullToNewEnd);
nfa.End.AddTransistion(nullToNewEnd);
return(new Nfa(start, end));
}
SubsetConstruction(INfa nfa)
{
var first = EpsilonClosure(nfa, Group <State> .From(nfa.Initial));
var groupTransitions = new GroupTransitions();
var terminalGroups = new List <Group <State> >();
// Create work queue of groups to look at
var unmarkedGroups = new Queue <Group <State> >();
unmarkedGroups.Enqueue(first);
var markedGroups = new HashSet <Group <State> >();
while (unmarkedGroups.Count != 0)
{
var group = unmarkedGroups.Dequeue();
markedGroups.Add(group);
if (group.Contains(nfa.Final))
{
terminalGroups.Add(group);
}
// Find a new epsilon closed group for each possible input
foreach (var input in nfa.Inputs)
{
var closure = EpsilonClosure(nfa, nfa.Move(group, input));
if (!closure.Any())
{
continue;
}
// Add new groups to unmarked groups
if (!unmarkedGroups.Contains(closure) && !markedGroups.Contains(closure))
{
unmarkedGroups.Enqueue(closure);
}
var transitions = new GroupTransition(group, input, closure);
groupTransitions.Add(transitions);
}
}
return(first, groupTransitions, terminalGroups);
}
public Nfa BuildAlter(INfa alter1, INfa alter2)
{
var shiftedAlter1 = new ShiftedNfa(alter1, 1);
var shiftedAlter2 = new ShiftedNfa(alter2, alter1.Size + 1);
var newNfa = new Nfa(alter1.Size + alter2.Size + 2);
newNfa.FillStates(shiftedAlter1);
newNfa.FillStates(shiftedAlter2);
newNfa.AddEpsilonTransition(0, shiftedAlter1.Initial);
newNfa.AddEpsilonTransition(0, shiftedAlter2.Initial);
newNfa.AddEpsilonTransition(shiftedAlter1.Final, newNfa.Final);
newNfa.AddEpsilonTransition(shiftedAlter2.Final, newNfa.Final);
return(newNfa);
}
private static HashSet <Symbol> GetAlphabet(INfa <Symbol> nfa)
{
HashSet <Symbol> alphabet = new HashSet <Symbol>();
Queue <IState> q = new Queue <IState>();
IState state = nfa.StartingState();
Util.HashableHashSet <IState> visited = new Util.HashableHashSet <IState>();
q.Enqueue(state);
visited.Add(state);
while (q.Count != 0)
{
state = q.Dequeue();
IReadOnlyDictionary <Symbol, IReadOnlyCollection <IState> > edges = nfa.Transitions(state);
foreach (Symbol key in edges.Keys)
{
alphabet.Add(key);
foreach (IState neighbour in edges[key])
{
if (!visited.Contains(neighbour))
{
visited.Add(neighbour);
q.Enqueue(neighbour);
}
}
}
foreach (IState neighbour in nfa.EpsilonTransitions(state))
{
if (!visited.Contains(neighbour))
{
visited.Add(neighbour);
q.Enqueue(neighbour);
}
}
}
return(alphabet);
}
/// <summary>
/// Implements <see cref="INfa{TState, TSymbol}.EliminateEpsilonTransitions"/>.
/// </summary>
/// <typeparam name="TState">The state type.</typeparam>
/// <typeparam name="TSymbol">The symbol type.</typeparam>
/// <param name="nfa">The NFA to eliminate epsilon-transitions from.</param>
/// <param name="copyTransitions">A method to copy transitions from one state to another.</param>
/// <returns>True, if there were epsilon-transitions to eliminate.</returns>
public static bool EliminateEpsilonTransitions <TState, TSymbol>(
INfa <TState, TSymbol> nfa,
Action <TState, TState> copyTransitions)
{
if (nfa.EpsilonTransitions.Count == 0)
{
return(false);
}
foreach (var state in nfa.States)
{
// For each state we look at its epsilon closure
// For each element in the closure we copy the non-epsilon transitions from state to the others
// We can omit the state itself from the copy
var epsilonClosure = nfa
.EpsilonClosure(state)
.Where(s => !nfa.StateComparer.Equals(s, state))
.ToHashSet(nfa.StateComparer);
foreach (var toState in epsilonClosure)
{
copyTransitions(state, toState);
// If v1 is a starting state, we need to make v2 one aswell
if (nfa.InitialStates.Contains(state))
{
nfa.InitialStates.Add(toState);
}
// If v2 is a final state, v1 needs to be aswell
if (nfa.AcceptingStates.Contains(toState))
{
nfa.AcceptingStates.Add(state);
}
}
}
nfa.EpsilonTransitions.Clear();
return(true);
}
private Group <State> EpsilonClosure(INfa nfa, Group <State> group)
{
var queue = new Queue <State>(group);
var closure = new Group <State>();
while (queue.Count != 0)
{
var state = queue.Dequeue();
closure.Add(state);
var epsilonTargets = nfa.Transitions
.Where(t => t.Source == state && t.Input == (char)Nfa.Constants.Epsilon)
.Where(t => !closure.Contains(t.Target))
.Select(t => t.Target);
foreach (var target in epsilonTargets)
{
queue.Enqueue(target);
}
}
return(closure);
}
public Dfa DfaFromNfa(INfa nfa)
{
var(startGroup, groupTransitions, terminalGroups) = SubsetConstruction(nfa);
var groups = groupTransitions
.Select(t => t.Target)
.Union(new[] { startGroup }).Distinct();
var id = 0;
var stateFromGroup = groups
.OrderBy(g => g.Min())
.Select(g => (Group: g, Id: id++))
.ToDictionary(kvp => kvp.Group, kvp => kvp.Id);
var start = stateFromGroup[startGroup];
var transitions = groupTransitions.ToDictionary(
t => (stateFromGroup[t.Source], t.Input),
t => stateFromGroup[t.Target]
);
var terminals = terminalGroups.Select(g => stateFromGroup[g]);
return(new Dfa(start, transitions, terminals));
}
// tokenCategories - List of pair (Token, Regex for thie Token)
public Lexer(
IEnumerable <KeyValuePair <TLabel, string> > tokenCategories,
TLabel eof,
TLabel noneValue,
Func <IEnumerable <TLabel>, TLabel> conflictSolver)
{
this.eof = eof;
this.noneValue = noneValue;
var converter = new StringToRegexConverterFactory().CreateConverter();
Dictionary <TLabel, IDfa <bool, char> > multipleDfa = tokenCategories.ToDictionary(
x => x.Key,
x =>
{
Regex <char> regex = converter.Convert(x.Value);
INfa <char> nfaPre = RegexToNfaConverter <char> .Convert(regex);
INfa <char> nfa = ConcreteNfa <char> .CreateFromNfa(nfaPre);
IDfa <bool, char> dfa = NfaToDfaConverter <char> .Convert(nfa);
return(DfaMinimizer <bool, char> .Minimize(dfa));
});
var mergedDfa = DfaMerger <TLabel, char> .Merge(multipleDfa, conflictSolver);
this.minimalizedDfa = DfaMinimizer <TLabel, char> .Minimize(mergedDfa);
}
private int ConstructStates(Dictionary <IState, int> stateIds, INfa <Symbol> nfa, IState state)
{
if (stateIds.ContainsKey(state))
{
return(stateIds[state]);
}
int stateId = stateIds.Count;
stateIds[state] = stateId;
this.epsilonTransitions.Add(null);
this.transitions.Add(null);
this.isAccepting.Add(nfa.IsAccepting(state));
this.epsilonTransitions[stateId] = nfa.EpsilonTransitions(state).Select(target => new ValueState <int>(this.ConstructStates(stateIds, nfa, target)) as IState).ToList();
this.transitions[stateId] = nfa.Transitions(state)
.ToDictionary(
keySelector: kv => kv.Key,
elementSelector: kv => kv.Value.Select(target => new ValueState <int>(this.ConstructStates(stateIds, nfa, target))).ToList() as IReadOnlyCollection <IState>);
return(stateId);
}
public Nfa BuildPlus(INfa inner) => BuildConcat(inner, BuildStar(inner));
private static void VerifyNfaIsNotNullAndHasValidStartAndEndStates(INfa nfa)
{
Assert.IsNotNull(nfa);
Assert.IsNotNull(nfa.Start);
Assert.IsNotNull(nfa.End);
Assert.AreEqual(0, nfa.End.Transitions.Count());
}
public INfa Concatenation(INfa nfa)
{
End.AddTransistion(
new NullNfaTransition(nfa.Start));
return(this);
}
public INfa Concatenation(INfa nfa)
{
End.AddTransistion(
new NullNfaTransition(nfa.Start));
return this;
}
private static INfa Union(INfa first, INfa second)
{
var start = new NfaState();
start.AddTransistion(new NullNfaTransition(first.Start));
start.AddTransistion(new NullNfaTransition(second.Start));
var end = new NfaState();
var endTransition = new NullNfaTransition(end);
first.End.AddTransistion(endTransition);
second.End.AddTransistion(endTransition);
return new Nfa(start, end);
}
private IDfaState ConvertNfaToDfa(INfa nfa)
{
return new SubsetConstructionAlgorithm().Transform(nfa);
}
public static IDfa <bool, Symbol> Convert(INfa <Symbol> nfa)
{
HashSet <Symbol> alphabet = GetAlphabet(nfa);
Dictionary <Util.HashableHashSet <IState>, DfaState> map = new Dictionary <Util.HashableHashSet <IState>, DfaState>();
HashSet <DfaState> accepting = new HashSet <DfaState>();
Dictionary <DfaState, Dictionary <Symbol, IState> > trans = new Dictionary <DfaState, Dictionary <Symbol, IState> >();
Queue <Util.HashableHashSet <IState> > q = new Queue <Util.HashableHashSet <IState> >();
Util.HashableHashSet <IState> start = new Util.HashableHashSet <IState> {
nfa.StartingState()
};
Util.HashableHashSet <IState> startingClosure = EpsilonClosure(nfa, start);
map.Add(startingClosure, new DfaState());
q.Enqueue(startingClosure);
while (q.Count != 0)
{
Util.HashableHashSet <IState> currentSet = q.Dequeue();
foreach (IState state in currentSet)
{
if (nfa.IsAccepting(state))
{
accepting.Add(map[currentSet]);
break;
}
}
Dictionary <Symbol, Util.HashableHashSet <IState> > dict = new Dictionary <Symbol, Util.HashableHashSet <IState> >();
foreach (IState state in currentSet)
{
IReadOnlyDictionary <Symbol, IReadOnlyCollection <IState> > edges = nfa.Transitions(state);
foreach (Symbol key in edges.Keys)
{
if (!dict.ContainsKey(key))
{
dict.Add(key, new Util.HashableHashSet <IState>());
}
foreach (IState s in edges[key])
{
dict[key].Add(s);
}
}
}
foreach (Symbol key in alphabet)
{
if (!dict.ContainsKey(key))
{
dict.Add(key, new Util.HashableHashSet <IState>());
}
}
trans.Add(map[currentSet], new Dictionary <Symbol, IState>());
foreach (Symbol key in dict.Keys)
{
Util.HashableHashSet <IState> neighbour = EpsilonClosure(nfa, dict[key]);
if (!map.ContainsKey(neighbour))
{
map.Add(neighbour, new DfaState());
q.Enqueue(neighbour);
}
trans[map[currentSet]].Add(key, map[neighbour]);
}
}
return(new Dfa <Symbol>(map[startingClosure], accepting, trans));
}
private static List <string> GetAllAcceptedStringsUpToLength(int maxLength, List <char> vocabulary, INfa <char> nfa)
{
return(GetAllStringsUpToLength(maxLength, vocabulary).Where(nfa.Accepts).ToList());
}
private static INfa Concatenation(INfa first, INfa second)
{
first.End.AddTransistion(new NullNfaTransition(second.Start));
return(new Nfa(first.Start, second.End));
}
public StarNfa(INfa <Symbol> inner)
{
this.Inner = inner;
}
private static INfa Optional(INfa nfa)
{
var start = new NfaState();
var end = new NfaState();
start.AddTransistion(new NullNfaTransition(nfa.Start));
start.AddTransistion(new NullNfaTransition(end));
nfa.End.AddTransistion(new NullNfaTransition(end));
return new Nfa(start, end);
}
private static INfa KleeneStar(INfa nfa)
{
var start = new NfaState();
var nullToNfaStart = new NullNfaTransition(nfa.Start);
start.AddTransistion(nullToNfaStart);
nfa.End.AddTransistion(nullToNfaStart);
var end = new NfaState();
var nullToNewEnd = new NullNfaTransition(end);
start.AddTransistion(nullToNewEnd);
nfa.End.AddTransistion(nullToNewEnd);
return new Nfa(start, end);
}
private IDfaState ConvertNfaToDfa(INfa nfa)
{
return(new SubsetConstructionAlgorithm().Transform(nfa));
}
private static INfa Concatenation(INfa first, INfa second)
{
first.End.AddTransistion(new NullNfaTransition(second.Start));
return new Nfa(first.Start, second.End);
}
public ShiftedNfa(INfa inner, int shift)
{
Inner = inner;
Shift = shift;
}
public IDfaState Transform(INfa nfa)
{
var processOnceQueue = new ProcessOnceQueue <NfaClosure>();
var set = SharedPools.Default <SortedSet <INfaState> >().AllocateAndClear();
foreach (var state in nfa.Start.Closure())
{
set.Add(state);
}
var start = new NfaClosure(set, nfa.Start.Equals(nfa.End));
processOnceQueue.Enqueue(start);
while (processOnceQueue.Count > 0)
{
var nfaClosure = processOnceQueue.Dequeue();
var transitions = SharedPools
.Default <Dictionary <ITerminal, SortedSet <INfaState> > >()
.AllocateAndClear();
for (int i = 0; i < nfaClosure.Closure.Length; i++)
{
var state = nfaClosure.Closure[i];
for (var t = 0; t < state.Transitions.Count; t++)
{
var transition = state.Transitions[t];
switch (transition.TransitionType)
{
case NfaTransitionType.Edge:
var terminalTransition = transition as TerminalNfaTransition;
var terminal = terminalTransition.Terminal;
if (!transitions.ContainsKey(terminalTransition.Terminal))
{
transitions[terminal] = SharedPools.Default <SortedSet <INfaState> >().AllocateAndClear();
}
transitions[terminal].Add(transition.Target);
break;
}
}
}
foreach (var terminal in transitions.Keys)
{
var targetStates = transitions[terminal];
var closure = Closure(targetStates, nfa.End);
closure = processOnceQueue.EnqueueOrGetExisting(closure);
nfaClosure.State.AddTransition(
new DfaTransition(terminal, closure.State));
SharedPools.Default <SortedSet <INfaState> >().ClearAndFree(targetStates);
}
SharedPools
.Default <SortedSet <INfaState> >()
.ClearAndFree(nfaClosure.Set);
SharedPools
.Default <Dictionary <ITerminal, SortedSet <INfaState> > >()
.ClearAndFree(transitions);
}
return(start.State);
}
public SumNfa(INfa <Symbol> left, INfa <Symbol> right)
{
this.Left = left;
this.Right = right;
}
private static INfa KleenePlus(INfa nfa)
{
var end = new NfaState();
nfa.End.AddTransistion(new NullNfaTransition(end));
nfa.End.AddTransistion(new NullNfaTransition(nfa.Start));
return new Nfa(nfa.Start, end);
}
