public static IReadOnlyCollection <IState> GetAllStates <TLabel, Symbol>(this IDfa <TLabel, Symbol> dfa)
{
var visited = new HashSet <IState>();
var queue = new Queue <IState>();
queue.Enqueue(dfa.StartingState());
visited.Add(dfa.StartingState());
while (queue.Count > 0)
{
var s = queue.Dequeue();
var transitions = dfa.Transitions(s);
foreach (var nextState in transitions.Values)
{
if (!visited.Contains(nextState))
{
queue.Enqueue(nextState);
visited.Add(nextState);
}
}
}
return(visited);
}
internal static int NumberOfPseudoDeadStates(this IDfa <char> dfa)
{
int number = 0;
Queue <IDfaState <char> > Q = new Queue <IDfaState <char> >();
Dictionary <IDfaState <char>, int> color = new Dictionary <IDfaState <char>, int>();
Q.Enqueue(dfa.Start);
while (Q.Count > 0)
{
IDfaState <char> current_state = Q.Dequeue();
if (!color.ContainsKey(current_state))
{
color[current_state] = 1;
if (current_state.IsPseudoDead())
{
number++;
}
foreach (var transition in current_state.Transitions)
{
if (!color.ContainsKey(transition.Value))
{
Q.Enqueue(transition.Value);
}
}
}
}
return(number);
}
private static IDfa <bool, TLabel> RegexToDfa(Regex <TLabel> regex)
{
INfa <TLabel> nfa = RegexToNfaConverter <TLabel> .Convert(regex);
IDfa <bool, TLabel> dfa = NfaToDfaConverter <TLabel> .Convert(nfa);
return(DfaMinimizer <bool, TLabel> .Minimize(dfa));
}
private void Call(IDfa <Optional <Rule <TLabel> >, TLabel> dfa, IState state, TLabel actionLabel)
{
this.statesStack.Pop();
this.statesStack.Push(dfa.Transitions(state)[actionLabel]);
this.dfaStack.Push(this.rules[actionLabel]);
this.statesStack.Push(this.rules[actionLabel].StartingState());
this.nodesStack.Push(new List <ParseTree <TLabel> >());
}
public MinimalDfa(IDfa <TLabel, Symbol> dfa, List <HashSet <IState> > statePartition)
{
this.dfa = dfa;
this.statePartition = statePartition;
this.CreateStateMapping();
this.FindStableStates();
}
private static DfaAndState <TLabel> CreateDfaAndState(IDfa <Optional <Rule <TLabel> >, TLabel> dfa, IState state)
{
var pair = new DfaAndState <TLabel>
{
Dfa = dfa,
State = state
};
return(pair);
}
public static IDfa <TSymbol> MakeMinimizedProductDfa <TSymbol>(IDfa <TSymbol> lastDfa, IDfa <TSymbol> newDfa,
AmbiguityHandler handler)
where TSymbol : IEquatable <TSymbol>, IComparable <TSymbol>
{
// This implementation assumes that all instances of IDfa are in fact instances of AbstractDfa
var lastDfaType = lastDfa.GetType();
var newDfaType = newDfa.GetType();
var lastDfaStateType = lastDfa.Start.GetType();
var newDfaStateType = newDfa.Start.GetType();
return((IDfa <TSymbol>)MakeMinimizedProductDfaMethod
.MakeGenericMethod(lastDfaType, lastDfaStateType, newDfaType, newDfaStateType, typeof(TSymbol))
.Invoke(null, new object[] { lastDfa, newDfa, handler }));
}
private static void CheckMinimization <TLabel>(IDfa <TLabel, char> dfa, int expectedNumberOfStates = -1)
{
var minimalDfa = DfaMinimizer <TLabel, char> .Minimize(dfa);
var numberOfStates = ReachableStates(minimalDfa).Count;
if (expectedNumberOfStates != -1)
{
Assert.AreEqual(expectedNumberOfStates, numberOfStates);
}
CheckAutomatonEquivalence(dfa, minimalDfa);
CheckStateStability(minimalDfa);
}
// Be aware! The minimal dfa for the a given one will reuse some states!
public static IDfa <TLabel, Symbol> Minimize(IDfa <TLabel, Symbol> dfa)
{
List <HashSet <IState> > partition = InitialPartition(dfa);
while (true)
{
var newPartition = MooresStep(dfa, partition);
if (PartitionRepresentation(partition) == PartitionRepresentation(newPartition))
{
break;
}
partition = newPartition;
}
return(new MinimalDfa(dfa, partition));
}
private static TLabel GetTargetLabel <TLabel>(IDfa <TLabel, char> dfa, string s, TLabel defaultLabel)
{
var state = dfa.StartingState();
foreach (var ch in s)
{
var transitions = dfa.Transitions(state);
if (!transitions.ContainsKey(ch))
{
return(defaultLabel);
}
state = transitions[ch];
}
return(dfa.Label(state));
}
private static void CheckStateStability <TLabel>(IDfa <TLabel, char> dfa)
{
foreach (var state in ReachableStates(dfa))
{
bool expectedStability = true;
foreach (var(_, nextState) in dfa.Transitions(state))
{
if (nextState != state)
{
expectedStability = false;
break;
}
}
Assert.AreEqual(expectedStability, dfa.IsStable(state), "Stability of the state is incorrect");
}
}
public static bool AreEquivalent(IDfa <TLabel, Symbol> firstDfa, IDfa <TLabel, Symbol> secondDfa)
{
var reached = new HashSet <Tuple <IState, IState> >();
var queue = new Queue <Tuple <IState, IState> >();
queue.Enqueue(Tuple.Create(firstDfa.StartingState(), secondDfa.StartingState()));
reached.Add(queue.Peek());
while (queue.Count > 0)
{
var(firstState, secondState) = queue.Dequeue();
if (!firstDfa.Label(firstState).Equals(secondDfa.Label(secondState)))
{
return(false);
}
var firstStateTransitions = firstDfa.Transitions(firstState);
var secondStateTransitions = secondDfa.Transitions(secondState);
if (!firstStateTransitions.Count.Equals(secondStateTransitions.Count))
{
return(false);
}
foreach (var(c, firstNextState) in firstStateTransitions)
{
if (!secondStateTransitions.ContainsKey(c))
{
return(false);
}
var secondNextState = secondStateTransitions[c];
var pair = Tuple.Create(firstNextState, secondNextState);
if (!reached.Contains(pair))
{
reached.Add(pair);
queue.Enqueue(pair);
}
}
}
return(true);
}
private static List <HashSet <IState> > MooresStep(IDfa <TLabel, Symbol> dfa, List <HashSet <IState> > partition)
{
var stateClassId = new Dictionary <IState, int>();
for (int i = 0; i < partition.Count; ++i)
{
foreach (var state in partition[i])
{
stateClassId[state] = i;
}
}
var stateDerivatives = new List <Tuple <IState, List <int> > >();
foreach (var equivalenceClass in partition)
{
foreach (var state in equivalenceClass)
{
var derivative = new List <int> {
stateClassId[state]
};
derivative.AddRange(dfa.Transitions(state).OrderBy(t => t.Key).Select(t => stateClassId[t.Value]));
stateDerivatives.Add(Tuple.Create(state, derivative));
}
}
var newPartition = new List <HashSet <IState> >();
var groups = stateDerivatives.GroupBy(d => string.Join(" ", d.Item2.Select(t => Convert.ToString(t))));
foreach (var group in groups)
{
var partitionGroup = new HashSet <IState>();
foreach (var(state, _) in group)
{
partitionGroup.Add(state);
}
newPartition.Add(partitionGroup);
}
return(newPartition);
}
public void MakeMinimizedProductDfaTypePunnedInvocation()
{
IDfa <char> someDfa = DfaUtils.MakeEmptyLanguageDfa <char>();
var someOtherDfa = DfaUtils.MakeMinimizedProductDfa(someDfa, someDfa,
(a, b) =>
{
if (a != 0 && b != 0)
{
throw new ArgumentException();
}
return(a + b);
});
Assert.NotNull(someOtherDfa);
Assert.NotNull(someOtherDfa.Start);
Assert.AreEqual(someOtherDfa.Start.Accepting, 0);
Assert.AreEqual(someOtherDfa.Start.Transitions.Count, 1);
Assert.AreEqual(someOtherDfa.Start.Transitions[0].Key, '\0');
Assert.AreSame(someOtherDfa.Start.Transitions[0].Value, someOtherDfa.Start);
}
private void Shift(
ParseTree <TLabel> currentToken,
IDfa <Optional <Rule <TLabel> >, TLabel> dfa,
IState state,
TLabel currentCategory,
IEnumerator enumerator)
{
this.nodesStack.Peek().Add(currentToken);
this.statesStack.Pop();
this.statesStack.Push(dfa.Transitions(state)[currentCategory]);
if (!enumerator.MoveNext())
{
this.diagnostics.Add(new Diagnostic(
DiagnosticStatus.Error,
PrematureEofDiagnosticType,
"Parsing finished before reading all tokens",
new List <Range>()));
throw new ParseException("Premature EOF");
}
}
private static List <HashSet <IState> > InitialPartition(IDfa <TLabel, Symbol> dfa)
{
var partition = new List <HashSet <IState> >();
var labelId = new Dictionary <TLabel, int>();
int labelCount = 0;
foreach (IState state in ReachableStates(dfa))
{
TLabel stateLabel = dfa.Label(state);
if (!labelId.ContainsKey(stateLabel))
{
labelId.Add(stateLabel, labelCount);
partition.Add(new HashSet <IState>());
++labelCount;
}
partition[labelId[stateLabel]].Add(state);
}
return(partition);
}
private static HashSet <IState> ReachableStates <TLabel>(IDfa <TLabel, char> dfa)
{
var reachedStates = new HashSet <IState>();
var queue = new Queue <IState>(new[] { dfa.StartingState() });
while (queue.Count > 0)
{
var state = queue.Dequeue();
foreach (var(_, value) in dfa.Transitions(state))
{
if (!reachedStates.Contains(value))
{
reachedStates.Add(value);
queue.Enqueue(value);
}
}
}
return(reachedStates);
}
private static void CheckAutomatonEquivalence <TLabel>(IDfa <TLabel, char> firstDfa, IDfa <TLabel, char> secondDfa)
{
var reached = new HashSet <Tuple <IState, IState> >();
var queue = new Queue <Tuple <IState, IState> >();
queue.Enqueue(Tuple.Create(firstDfa.StartingState(), secondDfa.StartingState()));
reached.Add(queue.Peek());
while (queue.Count > 0)
{
var(firstState, secondState) = queue.Dequeue();
Assert.AreEqual(firstDfa.Label(firstState), secondDfa.Label(secondState), "Labels should be equal");
var firstStateTransitions = firstDfa.Transitions(firstState);
var secondStateTransitions = secondDfa.Transitions(secondState);
var firstTransitionCount = firstStateTransitions.Count;
var secondTransitionCount = secondStateTransitions.Count;
Assert.AreEqual(
firstTransitionCount,
secondTransitionCount,
$"States {firstState} and {secondState} have different number of transitions: {firstTransitionCount} != {secondTransitionCount}");
foreach (var(c, firstNextState) in firstStateTransitions)
{
Assert.IsTrue(
secondStateTransitions.ContainsKey(c),
$"States have different sets of transitions, {secondTransitionCount} : {string.Join(",", firstStateTransitions.Select((x, y) => ((int)x.Key).ToString()).ToList())}");
var secondNextState = secondStateTransitions[c];
var pair = Tuple.Create(firstNextState, secondNextState);
if (!reached.Contains(pair))
{
reached.Add(pair);
queue.Enqueue(pair);
}
}
}
}
// tokenCategories - List of pair (Token, Regex for thie Token)
public Lexer(
IEnumerable <KeyValuePair <TLabel, string> > tokenCategories,
TLabel eof,
TLabel noneValue,
Func <IEnumerable <TLabel>, TLabel> conflictSolver)
{
this.eof = eof;
this.noneValue = noneValue;
var converter = new StringToRegexConverterFactory().CreateConverter();
Dictionary <TLabel, IDfa <bool, char> > multipleDfa = tokenCategories.ToDictionary(
x => x.Key,
x =>
{
Regex <char> regex = converter.Convert(x.Value);
INfa <char> nfaPre = RegexToNfaConverter <char> .Convert(regex);
INfa <char> nfa = ConcreteNfa <char> .CreateFromNfa(nfaPre);
IDfa <bool, char> dfa = NfaToDfaConverter <char> .Convert(nfa);
return(DfaMinimizer <bool, char> .Minimize(dfa));
});
var mergedDfa = DfaMerger <TLabel, char> .Merge(multipleDfa, conflictSolver);
this.minimalizedDfa = DfaMinimizer <TLabel, char> .Minimize(mergedDfa);
}
private ParseTree <TLabel> Reduce(
IDfa <Optional <Rule <TLabel> >, TLabel> dfa,
IState state,
ParseTree <TLabel> currentToken)
{
if (dfa.Label(state).IsNone())
{
this.diagnostics.Add(new Diagnostic(
DiagnosticStatus.Error,
InvalidReduceActionDiagnosticType,
"Invalid reduce action at {0}",
new List <Range> {
currentToken.InputRange
}));
throw new ParseException("Invalid reduce action");
}
var rule = dfa.Label(state).Get();
var topNodes = this.nodesStack.Peek();
var newCategory = rule.Lhs;
var range = GetRange(topNodes);
ParseTree <TLabel> root = new Brunch <TLabel>
{
Rule = rule, Children = topNodes, Category = newCategory, InputRange = range
};
this.statesStack.Pop();
this.dfaStack.Pop();
this.nodesStack.Pop();
if (this.nodesStack.Count != 0)
{
this.nodesStack.Peek().Add(root);
}
return(root);
}
private static HashSet <IState> ReachableStates(IDfa <TLabel, Symbol> dfa)
{
var reachedStates = new HashSet <IState>();
var queue = new Queue <IState>();
queue.Enqueue(dfa.StartingState());
reachedStates.Add(dfa.StartingState());
while (queue.Count > 0)
{
IState state = queue.Dequeue();
foreach (KeyValuePair <Symbol, IState> transition in dfa.Transitions(state))
{
if (!reachedStates.Contains(transition.Value))
{
reachedStates.Add(transition.Value);
queue.Enqueue(transition.Value);
}
}
}
return(reachedStates);
}
private static DfaAndState <Symbol> StateEntity(IDfa <Optional <Rule <Symbol> >, Symbol> dfa, IState state)
{
return(new DfaAndState <Symbol> {
Dfa = dfa, State = state
});
}
public Lexer(IDfa <TLabel, char> dfa, TLabel eof)
{
this.minimalizedDfa = dfa;
this.eof = eof;
}
private static Dictionary <IState, Dictionary <TLabel, HashSet <IState> > > GetReverseTransitions(IDfa <Optional <Rule <TLabel> >, TLabel> dfa)
{
var allStates = dfa.GetAllStates();
var reverseTransitions = new Dictionary <IState, Dictionary <TLabel, HashSet <IState> > >();
foreach (IState state in allStates)
{
reverseTransitions.Add(state, new Dictionary <TLabel, HashSet <IState> >());
}
foreach (IState state in allStates)
{
foreach (var transition in dfa.Transitions(state))
{
var symbol = transition.Key;
var newState = transition.Value;
if (!reverseTransitions[newState].ContainsKey(symbol))
{
reverseTransitions[newState].Add(symbol, new HashSet <IState>());
}
reverseTransitions[transition.Value][transition.Key].Add(state);
}
}
return(reverseTransitions);
}
private static bool IsAccepting(IDfa <Optional <Rule <TLabel> >, TLabel> dfa, IState state)
{
return(!dfa.Label(state).IsNone());
}
private static IReadOnlyCollection <ParseAction <TLabel> > GetAllParseActions(
CompiledGrammar <TLabel> grammar,
IState state,
IDfa <Optional <Rule <TLabel> >, TLabel> dfa,
TLabel label,
IReadOnlyDictionary <TLabel, IReadOnlyCollection <DfaAndState <TLabel> > > follow,
IReadOnlyDictionary <TLabel, IReadOnlyCollection <DfaAndState <TLabel> > > firstPlus)
{
var actions = new List <ParseAction <TLabel> >();
var stateLabel = dfa.Label(state);
var dfaAndState = new DfaAndState <TLabel>()
{
Dfa = dfa, State = state
};
if (stateLabel.IsSome() && follow.ContainsKey(label) && follow[label].Contains(dfaAndState))
{
actions.Add(
new ParseAction <TLabel>()
{
Kind = ParseAction <TLabel> .ActionKind.Reduce, Label = stateLabel.Get().Lhs
});
}
var transitions = dfa.Transitions(state);
if (transitions.ContainsKey(label))
{
var nextState = transitions[label];
if (!dfa.IsStable(nextState) || dfa.Label(nextState).IsSome())
{
actions.Add(
new ParseAction <TLabel>()
{
Kind = ParseAction <TLabel> .ActionKind.Shift, Label = label
});
}
}
if (firstPlus.ContainsKey(label))
{
var firstPlusStates = firstPlus[label];
foreach (var transitionLabel in transitions.Keys)
{
var nextState = transitions[transitionLabel];
if ((!dfa.IsStable(nextState) || dfa.Label(nextState).IsSome()) && grammar.Rules.ContainsKey(transitionLabel))
{
var subDfa = grammar.Rules[transitionLabel];
var subDfaAndStartState = new DfaAndState <TLabel>()
{
Dfa = subDfa, State = subDfa.StartingState()
};
if (firstPlusStates.Contains(subDfaAndStartState))
{
actions.Add(
new ParseAction <TLabel>()
{
Kind = ParseAction <TLabel> .ActionKind.Call, Label = transitionLabel
});
}
}
}
}
return(actions);
}
internal static void CheckDfaStates(IDfa <TSymbol> dfa)
{
new DfaStatesConcpetCheck <TSymbol>().Check(dfa.Start);
}
