public LexerNFA(LexerSpec lexerSpec, IDictionary <Mode, State> modeMap, CodePointEquivalenceClasses equivalenceClasses, NFA <LexerAction> nfa)
{
this.LexerSpec = lexerSpec;
ModeMap = new Dictionary <Mode, State>(modeMap);
EquivalenceClasses = equivalenceClasses;
Nfa = nfa;
}
public LexerDFA(LexerSpec lexerSpec, IDictionary <Mode, State> modeMap, CodePointEquivalenceClasses equivalenceClasses, DFA <LexerAction> dfa)
{
LexerSpec = lexerSpec;
ModeMap = new Dictionary <Mode, State>(modeMap);
EquivalenceClasses = equivalenceClasses;
Dfa = dfa;
}
public override void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses)
{
foreach (var expression in Expressions)
{
expression.MakeEquivalenceClasses(equivalenceClasses);
}
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
return(Expressions.Select(exp => exp.AddTo(nfa, equivalenceClasses)).Aggregate((states1, states2) =>
{
nfa.AddEpsilonTransition(states1.End, states2.Start);
return new StateRange(states1.End, states2.Start);
}));
}
private static CodePointEquivalenceClasses MakeEquivalenceClasses(LexerSpec spec)
{
var equivalenceClasses = new CodePointEquivalenceClasses();
foreach (var rule in spec.Rules)
{
rule.Expression.MakeEquivalenceClasses(equivalenceClasses);
}
return(equivalenceClasses);
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
var startState = nfa.AddState();
var endState = nfa.AddState();
foreach (var equivalenceClass in equivalenceClasses.GetClasses(CodePoints))
{
nfa.AddTransition(startState, equivalenceClass, endState);
}
return(new StateRange(startState, endState));
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
var startState = nfa.AddState();
var endState = nfa.AddState();
foreach (var states in Expressions.Select(exp => exp.AddTo(nfa, equivalenceClasses)))
{
nfa.AddEpsilonTransition(startState, states.Start);
nfa.AddEpsilonTransition(states.End, endState);
}
return(new StateRange(startState, endState));
}
public void AddStates(IDictionary<Mode, State> modeMap, CodePointEquivalenceClasses equivalenceClasses, NFA<LexerAction> nfa, int priority, Channel defaultChannel)
{
var states = Expression.AddTo(nfa, equivalenceClasses);
foreach(var mode in Modes)
nfa.AddEpsilonTransition(modeMap[mode], states.Start);
nfa.SetFinal(states.End);
// Set Action
nfa.SetData(states.End, new LexerAction(priority, GetValueAction(), GetModeActions(), GetEmitAction(defaultChannel), GetCodeAction()));
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
var startState = nfa.AddState();
var endState = startState;
foreach (var codePoint in Value.GetCodePoints())
{
var lastState = endState;
endState = nfa.AddState();
nfa.AddTransition(lastState, equivalenceClasses.GetClass(codePoint), endState);
}
return(new StateRange(startState, endState));
}
public void AddStates(IDictionary <Mode, State> modeMap, CodePointEquivalenceClasses equivalenceClasses, NFA <LexerAction> nfa, int priority, Channel defaultChannel)
{
var states = Expression.AddTo(nfa, equivalenceClasses);
foreach (var mode in Modes)
{
nfa.AddEpsilonTransition(modeMap[mode], states.Start);
}
nfa.SetFinal(states.End);
// Set Action
nfa.SetData(states.End, new LexerAction(priority, GetValueAction(), GetModeActions(), GetEmitAction(defaultChannel), GetCodeAction()));
}
private static IDictionary <int, Input[]> GenEquivalenceTable(CodePointEquivalenceClasses equivalenceClasses)
{
var equivalenceTable = new Dictionary <int, Input[]>();
foreach (var plane in UnicodePlanes)
{
var planeEquivalenceClasses = new Input[0x10000];
equivalenceTable.Add(plane, planeEquivalenceClasses);
var planeBits = plane << 16;
for (var planePlace = 0; planePlace < 0x10000; planePlace++)
{
planeEquivalenceClasses[planePlace] = equivalenceClasses.GetClass((CodePoint)(planeBits + planePlace));
}
}
return(equivalenceTable);
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
var reps = AddReps(nfa, equivalenceClasses).ToList();
if (MaxRepititions == null)
{
// Repeat last rep as many times as needed
var lastRep = reps.Last();
nfa.AddEpsilonTransition(lastRep.End, lastRep.Start);
}
return(reps.Aggregate((states1, states2) =>
{
nfa.AddEpsilonTransition(states1.End, states2.Start);
return new StateRange(states1.Start, states2.End);
}));
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
// Create a separate NFA
var tempNFA = new NFA <State?>(equivalenceClasses.Count);
var states = Expression.AddTo(tempNFA, equivalenceClasses);
tempNFA.SetStart(states.Start);
tempNFA.SetFinal(states.End);
// Convert to a DFA
var dfa = tempNFA.ToDFA(stateData => null).Item2;
dfa = dfa.Minimize().Item2;
dfa.MakeComplete(); // Can only do complement on complete DFA
// Now add the complement to this nfa
var startState = nfa.AddState();
var endState = nfa.AddState();
foreach (var dfaState in dfa.States)
{
var nfaState = GetMatchingState(nfa, dfa, dfaState);
if (dfa.IsStart(dfaState))
{
nfa.AddEpsilonTransition(startState, nfaState);
}
if (!dfa.IsFinal(dfaState))
{
nfa.AddEpsilonTransition(nfaState, endState);
}
// Copy transitions
foreach (var input in nfa.Inputs)
{
var toDFAState = dfa.GetTransition(dfaState, input);
if (toDFAState != null)
{
var toNFAState = GetMatchingState(nfa, dfa, toDFAState.Value);
nfa.AddTransition(nfaState, input, toNFAState);
}
}
}
return(new StateRange(startState, endState));
}
private IEnumerable <StateRange> AddReps <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
// Add required reps
for (var i = 0; i < MinRepititions; i++)
{
yield return(Expression.AddTo(nfa, equivalenceClasses));
}
// Add optional reps
// If min=0 and max=*, then we want to make one optional rep to use for the * hence `MaxRepititions ?? 1`
var optionalReps = (MaxRepititions ?? 1) - MinRepititions;
for (var i = 0; i < optionalReps; i++)
{
var states = Expression.AddTo(nfa, equivalenceClasses);
// Can skip these reps
nfa.AddEpsilonTransition(states.Start, states.End);
yield return(states);
}
}
private static IDictionary<int, Input[]> GenEquivalenceTable(CodePointEquivalenceClasses equivalenceClasses)
{
var equivalenceTable = new Dictionary<int, Input[]>();
foreach(var plane in UnicodePlanes)
{
var planeEquivalenceClasses = new Input[0x10000];
equivalenceTable.Add(plane, planeEquivalenceClasses);
var planeBits = plane << 16;
for(var planePlace = 0; planePlace < 0x10000; planePlace++)
planeEquivalenceClasses[planePlace] = equivalenceClasses.GetClass((CodePoint)(planeBits + planePlace));
}
return equivalenceTable;
}
public override StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses)
{
throw new NotSupportedException("Adding Difference to NFA not supported, should be simplified away");
}
public override void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses)
{
throw new NotSupportedException("RuleReference should be simplified away before making equivalence classes");
}
public override void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses)
{
equivalenceClasses.MakeClasses(CodePoints);
}
private static CodePointEquivalenceClasses MakeEquivalenceClasses(LexerSpec spec)
{
var equivalenceClasses = new CodePointEquivalenceClasses();
foreach(var rule in spec.Rules)
rule.Expression.MakeEquivalenceClasses(equivalenceClasses);
return equivalenceClasses;
}
public override void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses)
{
Expression.MakeEquivalenceClasses(equivalenceClasses);
}
public abstract void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses);
public abstract StateRange AddTo <T>(NFA <T> nfa, CodePointEquivalenceClasses equivalenceClasses);
public override void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses)
{
Expression.MakeEquivalenceClasses(equivalenceClasses);
}
public override void MakeEquivalenceClasses(CodePointEquivalenceClasses equivalenceClasses)
{
foreach(var expression in Expressions)
expression.MakeEquivalenceClasses(equivalenceClasses);
}
