private static BaseLexerRule CreateMultiLineCommentLexerRule()
{
var pattern = @"\/[*]([*][^\/]|[^*])*[*][\/]";
var states = new DfaState[5];
for (int i = 0; i < states.Length; i++)
{
states[i] = new DfaState();
}
var slash = new CharacterTerminal('/');
var star = new CharacterTerminal('*');
var notStar = new NegationTerminal(star);
var notSlash = new NegationTerminal(slash);
var firstSlash = new DfaTransition(slash, states[1]);
var firstStar = new DfaTransition(star, states[2]);
var repeatNotStar = new DfaTransition(notStar, states[2]);
var lastStar = new DfaTransition(star, states[3]);
var goBackNotSlash = new DfaTransition(notSlash, states[2]);
var lastSlash = new DfaTransition(slash, states[4]);
states[0].AddTransition(firstSlash);
states[1].AddTransition(firstStar);
states[2].AddTransition(repeatNotStar);
states[2].AddTransition(lastStar);
states[3].AddTransition(goBackNotSlash);
states[3].AddTransition(lastSlash);
return(new DfaLexerRule(states[0], pattern));
}
static NumberLexerRule()
{
var states = new DfaState[5];
for (int i = 0; i < states.Length; i++)
{
states[i] = new DfaState(i == 4 || i == 2);
}
var zeroThroughNine = new RangeTerminal('0', '9');
var plusOrMinusTo1 = new DfaTransition(new SetTerminal('+', '-'), states[1]);
var dotTo3 = new DfaTransition(new CharacterTerminal('.'), states[3]);
var zeroThroughNineTo2 = new DfaTransition(zeroThroughNine, states[2]);
var zeroThroughNineTo4 = new DfaTransition(zeroThroughNine, states[4]);
states[0].AddTransition(dotTo3);
states[0].AddTransition(plusOrMinusTo1);
states[0].AddTransition(zeroThroughNineTo2);
states[1].AddTransition(dotTo3);
states[1].AddTransition(zeroThroughNineTo2);
states[2].AddTransition(zeroThroughNineTo2);
states[2].AddTransition(dotTo3);
states[3].AddTransition(zeroThroughNineTo4);
states[4].AddTransition(zeroThroughNineTo4);
_start = states[0];
}
private static ILexerRule CreateNotDoubleQuoteLexerRule()
{
// ( [^"\\] | (\\ .) ) +
var start = new DfaState();
var escape = new DfaState();
var final = new DfaState(true);
var notQuoteTerminal = new NegationTerminal(
new SetTerminal('"', '\\'));
var escapeTerminal = new CharacterTerminal('\\');
var anyTerminal = new AnyTerminal();
var notQuoteEdge = new DfaTransition(notQuoteTerminal, final);
start.AddTransition(notQuoteEdge);
final.AddTransition(notQuoteEdge);
var escapeEdge = new DfaTransition(escapeTerminal, escape);
start.AddTransition(escapeEdge);
final.AddTransition(escapeEdge);
var anyEdge = new DfaTransition(anyTerminal, final);
escape.AddTransition(anyEdge);
return(new DfaLexerRule(start, new TokenType("not-double-quote")));
}
private static BaseLexerRule NotDoubleQuote()
{
// ([^"]|(\\.))*
var start = new DfaState();
var escape = new DfaState();
var final = new DfaState(true);
var notDoubleQuoteTerminal = new NegationTerminal(
new CharacterTerminal('"'));
var escapeTerminal = new CharacterTerminal('\\');
var anyTerminal = new AnyTerminal();
var notDoubleQuoteEdge = new DfaTransition(notDoubleQuoteTerminal, final);
start.AddTransition(notDoubleQuoteEdge);
final.AddTransition(notDoubleQuoteEdge);
var escapeEdge = new DfaTransition(escapeTerminal, escape);
start.AddTransition(escapeEdge);
final.AddTransition(escapeEdge);
var anyEdge = new DfaTransition(anyTerminal, final);
escape.AddTransition(anyEdge);
return(new DfaLexerRule(start, TokenTypes.NotDoubleQuote));
}
private static BaseLexerRule MultiLineComment()
{
var states = new DfaState[5];
for (int i = 0; i < states.Length; i++)
{
states[i] = new DfaState();
}
var slash = new CharacterTerminal('/');
var star = new CharacterTerminal('*');
var notStar = new NegationTerminal(star);
var notSlash = new NegationTerminal(slash);
var firstSlash = new DfaTransition(slash, states[1]);
var firstStar = new DfaTransition(star, states[2]);
var repeatNotStar = new DfaTransition(notStar, states[2]);
var lastStar = new DfaTransition(star, states[3]);
var goBackNotSlash = new DfaTransition(notSlash, states[2]);
var lastSlash = new DfaTransition(slash, states[4]);
states[0].AddTransition(firstSlash);
states[1].AddTransition(firstStar);
states[2].AddTransition(repeatNotStar);
states[2].AddTransition(lastStar);
states[3].AddTransition(goBackNotSlash);
states[3].AddTransition(lastSlash);
return(new DfaLexerRule(states[0], TokenTypes.MultiLineComment));
}
private DfaNode CreateDfaNode(Node node, DfaTransition transition, ExecutionPath predicatePath)
{
var tuple = (node, transition, predicatePath);
if (_dfaNodesDictionary.TryGetValue(tuple, out var dfaNode))
{
return(dfaNode);
}
return(_dfaNodesDictionary[tuple] = new DfaNode(node, transition, predicatePath));
}
private static BaseLexerRule CreateWhitespaceLexerRule()
{
var whitespaceTerminal = new WhitespaceTerminal();
var startState = new DfaState();
var finalState = new DfaState(true);
var whitespaceTransition = new DfaTransition(whitespaceTerminal, finalState);
startState.AddTransition(whitespaceTransition);
finalState.AddTransition(whitespaceTransition);
return(new DfaLexerRule(startState, new TokenType("[\\s]+")));
}
private static ILexerRule CreateWhitespaceRule()
{
var start = new DfaState();
var end = new DfaState(true);
var startToEnd = new DfaTransition(new WhitespaceTerminal(), end);
var endToEnd = new DfaTransition(new WhitespaceTerminal(), end);
start.AddTransition(startToEnd);
end.AddTransition(endToEnd);
return(new DfaLexerRule(start, new TokenType("whitespace")));
}
private static ILexerRule CreateNotSingleQuoteLexerRule()
{
var start = new DfaState();
var final = new DfaState(true);
var terminal = new NegationTerminal(new CharacterTerminal('\''));
var edge = new DfaTransition(terminal, final);
start.AddTransition(edge);
final.AddTransition(edge);
return(new DfaLexerRule(start, new TokenType("not-single-quote")));
}
static WordLexerRule()
{
_start = new DfaState();
var end = new DfaState(isFinal: true);
var transition = new DfaTransition(
new WordTerminal(),
end);
_start.AddTransition(transition);
end.AddTransition(transition);
}
private static ILexerRule Whitespace()
{
var start = new DfaState();
var end = new DfaState(isFinal: true);
var transition = new DfaTransition(
new WhitespaceTerminal(),
end);
start.AddTransition(transition);
end.AddTransition(transition);
return(new DfaLexerRule(start, "\\w+"));
}
private static BaseLexerRule NotSingleQuote()
{
// ([^']|(\\.))*
var start = new DfaState();
var final = new DfaState(true);
var terminal = new NegationTerminal(new CharacterTerminal('\''));
var edge = new DfaTransition(terminal, final);
start.AddTransition(edge);
final.AddTransition(edge);
return(new DfaLexerRule(start, TokenTypes.NotSingleQuote));
}
protected DfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode, DfaBuilder <TDfaState> builder)
{
Index = builder.Register((TDfaState)this);
Array = builder.FastLookup ? new TDfaState[ArrayLimit] : System.Array.Empty <TDfaState>();
Dictionary = new DfaDictionary <TDfaState>();
Builder = builder;
Nodes = nodes;
LazyTransitions = lazyTransitions;
HashCode = hashCode;
SuccessTransition = successTransition;
PrevSuccessTransition = prevSuccessTransition;
SuccessSubGraph = SuccessTransition?.SubGraph;
if (SuccessSubGraph != null)
{
SavePointer = true;
}
if (nodes.Length == 1 && SuccessTransition != null)
{
var dfaNode = Nodes[0];
var node = (Node)dfaNode.Node;
if ((node is ReturnStateNode || node.ReturnPath.IsInvalid) && node.ExecutionPaths.Length == 0)
{
Break = true;
}
}
Predicate = Nodes.FirstOrDefault(a => a.ExecutionPathObject != null)?.ExecutionPathObject;
if (Predicate != null)
{
TruePredicateArray = builder.FastLookup ? new TDfaState[ArrayLimit] : System.Array.Empty <TDfaState>();
TruePredicateDictionary = new Dictionary <int, TDfaState>();
FalsePredicateArray = builder.FastLookup ? new TDfaState[ArrayLimit] : System.Array.Empty <TDfaState>();
FalsePredicateDictionary = new Dictionary <int, TDfaState>();
}
// ReSharper disable once VirtualMemberCallInConstructor
NullState = CreateNullState();
if (SavePointer == false && Break == false && Predicate == null)
{
Continue = true;
}
}
private void AddNode(Node node, DfaTransition transition, ExecutionPath predicatePath)
{
if (_nodeSet.Add(node) == false)
{
return;
}
var dfaNode = _builder.CreateDfaNode(node, transition, predicatePath);
_nodesCollection.Add(dfaNode);
if (predicatePath == null && (node is ReturnStateNode || node.ReturnPath.IsInvalid == false))
{
_successTransition = transition;
}
}
private void Clear()
{
_stateHashCode = 0;
_successTransition = null;
_prevSuccessTransition = null;
_nodesCollection.Clear();
_nodeSet.Clear();
if (_lazyTransitionsCollection.Count > 0)
{
Array.Clear(_lazyTransitionsState, 0, _lazyTransitionsState.Length);
}
_lazyTransitionsCollection.Clear();
}
public TDfaState Build(int operand, TDfaState currentState)
{
_prevSuccessTransition = currentState.PrevSuccessTransition;
foreach (var lazyTransition in currentState.LazyTransitions)
{
AddLazyTransition(lazyTransition);
}
foreach (var dfaNode in currentState.Nodes)
{
var transition = dfaNode.Transition;
var lazyIndex = transition.LazyIndex;
if (ReferenceEquals(transition, _prevSuccessTransition) && _lazyTransitionsState[lazyIndex])
{
continue;
}
var passLazyNode = _lazyTransitionsState[lazyIndex];
var node = (Node)dfaNode.Node;
var executionPaths = operand == -1 ? node.GetExecutionPaths() : node.GetExecutionPaths(operand);
foreach (var executionPath in executionPaths)
{
AddNode(executionPath.Output, transition, executionPath.IsPredicate ? executionPath : null);
passLazyNode |= executionPath.PassLazyNode;
}
if (_lazyTransitionsState[lazyIndex] == passLazyNode)
{
continue;
}
AddLazyTransition(transition);
}
_prevSuccessTransition = _successTransition ?? _prevSuccessTransition;
return(BuildState() ?? currentState.NullState);
}
public void DfaLexerRuleShouldApplyToCharacterIfFirstStateHasTransition()
{
var states = new DfaState[2];
for (var i = 0; i < states.Length; i++)
{
states[i] = new DfaState(i == states.Length - 1);
}
var whitespaceToFinal = new DfaTransition(new WhitespaceTerminal(), states[1]);
states[0].AddTransition(whitespaceToFinal);
states[1].AddTransition(whitespaceToFinal);
var dfaLexerRule = new DfaLexerRule(states[0], new TokenType(@"\s+"));
Assert.IsTrue(dfaLexerRule.CanApply(' '));
Assert.IsTrue(dfaLexerRule.CanApply('\t'));
Assert.IsTrue(dfaLexerRule.CanApply('\r'));
Assert.IsFalse(dfaLexerRule.CanApply('a'));
}
static SingleQuoteStringLexerRule()
{
var states = new DfaState[3];
for (var i = 0; i < states.Length; i++)
{
states[i] = new DfaState(i == 2);
}
var quote = new CharacterTerminal('\'');
var notQuote = new NegationTerminal(quote);
var quoteToNotQuote = new DfaTransition(quote, states[1]);
var notQuoteToNotQuote = new DfaTransition(notQuote, states[1]);
var notQuoteToQuote = new DfaTransition(quote, states[2]);
states[0].AddTransition(quoteToNotQuote);
states[1].AddTransition(notQuoteToNotQuote);
states[1].AddTransition(notQuoteToQuote);
_start = states[0];
}
public TDfaState BuildPredicateState(int operand, TDfaState state, bool predicateResult)
{
_prevSuccessTransition = state.PrevSuccessTransition;
_successTransition = state.SuccessTransition;
foreach (var lazyTransition in state.LazyTransitions)
{
AddLazyTransition(lazyTransition);
}
var predicateFound = false;
foreach (var dfaNode in state.Nodes)
{
var node = (Node)dfaNode.Node;
var predicateExecutionPath = (ExecutionPath)dfaNode.ExecutionPathObject;
if (predicateExecutionPath == null || predicateFound)
{
AddNode(dfaNode);
continue;
}
predicateFound = true;
if (predicateResult == false)
{
continue;
}
var transition = dfaNode.Transition;
var lazyIndex = transition.LazyIndex;
if (ReferenceEquals(transition, _prevSuccessTransition) && _lazyTransitionsState[lazyIndex])
{
continue;
}
if (node.ReturnPath.IsInvalid == false)
{
AddNode(node.ReturnPath.Output, transition, null);
_successTransition = transition;
}
var passLazyNode = _lazyTransitionsState[lazyIndex];
var executionPaths = operand == -1 ? node.GetExecutionPaths() : node.GetExecutionPaths(operand);
foreach (var executionPath in executionPaths)
{
AddNode(executionPath.Output, transition, executionPath.IsPredicate ? executionPath : null);
passLazyNode |= executionPath.PassLazyNode;
}
if (_lazyTransitionsState[lazyIndex] == passLazyNode)
{
continue;
}
AddLazyTransition(transition);
}
_prevSuccessTransition = _successTransition ?? _prevSuccessTransition;
return(BuildState() ?? state.NullState);
}
protected abstract TDfaState CreateDfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode);
private void AddLazyTransition(DfaTransition transition)
{
_lazyTransitionsCollection.Add(transition);
_lazyTransitionsState[transition.LazyIndex] = true;
}
public LexerDfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode, LexerDfaBuilder builder, bool build = true,
LexerDfaState[] array = null) : base(nodes, lazyTransitions, successTransition,
prevSuccessTransition, hashCode, builder)
{
if (SuccessSubGraph == null)
{
return;
}
var lexerStateRule = ((LexerState)SuccessSubGraph.State).Rule;
Token = lexerStateRule.Token;
TokenCode = lexerStateRule.TokenCode;
Skip = lexerStateRule.Skip;
}
protected override LexerDfaState CreateDfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode)
{
return(new LexerDfaState(nodes, lazyTransitions, successTransition, prevSuccessTransition, hashCode, this));
}
public void SubsetConstructionAlgorithmShouldConvertComplexNfaToDfa()
{
var a = new CharacterTerminal('a');
var b = new CharacterTerminal('b');
var c = new CharacterTerminal('c');
var states = CreateStates(4);
states[0].AddTransition(a, states[1]);
states[0].AddTransition(c, states[3]);
states[1].AddEpsilon(states[0]);
states[1].AddTransition(b, states[2]);
states[2].AddTransition(a, states[1]);
states[3].AddTransition(c, states[2]);
states[3].AddEpsilon(states[2]);
var dfa_0 = ConvertNfaToDfa(new Nfa(states[0], states[2]));
Assert.IsNotNull(dfa_0);
DfaTransition transition_0_01 = null;
DfaTransition transition_0_23 = null;
var count = 0;
foreach (var transition in dfa_0.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_0_01 = transition;
}
else if (terminal.CanApply('c'))
{
transition_0_23 = transition;
}
count++;
}
Assert.AreEqual(2, count);
var dfa_01 = transition_0_01.Target;
DfaTransition transition_01_01 = null;
DfaTransition transition_01_23 = null;
DfaTransition transition_01_2 = null;
count = 0;
foreach (var transition in dfa_01.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_01_01 = transition;
}
else if (terminal.CanApply('b'))
{
transition_01_2 = transition;
}
else if (terminal.CanApply('c'))
{
transition_01_23 = transition;
}
count++;
}
Assert.AreEqual(3, count);
var dfa_23 = transition_0_23.Target;
DfaTransition transition_23_01 = null;
DfaTransition transition_23_2 = null;
count = 0;
foreach (var transition in dfa_23.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_23_01 = transition;
}
else if (terminal.CanApply('c'))
{
transition_23_2 = transition;
}
count++;
}
Assert.AreEqual(2, count);
var dfa_2 = transition_23_2.Target;
DfaTransition transition_2_01 = null;
count = 0;
foreach (var transition in dfa_2.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_2_01 = transition;
}
count++;
}
Assert.AreEqual(1, count);
}
public DfaNode(NodeBase node, DfaTransition transition)
{
Node = node;
Transition = transition;
}
public DfaNode(NodeBase node, DfaTransition transition, object executionPathObject)
{
Node = node;
Transition = transition;
ExecutionPathObject = executionPathObject;
}
