private static BaseLexerRule CreateMultiLineCommentLexerRule() { var pattern = @"\/[*]([*][^\/]|[^*])*[*][\/]"; var states = new DfaState[5]; for (int i = 0; i < states.Length; i++) { states[i] = new DfaState(); } var slash = new CharacterTerminal('/'); var star = new CharacterTerminal('*'); var notStar = new NegationTerminal(star); var notSlash = new NegationTerminal(slash); var firstSlash = new DfaTransition(slash, states[1]); var firstStar = new DfaTransition(star, states[2]); var repeatNotStar = new DfaTransition(notStar, states[2]); var lastStar = new DfaTransition(star, states[3]); var goBackNotSlash = new DfaTransition(notSlash, states[2]); var lastSlash = new DfaTransition(slash, states[4]); states[0].AddTransition(firstSlash); states[1].AddTransition(firstStar); states[2].AddTransition(repeatNotStar); states[2].AddTransition(lastStar); states[3].AddTransition(goBackNotSlash); states[3].AddTransition(lastSlash); return(new DfaLexerRule(states[0], pattern)); }
static NumberLexerRule() { var states = new DfaState[5]; for (int i = 0; i < states.Length; i++) { states[i] = new DfaState(i == 4 || i == 2); } var zeroThroughNine = new RangeTerminal('0', '9'); var plusOrMinusTo1 = new DfaTransition(new SetTerminal('+', '-'), states[1]); var dotTo3 = new DfaTransition(new CharacterTerminal('.'), states[3]); var zeroThroughNineTo2 = new DfaTransition(zeroThroughNine, states[2]); var zeroThroughNineTo4 = new DfaTransition(zeroThroughNine, states[4]); states[0].AddTransition(dotTo3); states[0].AddTransition(plusOrMinusTo1); states[0].AddTransition(zeroThroughNineTo2); states[1].AddTransition(dotTo3); states[1].AddTransition(zeroThroughNineTo2); states[2].AddTransition(zeroThroughNineTo2); states[2].AddTransition(dotTo3); states[3].AddTransition(zeroThroughNineTo4); states[4].AddTransition(zeroThroughNineTo4); _start = states[0]; }
private static ILexerRule CreateNotDoubleQuoteLexerRule() { // ( [^"\\] | (\\ .) ) + var start = new DfaState(); var escape = new DfaState(); var final = new DfaState(true); var notQuoteTerminal = new NegationTerminal( new SetTerminal('"', '\\')); var escapeTerminal = new CharacterTerminal('\\'); var anyTerminal = new AnyTerminal(); var notQuoteEdge = new DfaTransition(notQuoteTerminal, final); start.AddTransition(notQuoteEdge); final.AddTransition(notQuoteEdge); var escapeEdge = new DfaTransition(escapeTerminal, escape); start.AddTransition(escapeEdge); final.AddTransition(escapeEdge); var anyEdge = new DfaTransition(anyTerminal, final); escape.AddTransition(anyEdge); return(new DfaLexerRule(start, new TokenType("not-double-quote"))); }
private static BaseLexerRule NotDoubleQuote() { // ([^"]|(\\.))* var start = new DfaState(); var escape = new DfaState(); var final = new DfaState(true); var notDoubleQuoteTerminal = new NegationTerminal( new CharacterTerminal('"')); var escapeTerminal = new CharacterTerminal('\\'); var anyTerminal = new AnyTerminal(); var notDoubleQuoteEdge = new DfaTransition(notDoubleQuoteTerminal, final); start.AddTransition(notDoubleQuoteEdge); final.AddTransition(notDoubleQuoteEdge); var escapeEdge = new DfaTransition(escapeTerminal, escape); start.AddTransition(escapeEdge); final.AddTransition(escapeEdge); var anyEdge = new DfaTransition(anyTerminal, final); escape.AddTransition(anyEdge); return(new DfaLexerRule(start, TokenTypes.NotDoubleQuote)); }
private static BaseLexerRule MultiLineComment() { var states = new DfaState[5]; for (int i = 0; i < states.Length; i++) { states[i] = new DfaState(); } var slash = new CharacterTerminal('/'); var star = new CharacterTerminal('*'); var notStar = new NegationTerminal(star); var notSlash = new NegationTerminal(slash); var firstSlash = new DfaTransition(slash, states[1]); var firstStar = new DfaTransition(star, states[2]); var repeatNotStar = new DfaTransition(notStar, states[2]); var lastStar = new DfaTransition(star, states[3]); var goBackNotSlash = new DfaTransition(notSlash, states[2]); var lastSlash = new DfaTransition(slash, states[4]); states[0].AddTransition(firstSlash); states[1].AddTransition(firstStar); states[2].AddTransition(repeatNotStar); states[2].AddTransition(lastStar); states[3].AddTransition(goBackNotSlash); states[3].AddTransition(lastSlash); return(new DfaLexerRule(states[0], TokenTypes.MultiLineComment)); }
private DfaNode CreateDfaNode(Node node, DfaTransition transition, ExecutionPath predicatePath) { var tuple = (node, transition, predicatePath); if (_dfaNodesDictionary.TryGetValue(tuple, out var dfaNode)) { return(dfaNode); } return(_dfaNodesDictionary[tuple] = new DfaNode(node, transition, predicatePath)); }
private static BaseLexerRule CreateWhitespaceLexerRule() { var whitespaceTerminal = new WhitespaceTerminal(); var startState = new DfaState(); var finalState = new DfaState(true); var whitespaceTransition = new DfaTransition(whitespaceTerminal, finalState); startState.AddTransition(whitespaceTransition); finalState.AddTransition(whitespaceTransition); return(new DfaLexerRule(startState, new TokenType("[\\s]+"))); }
private static ILexerRule CreateWhitespaceRule() { var start = new DfaState(); var end = new DfaState(true); var startToEnd = new DfaTransition(new WhitespaceTerminal(), end); var endToEnd = new DfaTransition(new WhitespaceTerminal(), end); start.AddTransition(startToEnd); end.AddTransition(endToEnd); return(new DfaLexerRule(start, new TokenType("whitespace"))); }
private static ILexerRule CreateNotSingleQuoteLexerRule() { var start = new DfaState(); var final = new DfaState(true); var terminal = new NegationTerminal(new CharacterTerminal('\'')); var edge = new DfaTransition(terminal, final); start.AddTransition(edge); final.AddTransition(edge); return(new DfaLexerRule(start, new TokenType("not-single-quote"))); }
static WordLexerRule() { _start = new DfaState(); var end = new DfaState(isFinal: true); var transition = new DfaTransition( new WordTerminal(), end); _start.AddTransition(transition); end.AddTransition(transition); }
private static ILexerRule Whitespace() { var start = new DfaState(); var end = new DfaState(isFinal: true); var transition = new DfaTransition( new WhitespaceTerminal(), end); start.AddTransition(transition); end.AddTransition(transition); return(new DfaLexerRule(start, "\\w+")); }
private static BaseLexerRule NotSingleQuote() { // ([^']|(\\.))* var start = new DfaState(); var final = new DfaState(true); var terminal = new NegationTerminal(new CharacterTerminal('\'')); var edge = new DfaTransition(terminal, final); start.AddTransition(edge); final.AddTransition(edge); return(new DfaLexerRule(start, TokenTypes.NotSingleQuote)); }
protected DfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode, DfaBuilder <TDfaState> builder) { Index = builder.Register((TDfaState)this); Array = builder.FastLookup ? new TDfaState[ArrayLimit] : System.Array.Empty <TDfaState>(); Dictionary = new DfaDictionary <TDfaState>(); Builder = builder; Nodes = nodes; LazyTransitions = lazyTransitions; HashCode = hashCode; SuccessTransition = successTransition; PrevSuccessTransition = prevSuccessTransition; SuccessSubGraph = SuccessTransition?.SubGraph; if (SuccessSubGraph != null) { SavePointer = true; } if (nodes.Length == 1 && SuccessTransition != null) { var dfaNode = Nodes[0]; var node = (Node)dfaNode.Node; if ((node is ReturnStateNode || node.ReturnPath.IsInvalid) && node.ExecutionPaths.Length == 0) { Break = true; } } Predicate = Nodes.FirstOrDefault(a => a.ExecutionPathObject != null)?.ExecutionPathObject; if (Predicate != null) { TruePredicateArray = builder.FastLookup ? new TDfaState[ArrayLimit] : System.Array.Empty <TDfaState>(); TruePredicateDictionary = new Dictionary <int, TDfaState>(); FalsePredicateArray = builder.FastLookup ? new TDfaState[ArrayLimit] : System.Array.Empty <TDfaState>(); FalsePredicateDictionary = new Dictionary <int, TDfaState>(); } // ReSharper disable once VirtualMemberCallInConstructor NullState = CreateNullState(); if (SavePointer == false && Break == false && Predicate == null) { Continue = true; } }
private void AddNode(Node node, DfaTransition transition, ExecutionPath predicatePath) { if (_nodeSet.Add(node) == false) { return; } var dfaNode = _builder.CreateDfaNode(node, transition, predicatePath); _nodesCollection.Add(dfaNode); if (predicatePath == null && (node is ReturnStateNode || node.ReturnPath.IsInvalid == false)) { _successTransition = transition; } }
private void Clear() { _stateHashCode = 0; _successTransition = null; _prevSuccessTransition = null; _nodesCollection.Clear(); _nodeSet.Clear(); if (_lazyTransitionsCollection.Count > 0) { Array.Clear(_lazyTransitionsState, 0, _lazyTransitionsState.Length); } _lazyTransitionsCollection.Clear(); }
public TDfaState Build(int operand, TDfaState currentState) { _prevSuccessTransition = currentState.PrevSuccessTransition; foreach (var lazyTransition in currentState.LazyTransitions) { AddLazyTransition(lazyTransition); } foreach (var dfaNode in currentState.Nodes) { var transition = dfaNode.Transition; var lazyIndex = transition.LazyIndex; if (ReferenceEquals(transition, _prevSuccessTransition) && _lazyTransitionsState[lazyIndex]) { continue; } var passLazyNode = _lazyTransitionsState[lazyIndex]; var node = (Node)dfaNode.Node; var executionPaths = operand == -1 ? node.GetExecutionPaths() : node.GetExecutionPaths(operand); foreach (var executionPath in executionPaths) { AddNode(executionPath.Output, transition, executionPath.IsPredicate ? executionPath : null); passLazyNode |= executionPath.PassLazyNode; } if (_lazyTransitionsState[lazyIndex] == passLazyNode) { continue; } AddLazyTransition(transition); } _prevSuccessTransition = _successTransition ?? _prevSuccessTransition; return(BuildState() ?? currentState.NullState); }
public void DfaLexerRuleShouldApplyToCharacterIfFirstStateHasTransition() { var states = new DfaState[2]; for (var i = 0; i < states.Length; i++) { states[i] = new DfaState(i == states.Length - 1); } var whitespaceToFinal = new DfaTransition(new WhitespaceTerminal(), states[1]); states[0].AddTransition(whitespaceToFinal); states[1].AddTransition(whitespaceToFinal); var dfaLexerRule = new DfaLexerRule(states[0], new TokenType(@"\s+")); Assert.IsTrue(dfaLexerRule.CanApply(' ')); Assert.IsTrue(dfaLexerRule.CanApply('\t')); Assert.IsTrue(dfaLexerRule.CanApply('\r')); Assert.IsFalse(dfaLexerRule.CanApply('a')); }
static SingleQuoteStringLexerRule() { var states = new DfaState[3]; for (var i = 0; i < states.Length; i++) { states[i] = new DfaState(i == 2); } var quote = new CharacterTerminal('\''); var notQuote = new NegationTerminal(quote); var quoteToNotQuote = new DfaTransition(quote, states[1]); var notQuoteToNotQuote = new DfaTransition(notQuote, states[1]); var notQuoteToQuote = new DfaTransition(quote, states[2]); states[0].AddTransition(quoteToNotQuote); states[1].AddTransition(notQuoteToNotQuote); states[1].AddTransition(notQuoteToQuote); _start = states[0]; }
public TDfaState BuildPredicateState(int operand, TDfaState state, bool predicateResult) { _prevSuccessTransition = state.PrevSuccessTransition; _successTransition = state.SuccessTransition; foreach (var lazyTransition in state.LazyTransitions) { AddLazyTransition(lazyTransition); } var predicateFound = false; foreach (var dfaNode in state.Nodes) { var node = (Node)dfaNode.Node; var predicateExecutionPath = (ExecutionPath)dfaNode.ExecutionPathObject; if (predicateExecutionPath == null || predicateFound) { AddNode(dfaNode); continue; } predicateFound = true; if (predicateResult == false) { continue; } var transition = dfaNode.Transition; var lazyIndex = transition.LazyIndex; if (ReferenceEquals(transition, _prevSuccessTransition) && _lazyTransitionsState[lazyIndex]) { continue; } if (node.ReturnPath.IsInvalid == false) { AddNode(node.ReturnPath.Output, transition, null); _successTransition = transition; } var passLazyNode = _lazyTransitionsState[lazyIndex]; var executionPaths = operand == -1 ? node.GetExecutionPaths() : node.GetExecutionPaths(operand); foreach (var executionPath in executionPaths) { AddNode(executionPath.Output, transition, executionPath.IsPredicate ? executionPath : null); passLazyNode |= executionPath.PassLazyNode; } if (_lazyTransitionsState[lazyIndex] == passLazyNode) { continue; } AddLazyTransition(transition); } _prevSuccessTransition = _successTransition ?? _prevSuccessTransition; return(BuildState() ?? state.NullState); }
protected abstract TDfaState CreateDfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode);
private void AddLazyTransition(DfaTransition transition) { _lazyTransitionsCollection.Add(transition); _lazyTransitionsState[transition.LazyIndex] = true; }
public LexerDfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode, LexerDfaBuilder builder, bool build = true, LexerDfaState[] array = null) : base(nodes, lazyTransitions, successTransition, prevSuccessTransition, hashCode, builder) { if (SuccessSubGraph == null) { return; } var lexerStateRule = ((LexerState)SuccessSubGraph.State).Rule; Token = lexerStateRule.Token; TokenCode = lexerStateRule.TokenCode; Skip = lexerStateRule.Skip; }
protected override LexerDfaState CreateDfaState(DfaNode[] nodes, DfaTransition[] lazyTransitions, DfaTransition successTransition, DfaTransition prevSuccessTransition, int hashCode) { return(new LexerDfaState(nodes, lazyTransitions, successTransition, prevSuccessTransition, hashCode, this)); }
public void SubsetConstructionAlgorithmShouldConvertComplexNfaToDfa() { var a = new CharacterTerminal('a'); var b = new CharacterTerminal('b'); var c = new CharacterTerminal('c'); var states = CreateStates(4); states[0].AddTransition(a, states[1]); states[0].AddTransition(c, states[3]); states[1].AddEpsilon(states[0]); states[1].AddTransition(b, states[2]); states[2].AddTransition(a, states[1]); states[3].AddTransition(c, states[2]); states[3].AddEpsilon(states[2]); var dfa_0 = ConvertNfaToDfa(new Nfa(states[0], states[2])); Assert.IsNotNull(dfa_0); DfaTransition transition_0_01 = null; DfaTransition transition_0_23 = null; var count = 0; foreach (var transition in dfa_0.Transitions) { var terminal = transition.Terminal; if (terminal.CanApply('a')) { transition_0_01 = transition; } else if (terminal.CanApply('c')) { transition_0_23 = transition; } count++; } Assert.AreEqual(2, count); var dfa_01 = transition_0_01.Target; DfaTransition transition_01_01 = null; DfaTransition transition_01_23 = null; DfaTransition transition_01_2 = null; count = 0; foreach (var transition in dfa_01.Transitions) { var terminal = transition.Terminal; if (terminal.CanApply('a')) { transition_01_01 = transition; } else if (terminal.CanApply('b')) { transition_01_2 = transition; } else if (terminal.CanApply('c')) { transition_01_23 = transition; } count++; } Assert.AreEqual(3, count); var dfa_23 = transition_0_23.Target; DfaTransition transition_23_01 = null; DfaTransition transition_23_2 = null; count = 0; foreach (var transition in dfa_23.Transitions) { var terminal = transition.Terminal; if (terminal.CanApply('a')) { transition_23_01 = transition; } else if (terminal.CanApply('c')) { transition_23_2 = transition; } count++; } Assert.AreEqual(2, count); var dfa_2 = transition_23_2.Target; DfaTransition transition_2_01 = null; count = 0; foreach (var transition in dfa_2.Transitions) { var terminal = transition.Terminal; if (terminal.CanApply('a')) { transition_2_01 = transition; } count++; } Assert.AreEqual(1, count); }
public DfaNode(NodeBase node, DfaTransition transition) { Node = node; Transition = transition; }
public DfaNode(NodeBase node, DfaTransition transition, object executionPathObject) { Node = node; Transition = transition; ExecutionPathObject = executionPathObject; }