private static ILexerRule DoubleQuoteString()
{
var start = new DfaState();
var loop = new DfaState();
var esc = new DfaState();
var end = new DfaState(isFinal: true);
var doubleQuote = new CharacterTerminal('"');
var notDoubleQuote = new NegationTerminal(doubleQuote);
var escape = new CharacterTerminal('\\');
var any = new AnyTerminal();
// (start) - " -> (loop)
start.AddTransition(doubleQuote, loop);
// (loop) - [^"] -> (loop)
// - \ -> (esc)
// - " -> (end)
loop.AddTransition(notDoubleQuote, loop);
loop.AddTransition(escape, esc);
loop.AddTransition(doubleQuote, end);
// (esc) - . -> (loop)
esc.AddTransition(any, loop);
return(new DfaLexerRule(start, "double-quote-string"));
}
private static ILexerRule CreateNotDoubleQuoteLexerRule()
{
// ( [^"\\] | (\\ .) ) +
var start = new DfaState();
var escape = new DfaState();
var final = new DfaState(true);
var notQuoteTerminal = new NegationTerminal(
new SetTerminal('"', '\\'));
var escapeTerminal = new CharacterTerminal('\\');
var anyTerminal = new AnyTerminal();
var notQuoteEdge = new DfaTransition(notQuoteTerminal, final);
start.AddTransition(notQuoteEdge);
final.AddTransition(notQuoteEdge);
var escapeEdge = new DfaTransition(escapeTerminal, escape);
start.AddTransition(escapeEdge);
final.AddTransition(escapeEdge);
var anyEdge = new DfaTransition(anyTerminal, final);
escape.AddTransition(anyEdge);
return(new DfaLexerRule(start, new TokenType("not-double-quote")));
}
private static BaseLexerRule NotDoubleQuote()
{
// ([^"]|(\\.))*
var start = new DfaState();
var escape = new DfaState();
var final = new DfaState(true);
var notDoubleQuoteTerminal = new NegationTerminal(
new CharacterTerminal('"'));
var escapeTerminal = new CharacterTerminal('\\');
var anyTerminal = new AnyTerminal();
var notDoubleQuoteEdge = new DfaTransition(notDoubleQuoteTerminal, final);
start.AddTransition(notDoubleQuoteEdge);
final.AddTransition(notDoubleQuoteEdge);
var escapeEdge = new DfaTransition(escapeTerminal, escape);
start.AddTransition(escapeEdge);
final.AddTransition(escapeEdge);
var anyEdge = new DfaTransition(anyTerminal, final);
escape.AddTransition(anyEdge);
return(new DfaLexerRule(start, TokenTypes.NotDoubleQuote));
}
private static BaseLexerRule CreateMultiLineCommentLexerRule()
{
var pattern = @"\/[*]([*][^\/]|[^*])*[*][\/]";
var states = new DfaState[5];
for (int i = 0; i < states.Length; i++)
{
states[i] = new DfaState();
}
var slash = new CharacterTerminal('/');
var star = new CharacterTerminal('*');
var notStar = new NegationTerminal(star);
var notSlash = new NegationTerminal(slash);
var firstSlash = new DfaTransition(slash, states[1]);
var firstStar = new DfaTransition(star, states[2]);
var repeatNotStar = new DfaTransition(notStar, states[2]);
var lastStar = new DfaTransition(star, states[3]);
var goBackNotSlash = new DfaTransition(notSlash, states[2]);
var lastSlash = new DfaTransition(slash, states[4]);
states[0].AddTransition(firstSlash);
states[1].AddTransition(firstStar);
states[2].AddTransition(repeatNotStar);
states[2].AddTransition(lastStar);
states[3].AddTransition(goBackNotSlash);
states[3].AddTransition(lastSlash);
return(new DfaLexerRule(states[0], pattern));
}
private static BaseLexerRule MultiLineComment()
{
var states = new DfaState[5];
for (int i = 0; i < states.Length; i++)
{
states[i] = new DfaState();
}
var slash = new CharacterTerminal('/');
var star = new CharacterTerminal('*');
var notStar = new NegationTerminal(star);
var notSlash = new NegationTerminal(slash);
var firstSlash = new DfaTransition(slash, states[1]);
var firstStar = new DfaTransition(star, states[2]);
var repeatNotStar = new DfaTransition(notStar, states[2]);
var lastStar = new DfaTransition(star, states[3]);
var goBackNotSlash = new DfaTransition(notSlash, states[2]);
var lastSlash = new DfaTransition(slash, states[4]);
states[0].AddTransition(firstSlash);
states[1].AddTransition(firstStar);
states[2].AddTransition(repeatNotStar);
states[2].AddTransition(lastStar);
states[3].AddTransition(goBackNotSlash);
states[3].AddTransition(lastSlash);
return(new DfaLexerRule(states[0], TokenTypes.MultiLineComment));
}
private static LexerRule CreateMultiLineCommentLexerRule()
{
var states = new[]
{
DfaState.Inner(),
DfaState.Inner(),
DfaState.Inner(),
DfaState.Inner(),
DfaState.Final(),
};
var slash = new CharacterTerminal('/');
var star = new CharacterTerminal('*');
var notStar = new NegationTerminal(star);
var notSlash = new NegationTerminal(slash);
//var firstSlash = new DfaTransition(slash, states[1]);
//var firstStar = new DfaTransition(star, states[2]);
//var repeatNotStar = new DfaTransition(notStar, states[2]);
//var lastStar = new DfaTransition(star, states[3]);
//var goBackNotSlash = new DfaTransition(notSlash, states[2]);
//var lastSlash = new DfaTransition(slash, states[4]);
states[0].AddTransition(slash, states[1]);
states[1].AddTransition(star, states[2]);
states[2].AddTransition(notStar, states[2]);
states[2].AddTransition(star, states[3]);
states[3].AddTransition(notSlash, states[2]);
states[3].AddTransition(slash, states[4]);
return(new DfaLexerRule(states[0], new TokenName(@"\/[*]([*][^\/]|[^*])*[*][\/]")));
}
public void CharacterTerminalShouldReturnSingleInterval()
{
var characterTerminal = new CharacterTerminal('a');
var intervals = characterTerminal.GetIntervals();
Assert.AreEqual(1, intervals.Count);
Assert.AreEqual('a', intervals[0].Min);
Assert.AreEqual('a', intervals[0].Max);
}
private static INfa Character(RegexCharacter character)
{
var start = new NfaState();
var end = new NfaState();
var terminal = new CharacterTerminal(character.Value);
var transition = new TerminalNfaTransition(
terminal: terminal,
target: end);
start.AddTransistion(transition);
return(new Nfa(start, end));
}
public void SubsetConstructionAlgorithmShouldConvertSingleCharacterNfaToEquivalentDfa()
{
var a = new CharacterTerminal('a');
var states = CreateStates(2);
states[0].AddTransition(a, states[1]);
var nfa = new Nfa(states[0], states[1]);
var dfa = ConvertNfaToDfa(nfa);
Assert.IsNotNull(dfa);
}
private static AtomTerminal CreateTerminalForCharacter(char value, bool isEscaped, bool negate)
{
AtomTerminal terminal;
if (!isEscaped)
{
terminal = new CharacterTerminal(value);
}
else
{
switch (value)
{
case 's':
terminal = WhitespaceTerminal.Instance;
break;
case 'd':
terminal = DigitTerminal.Instance;
break;
case 'w':
terminal = WordTerminal.Instance;
break;
case 'D':
terminal = DigitTerminal.Instance;
negate = !negate;
break;
case 'S':
terminal = WhitespaceTerminal.Instance;
negate = !negate;
break;
case 'W':
terminal = WordTerminal.Instance;
negate = !negate;
break;
default:
terminal = new CharacterTerminal(value);
break;
}
}
if (negate)
{
terminal = new NegationTerminal(terminal);
}
return(terminal);
}
public Terminal this[int index]
{
get
{
Terminal result = engine.TerminalsCache[data[index]];
if (result == null)
{
result = new CharacterTerminal(data[index]);
engine.TerminalsCache[data[index]] = result;
}
return(result);
}
}
static SimpleDoubleQuoteStringLexerRule()
{
var start = DfaState.Inner();
var inner = DfaState.Inner();
var final = DfaState.Final();
var quote = new CharacterTerminal('"');
var notQuote = new NegationTerminal(quote);
start.AddTransition(quote, inner);
inner.AddTransition(notQuote, inner);
inner.AddTransition(quote, final);
enter = start;
}
private static LexerRule CreateSingleLineComment()
{
var slash = new CharacterTerminal('/');
var newLine = new CharacterTerminal('\n');
var notNewLine = new NegationTerminal(newLine);
var start = DfaState.Inner();
var oneSlash = DfaState.Inner();
var twoSlash = DfaState.Final();
start.AddTransition(slash, oneSlash);
oneSlash.AddTransition(slash, twoSlash);
twoSlash.AddTransition(notNewLine, twoSlash);
return(new DfaLexerRule(start, TokenClasses.SingleLineComment));
}
private static LexerRule CreateNotDoubleQuoteLexerRule()
{
// ( [^"\\] | (\\ .) ) +
var start = DfaState.Inner();
var escape = DfaState.Inner();
var final = DfaState.Final();
var notQuoteTerminal = new NegationTerminal(new SetTerminal("\"\\"));
var escapeTerminal = new CharacterTerminal('\\');
start.AddTransition(notQuoteTerminal, final)
.AddTransition(escapeTerminal, escape);
final.AddTransition(notQuoteTerminal, final)
.AddTransition(escapeTerminal, escape);
escape.AddTransition(AnyTerminal.Instance, final);
return(new DfaLexerRule(start, "not-double-quote"));
}
/// <summary>
/// Creates a terminal lexer rule. The grammar is defined as:
/// <code>
/// terminal ~
/// "'" character {character } "'"
/// | '"' character {character } '"';
/// </code>
/// </summary>
/// <returns></returns>
private static BaseLexerRule Terminal()
{
var start = new DfaState();
var doubleQuoteString = new DfaState();
var singleQuoteString = new DfaState();
var end = new DfaState(isFinal: true);
var doubleQuote = new CharacterTerminal('"');
var singleQuote = new CharacterTerminal('\'');
start.AddTransition(doubleQuote, doubleQuoteString);
start.AddTransition(singleQuote, singleQuoteString);
doubleQuoteString.AddTransition(new NegationTerminal(doubleQuote), doubleQuoteString);
doubleQuoteString.AddTransition(doubleQuote, end);
singleQuoteString.AddTransition(new NegationTerminal(singleQuote), singleQuoteString);
singleQuoteString.AddTransition(singleQuote, end);
return(new DfaLexerRule(start, "terminal"));
}
static SingleQuoteStringLexerRule()
{
var states = new DfaState[3];
for (var i = 0; i < states.Length; i++)
{
states[i] = new DfaState(i == 2);
}
var quote = new CharacterTerminal('\'');
var notQuote = new NegationTerminal(quote);
var quoteToNotQuote = new DfaTransition(quote, states[1]);
var notQuoteToNotQuote = new DfaTransition(notQuote, states[1]);
var notQuoteToQuote = new DfaTransition(quote, states[2]);
states[0].AddTransition(quoteToNotQuote);
states[1].AddTransition(notQuoteToNotQuote);
states[1].AddTransition(notQuoteToQuote);
_start = states[0];
}
private static LexerRule CreateMultiLineCommentLexerRule()
{
var states = new DfaState[5];
for (var i = 0; i < states.Length; i++)
{
states[i] = DfaState.Inner();
}
var slash = new CharacterTerminal('/');
var star = new CharacterTerminal('*');
var notStar = new NegationTerminal(star);
var notSlash = new NegationTerminal(slash);
states[0].AddTransition(slash, states[1]);
states[1].AddTransition(star, states[2]);
states[2].AddTransition(notStar, states[2]);
states[2].AddTransition(star, states[3]);
states[3].AddTransition(notSlash, states[2]);
states[3].AddTransition(slash, states[4]);
return(new DfaLexerRule(states[0], TokenClasses.MultiLineComment));
}
public void Visit(CharacterTerminal expression, Nonterminal context)
{
Add(context, expression, expression);
}
public override void Visit(CharacterTerminal expression, object context)
{
expressions.Add(expression);
base.Visit(expression, context);
}
protected override Int32 VisitCharacterTerminal(CharacterTerminal characterTerminal, Unit argument) => HashCode.Combine(characterTerminal.Value);
public void CharacterTerminalShouldMatchSingleCharacter()
{
var characterTerminal = new CharacterTerminal('c');
Assert.IsTrue(characterTerminal.CanApply('c'));
}
private static INfa Character(RegexCharacterClassCharacter character, bool negate)
{
var start = new NfaState();
var end = new NfaState();
ITerminal terminal = null;
if (!character.IsEscaped)
{
terminal = new CharacterTerminal(character.Value);
}
else
{
switch (character.Value)
{
case 's':
terminal = new WhitespaceTerminal();
break;
case 'd':
terminal = new DigitTerminal();
break;
case 'w':
terminal = new WordTerminal();
break;
case 'D':
terminal = new DigitTerminal();
negate = !negate;
break;
case 'S':
terminal = new WhitespaceTerminal();
negate = !negate;
break;
case 'W':
terminal = new WordTerminal();
negate = !negate;
break;
default:
terminal = new CharacterTerminal(character.Value);
break;
}
}
if (negate)
{
terminal = new NegationTerminal(terminal);
}
var transition = new TerminalNfaTransition(
terminal: terminal,
target: end);
start.AddTransistion(transition);
return(new Nfa(start, end));
}
public virtual void Visit(CharacterTerminal expression, T context)
{
}
public void SubsetConstructionAlgorithmShouldConvertComplexNfaToDfa()
{
var a = new CharacterTerminal('a');
var b = new CharacterTerminal('b');
var c = new CharacterTerminal('c');
var states = CreateStates(4);
states[0].AddTransition(a, states[1]);
states[0].AddTransition(c, states[3]);
states[1].AddEpsilon(states[0]);
states[1].AddTransition(b, states[2]);
states[2].AddTransition(a, states[1]);
states[3].AddTransition(c, states[2]);
states[3].AddEpsilon(states[2]);
var dfa_0 = ConvertNfaToDfa(new Nfa(states[0], states[2]));
Assert.IsNotNull(dfa_0);
DfaTransition transition_0_01 = null;
DfaTransition transition_0_23 = null;
var count = 0;
foreach (var transition in dfa_0.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_0_01 = transition;
}
else if (terminal.CanApply('c'))
{
transition_0_23 = transition;
}
count++;
}
Assert.AreEqual(2, count);
var dfa_01 = transition_0_01.Target;
DfaTransition transition_01_01 = null;
DfaTransition transition_01_23 = null;
DfaTransition transition_01_2 = null;
count = 0;
foreach (var transition in dfa_01.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_01_01 = transition;
}
else if (terminal.CanApply('b'))
{
transition_01_2 = transition;
}
else if (terminal.CanApply('c'))
{
transition_01_23 = transition;
}
count++;
}
Assert.AreEqual(3, count);
var dfa_23 = transition_0_23.Target;
DfaTransition transition_23_01 = null;
DfaTransition transition_23_2 = null;
count = 0;
foreach (var transition in dfa_23.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_23_01 = transition;
}
else if (terminal.CanApply('c'))
{
transition_23_2 = transition;
}
count++;
}
Assert.AreEqual(2, count);
var dfa_2 = transition_23_2.Target;
DfaTransition transition_2_01 = null;
count = 0;
foreach (var transition in dfa_2.Transitions)
{
var terminal = transition.Terminal;
if (terminal.CanApply('a'))
{
transition_2_01 = transition;
}
count++;
}
Assert.AreEqual(1, count);
}
