public NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum> CreateLexer()
{
var symbols_rep = StringRep.Create(Tuple.Create(SymbolEnum.Error, "Error"),
Tuple.Create(SymbolEnum.EOF, "EOF"),
Tuple.Create(SymbolEnum.LPAREN, "LPAREN"),
Tuple.Create(SymbolEnum.RPAREN, "RPAREN"),
Tuple.Create(SymbolEnum.NUM, "NUM"),
Tuple.Create(SymbolEnum.ATOM, "ATOM"),
Tuple.Create(SymbolEnum.comp, "comp"),
Tuple.Create(SymbolEnum.elem, "elem"),
Tuple.Create(SymbolEnum.__list___merged_elem_e____, "__list___merged_elem_e____"));
var states_rep = StringRep.Create(Tuple.Create(StateEnum.INIT, "INIT"));
var mre = new DfaTable(ConnectionTable.Create(40, new int [] { 1, 4, -1, -1, -1, -1, -1, -1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -1, -1, -1, -1, -1, -1, -1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }), //0
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(0, false)), //1
ConnectionTable.Create(97, new int [] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }, Tuple.Create(3, false)), //2
ConnectionTable.Create(48, new int [] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }, Tuple.Create(2, false)), //3
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(1, false)), //4
ConnectionTable.Create(97, new int [] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }, Tuple.Create(3, false))); //5
var lexer = new NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum>(symbols_rep, states_rep, StateEnum.INIT, SymbolEnum.EOF, SymbolEnum.Error, mre);
lexer.AddIdRule(0, "\"" + "(" + "\"", StringCaseComparison.Sensitive, SymbolEnum.LPAREN, StateEnum.INIT);
lexer.AddIdRule(1, "\"" + ")" + "\"", StringCaseComparison.Sensitive, SymbolEnum.RPAREN, StateEnum.INIT);
lexer.AddIdAction(2, "/" + @"[0-9]+" + "/", StringCaseComparison.Sensitive, match => { match.Token = SymbolEnum.NUM;
match.Value = Convert.ToInt32(match.Text); }, StateEnum.INIT);
lexer.AddIdRule(3, "/" + @"[A-Z][a-z]*" + "/", StringCaseComparison.Sensitive, SymbolEnum.ATOM, StateEnum.INIT);
lexer.EofAction = (TokenMatch <SymbolEnum> match) => match.Token = SymbolEnum.EOF;
return(lexer);
}
public static Productions <SYMBOL_ENUM, TREE_NODE> Create(
StringRep <SYMBOL_ENUM> symbolsRep,
IEnumerable <Production <SYMBOL_ENUM, TREE_NODE> > productions,
SYMBOL_ENUM eofSymbol,
SYMBOL_ENUM syntaxErrorSymbol,
GrammarReport <SYMBOL_ENUM, TREE_NODE> report)
{
try
{
if (ExperimentsSettings.UnfoldingAliases_EXPLOSION)
{
productions = unfoldIdentityProductions(symbolsRep, productions, eofSymbol, syntaxErrorSymbol);
}
if (ExperimentsSettings.UnfoldErrorProductions_NOT_USED)
{
productions = unfoldErrorProductions_NOT_USED(symbolsRep, productions, eofSymbol, syntaxErrorSymbol);
}
var result = new Productions <SYMBOL_ENUM, TREE_NODE>(symbolsRep, productions, eofSymbol, syntaxErrorSymbol, s => report.AddWarning(s));
return(result);
}
catch (Exception ex)
{
report.AddError(new GrammarError(ex.Message));
return(null);
}
}
public Lexer(StringRep <SYMBOL_ENUM> symbolsRep, StringRep <STATE_ENUM> statesRep, STATE_ENUM initialState, SYMBOL_ENUM eofToken,
SYMBOL_ENUM errorToken,
IPatternMatcher patternMatcher = null)
{
this.patternMatcher = patternMatcher ?? new PatternManager();
this.StatesRep = statesRep;
this.SymbolsRep = symbolsRep;
this.initialState = initialState;
this.ErrorToken = errorToken;
this.EofAction = match =>
{
if (!IsValidEofState)
{
match.Value = "Invalid state at EOF";
match.Token = ErrorToken;
}
else
{
match.Token = eofToken;
}
};
this.rules = new List <Rule <SYMBOL_ENUM, STATE_ENUM> >();
history = new List <MatchInfo>();
DefaultStates = new STATE_ENUM[] { initialState };
LineNumberInit = 1;
ColumnNumberInit = 1;
}
public NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum> CreateLexer()
{
var symbols_rep = StringRep.Create(Tuple.Create(SymbolEnum.Error, "Error"),
Tuple.Create(SymbolEnum.EOF, "EOF"),
Tuple.Create(SymbolEnum.PLUS, "PLUS"),
Tuple.Create(SymbolEnum.MINUS, "MINUS"),
Tuple.Create(SymbolEnum.LANGLE, "LANGLE"),
Tuple.Create(SymbolEnum.RANGLE, "RANGLE"),
Tuple.Create(SymbolEnum.comp, "comp"),
Tuple.Create(SymbolEnum.expr, "expr"),
Tuple.Create(SymbolEnum.NUM, "NUM"));
var states_rep = StringRep.Create(Tuple.Create(StateEnum.INIT, "INIT"));
var mre = new DfaTable(ConnectionTable.Create(32, new int [] { 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 6, -1, -1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -1, -1, 5, -1, 4 }), //0
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(0, false)), //1
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(5, false)), //2
ConnectionTable.Create(48, new int [] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }, Tuple.Create(4, false)), //3
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(3, false)), //4
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(2, false)), //5
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(1, false))); //6
var lexer = new NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum>(symbols_rep, states_rep, StateEnum.INIT, SymbolEnum.EOF, SymbolEnum.Error, mre);
lexer.AddIdRule(0, "\"" + "+" + "\"", StringCaseComparison.Sensitive, SymbolEnum.PLUS, StateEnum.INIT);
lexer.AddIdRule(1, "\"" + "-" + "\"", StringCaseComparison.Sensitive, SymbolEnum.MINUS, StateEnum.INIT);
lexer.AddIdRule(2, "\"" + "<" + "\"", StringCaseComparison.Sensitive, SymbolEnum.LANGLE, StateEnum.INIT);
lexer.AddIdRule(3, "\"" + ">" + "\"", StringCaseComparison.Sensitive, SymbolEnum.RANGLE, StateEnum.INIT);
lexer.AddIdAction(4, "/" + @"[0-9]+" + "/", StringCaseComparison.Sensitive, match => { match.Token = SymbolEnum.NUM;
match.Value = Convert.ToInt32(match.Text); }, StateEnum.INIT);
lexer.AddIdAction(5, "\"" + " " + "\"", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.EofAction = (TokenMatch <SymbolEnum> match) => match.Token = SymbolEnum.EOF;
return(lexer);
}
public void InitSymbolMapping()
{
this.symbolsMapping = symbolRegistry.ZipWithIndex().ToDictionary();
this.invSymbolsMapping = this.symbolsMapping.ToDictionary(it => it.Value, it => it.Key);
this.SymbolsRep = StringRep.Create(invSymbolsMapping);
}
public NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum> CreateLexer()
{
var symbols_rep = StringRep.Create(Tuple.Create(SymbolEnum.Error, "Error"),
Tuple.Create(SymbolEnum.EOF, "EOF"),
Tuple.Create(SymbolEnum.LANGLE, "LANGLE"),
Tuple.Create(SymbolEnum.RANGLE, "RANGLE"),
Tuple.Create(SymbolEnum.LBRACKET, "LBRACKET"),
Tuple.Create(SymbolEnum.RBRACKET, "RBRACKET"),
Tuple.Create(SymbolEnum.DOT, "DOT"),
Tuple.Create(SymbolEnum.COMMA, "COMMA"),
Tuple.Create(SymbolEnum.NEW, "NEW"),
Tuple.Create(SymbolEnum.IDENTIFIER, "IDENTIFIER"),
Tuple.Create(SymbolEnum.WHATEVER, "WHATEVER"),
Tuple.Create(SymbolEnum.s, "s"),
Tuple.Create(SymbolEnum.array, "array"),
Tuple.Create(SymbolEnum.jagged_array, "jagged_array"),
Tuple.Create(SymbolEnum.dot_identifier, "dot_identifier"),
Tuple.Create(SymbolEnum.typename_list, "typename_list"),
Tuple.Create(SymbolEnum.typename, "typename"),
Tuple.Create(SymbolEnum.whatever, "whatever"),
Tuple.Create(SymbolEnum.__list___merged_array_c____, "__list___merged_array_c____"),
Tuple.Create(SymbolEnum.__optList___merged_array_c____, "__optList___merged_array_c____"),
Tuple.Create(SymbolEnum.__list___merged_jagged_array_a____, "__list___merged_jagged_array_a____"),
Tuple.Create(SymbolEnum.__list___merged_dot_identifier_id____, "__list___merged_dot_identifier_id____"),
Tuple.Create(SymbolEnum.__list___merged_typename_list_t____, "__list___merged_typename_list_t____"),
Tuple.Create(SymbolEnum.__list_s__, "__list_s__"),
Tuple.Create(SymbolEnum.__optList_s__, "__optList_s__"));
var states_rep = StringRep.Create(Tuple.Create(StateEnum.INIT, "INIT"));
var mre = new DfaTable(ConnectionTable.Create(0, new int [] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 13, 4, 4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 4, 7, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 8, 4, 4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 4, 11, 4, 9, 4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 12, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }), //0
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(0, false), Tuple.Create(9, false)), //1
ConnectionTable.Create(160, new int [] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }), //2
ConnectionTable.Create(128, new int [] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }), //3
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(9, false)), //4
ConnectionTable.Create(9, new int [] { 13, 13, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13 }, Tuple.Create(7, false), Tuple.Create(9, false)), //5
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(5, false), Tuple.Create(9, false)), //6
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(4, false), Tuple.Create(9, false)), //7
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(1, false), Tuple.Create(9, false)), //8
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, -1, -1, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, 14, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }, Tuple.Create(8, false), Tuple.Create(9, false)), //9
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(2, false), Tuple.Create(9, false)), //10
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(3, false), Tuple.Create(9, false)), //11
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, -1, -1, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, 14, -1, 14, 14, 14, 14, 15, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }, Tuple.Create(8, false), Tuple.Create(9, false)), //12
ConnectionTable.Create(9, new int [] { 13, 13, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13 }, Tuple.Create(7, false)), //13
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, -1, -1, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, 14, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }, Tuple.Create(8, false)), //14
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, -1, -1, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, 14, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 16, 14, 14, 14 }, Tuple.Create(8, false)), //15
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, -1, -1, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, -1, -1, -1, -1, 14, -1, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }, Tuple.Create(6, false), Tuple.Create(8, false))); //16
var lexer = new NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum>(symbols_rep, states_rep, StateEnum.INIT, SymbolEnum.EOF, SymbolEnum.Error, mre);
lexer.AddIdRule(0, "\"" + "<" + "\"", StringCaseComparison.Sensitive, SymbolEnum.LANGLE, StateEnum.INIT);
lexer.AddIdRule(1, "\"" + ">" + "\"", StringCaseComparison.Sensitive, SymbolEnum.RANGLE, StateEnum.INIT);
lexer.AddIdRule(2, "\"" + "[" + "\"", StringCaseComparison.Sensitive, SymbolEnum.LBRACKET, StateEnum.INIT);
lexer.AddIdRule(3, "\"" + "]" + "\"", StringCaseComparison.Sensitive, SymbolEnum.RBRACKET, StateEnum.INIT);
lexer.AddIdRule(4, "\"" + "." + "\"", StringCaseComparison.Sensitive, SymbolEnum.DOT, StateEnum.INIT);
lexer.AddIdRule(5, "\"" + "," + "\"", StringCaseComparison.Sensitive, SymbolEnum.COMMA, StateEnum.INIT);
lexer.AddIdRule(6, "\"" + "new" + "\"", StringCaseComparison.Sensitive, SymbolEnum.NEW, StateEnum.INIT);
lexer.AddIdAction(7, "/" + @"[ \r\n\t]+" + "/", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.AddIdRule(8, "/" + @"[A-Za-z_][A-Za-z_0-9]*" + "/", StringCaseComparison.Sensitive, SymbolEnum.IDENTIFIER, StateEnum.INIT);
lexer.AddIdRule(9, "/" + @"." + "/", StringCaseComparison.Sensitive, SymbolEnum.WHATEVER, StateEnum.INIT);
return(lexer);
}
public string ToString(StringRep <SYMBOL_ENUM> symbolsRep)
{
return("COVERAGE INFO:" + Environment.NewLine
+ "--------------" + Environment.NewLine
+ "LHS symbol = symbols that are covered"
+ Environment.NewLine + Environment.NewLine
+ covers.Keys.Select(it => symbolsRep.Get(it) + " = " + coverAsChunkSet(it).ToString(symbolsRep, verboseMode: false)).Join(Environment.NewLine));
}
public string ToString(StringRep <SYMBOL_ENUM> symbolsRep)
{
var sb_states = new StringBuilder();
var sb_edges = new StringBuilder();
BuildString(symbolsRep, sb_states, sb_edges, null);
return(sb_states.ToString() + Environment.NewLine + Environment.NewLine + sb_edges.ToString());
}
public PrecedenceTable(StringRep <SYMBOL_ENUM> symbolsRep)
{
this.symbolsRep = symbolsRep;
this.operators = new Dictionary <SymbolChunk <SYMBOL_ENUM>, List <SymbolPrecedence <SYMBOL_ENUM> > >();
this.entryUseCounters = new Dictionary <ISymbolPrecedence <SYMBOL_ENUM>, List <int> >();
this.patterns = new Dictionary <SymbolChunk <SYMBOL_ENUM>, List <SymbolPrecedence <SYMBOL_ENUM> > >();
this.runningPriority = 0;
}
public string StateTransStr(StringRep <STATE_ENUM> statesRep)
{
string result = statesRep.Get(StateIn);
if (!StateIn.Equals(StateOut) || stateActions.Any())
{
result += " -> " + String.Join("", stateActions.Select(it => it + " -> ")) + statesRep.Get(StateOut);
}
return(result);
}
public ProductionsBuilder(StringRep <SYMBOL_ENUM> symbolsRep)
{
this.productions = new List <Production <SYMBOL_ENUM, TREE_NODE> >();
if (symbolsRep == null)
{
throw new ArgumentNullException();
}
this.symbolsRep = symbolsRep;
}
public Parser(ActionTableData <SYMBOL_ENUM, TREE_NODE> actionTable,
StringRep <SYMBOL_ENUM> symbolsRep)
{
this.symbolsRep = symbolsRep;
this.actionTable = actionTable;
ConsecutiveCorrectActionsLimit = 3;
LineNumberInit = 1;
ColumnNumberInit = 1;
}
public string ToString(StringRep <SYMBOL_ENUM> symbolsRep, StringRep <STATE_ENUM> statesRep)
{
string result = "states: " + StateTransStr(statesRep);
result += Environment.NewLine;
if (Text == null)
{
result += "EOF " + Position.XYString() + Environment.NewLine;
}
else
{
result += "text " + Position.XYString() + ": " + Text.PrintableString() + Environment.NewLine;
}
if (Rule == null)
{
result += "UNRECOGNIZED TEXT" + Environment.NewLine;
}
else if (!Rule.IsEofRule)
{
result += "rule [" + Rule.PatternId + "]: " + Rule.ToString(statesRep) + Environment.NewLine;
}
string indent = "";
if (tokens.Count > 1)
{
result += "multiple tokens {" + Environment.NewLine;
indent = " ";
}
foreach (TokenMatch <SYMBOL_ENUM> token in Tokens)
{
result += indent + "token [" + token.ID + "]: ";
if (!token.HasToken)
{
result += "*none*";
}
else
{
result += symbolsRep.Get(token.Token) + Environment.NewLine;
result += indent + "value assigned: " + (token.Value == null ? "null" : (Rule == null ? token.Value : token.Value.ToString().PrintableString()));
}
result += Environment.NewLine;
}
if (tokens.Count > 1)
{
result += "}" + Environment.NewLine;
}
return(result);
}
/// <summary></summary>
/// <param name="s"></param>
/// <returns></returns>
private static StringRep[] findReplacements(string s)
{
var oi = -1; var ci = -1; var id = "";
var reps = new List <StringRep>();
for (int i = 0; i < s.Length; i++)
{
var c = s[i].ToString();
if (c == "{")
{
if (oi >= 0)
{
oi = -1;
}
else
{
oi = i;
}
}
else if (c == "}")
{
if (oi >= 0)
{
var p = int.TryParse(id, out int ri);
if (!p)
{
throw new Exception($"Could not parser \"{{{id}}}\" to int index.");
}
var r = new StringRep()
{
oi = oi,
ci = i,
ri = ri
};
reps.Add(r);
oi = ci = -1;
}
else if (ci >= 0)
{
oi = -1;
}
else
{
ci = i;
}
}
else if (oi > 0)
{
id += c;
}
}
return(reps.ToArray());
}
public NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum> CreateLexer()
{
var symbols_rep = StringRep.Create(Tuple.Create(SymbolEnum.Error, "Error"),
Tuple.Create(SymbolEnum.EOF, "EOF"),
Tuple.Create(SymbolEnum.__term0, "__term0"),
Tuple.Create(SymbolEnum.__term1, "__term1"),
Tuple.Create(SymbolEnum.PLUS, "PLUS"),
Tuple.Create(SymbolEnum.MINUS, "MINUS"),
Tuple.Create(SymbolEnum.MULT, "MULT"),
Tuple.Create(SymbolEnum.DIV, "DIV"),
Tuple.Create(SymbolEnum.POWER, "POWER"),
Tuple.Create(SymbolEnum.s, "s"),
Tuple.Create(SymbolEnum.exp, "exp"),
Tuple.Create(SymbolEnum.NUM, "NUM"));
var states_rep = StringRep.Create(Tuple.Create(StateEnum.INIT, "INIT"),
Tuple.Create(StateEnum.COMMENT, "COMMENT"));
var mre = new DfaTable(ConnectionTable.Create(0, new int [] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 6, 4, 4, 4, 4, 1, 7, 8, 9, 4, 10, 4, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 13, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }), //0
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(0, false), Tuple.Create(10, false), Tuple.Create(11, false)), //1
ConnectionTable.Create(160, new int [] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }), //2
ConnectionTable.Create(128, new int [] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }), //3
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(10, false), Tuple.Create(11, false)), //4
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(8, false), Tuple.Create(10, false), Tuple.Create(11, false)), //5
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(9, false), Tuple.Create(10, false), Tuple.Create(11, false)), //6
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(1, false), Tuple.Create(10, false), Tuple.Create(11, false)), //7
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(4, false), Tuple.Create(10, false), Tuple.Create(11, false)), //8
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(2, false), Tuple.Create(10, false), Tuple.Create(11, false)), //9
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(3, false), Tuple.Create(10, false), Tuple.Create(11, false)), //10
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(5, false), Tuple.Create(10, false), Tuple.Create(11, false)), //11
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }, Tuple.Create(7, false), Tuple.Create(10, false), Tuple.Create(11, false)), //12
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(6, false), Tuple.Create(10, false), Tuple.Create(11, false)), //13
ConnectionTable.Create(48, new int [] { 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 }, Tuple.Create(7, false))); //14
var lexer = new NaiveLanguageTools.Lexer.Lexer <SymbolEnum, StateEnum>(symbols_rep, states_rep, StateEnum.INIT, SymbolEnum.EOF, SymbolEnum.Error, mre);
lexer.AddIdRule(0, "\"" + "(" + "\"", StringCaseComparison.Sensitive, SymbolEnum.__term0, StateEnum.INIT);
lexer.AddIdRule(1, "\"" + ")" + "\"", StringCaseComparison.Sensitive, SymbolEnum.__term1, StateEnum.INIT);
lexer.AddIdRule(2, "\"" + "+" + "\"", StringCaseComparison.Sensitive, SymbolEnum.PLUS, StateEnum.INIT);
lexer.AddIdRule(3, "\"" + "-" + "\"", StringCaseComparison.Sensitive, SymbolEnum.MINUS, StateEnum.INIT);
lexer.AddIdAction(4, "\"" + "*" + "\"", StringCaseComparison.Sensitive, match => { match.Token = SymbolEnum.MULT;
match.Value = SymbolEnum.MULT; }, StateEnum.INIT);
lexer.AddIdRule(5, "\"" + "/" + "\"", StringCaseComparison.Sensitive, SymbolEnum.DIV, StateEnum.INIT);
lexer.AddIdRule(6, "\"" + "^" + "\"", StringCaseComparison.Sensitive, SymbolEnum.POWER, StateEnum.INIT);
lexer.AddIdAction(7, "/" + @"[0-9]+" + "/", StringCaseComparison.Sensitive, match => { match.Token = SymbolEnum.NUM;
match.Value = Convert.ToDouble(match.Text); }, StateEnum.INIT);
lexer.AddIdAction(8, "\"" + " " + "\"", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.AddIdAction(9, "\"" + "#" + "\"", StringCaseComparison.Sensitive, match => { lexer.PushState(StateEnum.COMMENT); }, StateEnum.INIT);
lexer.AddIdAction(10, "/" + @"." + "/", StringCaseComparison.Sensitive, match => {}, StateEnum.COMMENT);
lexer.AddIdRule(11, "/" + @"." + "/", StringCaseComparison.Sensitive, SymbolEnum.Error, StateEnum.INIT);
lexer.EofAction = (TokenMatch <SymbolEnum> match) => match.Token = SymbolEnum.EOF;
return(lexer);
}
internal void BuildString(StringRep <SYMBOL_ENUM> symbolsRep,
StringBuilder sbStates,
StringBuilder sbEdges,
IEnumerable <string> nfaStateIndices)
{
sbStates.Append(State.ToString(nfaStateIndices, symbolsRep));
if (nfaStateIndices == null)
{
sbEdges.Append(EdgesTo
.Select(edge => State.Index + " -- " + symbolsRep.Get(edge.Key) + " --> " + edge.Value.State.Index)
.Join(Environment.NewLine));
}
}
public string ToString(IEnumerable <string> nfaStateIndices, StringRep <SYMBOL_ENUM> symbolsRep)
{
IEnumerable <SingleState <SYMBOL_ENUM, TREE_NODE> > states = Items
.Where(it => nfaStateIndices == null || nfaStateIndices.Contains(it.IndexStr));
if (!states.Any())
{
return("");
}
else
{
return(Index + "(" + InternalId + ")" + Environment.NewLine + String.Join(Environment.NewLine, states.Select(it => it.ToString(symbolsRep))));
}
}
internal Dfa(IEnumerable <Node <SYMBOL_ENUM, TREE_NODE> > nodes, StringRep <SYMBOL_ENUM> symbolsRep)
{
this.symbolsRep = symbolsRep;
this.nodes = nodes.ToList();
int node_idx = 0;
foreach (Node <SYMBOL_ENUM, TREE_NODE> node in nodes)
{
node.State.Index = node_idx++;
int state_idx = 0;
foreach (SingleState <SYMBOL_ENUM, TREE_NODE> state in node.State.Items)
{
state.Index = Tuple.Create(node.State.Index, state_idx++);
}
}
}
public static void PrintStringInfo(SwiftString str)
{
var stringRep = new StringRep(str);
var nsstringPtr = stringRep.BridgeObject;
Console.WriteLine("Swift string: " + str.ToString());
Console.Write($"Type: {stringRep.Discriminator} Ptr: {nsstringPtr.ToString ($"X{IntPtr.Size * 2}")} ");
if (nsstringPtr != IntPtr.Zero)
{
// Here be dragons. I didn't see a good API to get the reference count for this object.
// I intuited it by observing bytes in memory change on x64. This is clearly fragile.
var refPtr = nsstringPtr + (IntPtr.Size + sizeof(int));
var refCount = Marshal.ReadInt32(refPtr);
Console.Write($"RefCount: {refCount}");
}
Console.WriteLine();
}
public ActionTable(Dfa <SYMBOL_ENUM, TREE_NODE> dfa,
Productions <SYMBOL_ENUM, TREE_NODE> productions,
int lookaheadWidth)
: base(null, null, null,
productions.StartSymbol,
productions.EofSymbol,
productions.SyntaxErrorSymbol,
lookaheadWidth)
{
this.symbolsRep = productions.SymbolsRep;
this.symbols = productions.NonAndTerminals.ToList();
int symbolValuesWidth = 1 + productions.NonAndTerminals.Concat(productions.EofSymbol).Select(it => (int)(object)it).Max();
actionsTable = new IEnumerable <ParseAction <SYMBOL_ENUM, TREE_NODE> > [
dfa.IndexRange(),
(int)Math.Pow(symbolValuesWidth, lookaheadWidth)
];
edgesTable = CreateEdgesTable(dfa.IndexRange(), symbolValuesWidth);
recoveryTable = new IEnumerable <NfaCell <SYMBOL_ENUM, TREE_NODE> > [dfa.IndexRange(), symbolValuesWidth];
foreach (Node <SYMBOL_ENUM, TREE_NODE> node in dfa.Nodes)
{
foreach (KeyValuePair <SYMBOL_ENUM, Node <SYMBOL_ENUM, TREE_NODE> > edge in node.EdgesTo)
{
int edge_int = (int)(object)edge.Key;
edgesTable[node.State.Index, edge_int] = edge.Value.State.Index;
IEnumerable <NfaCell <SYMBOL_ENUM, TREE_NODE> > recovery_items =
edge.Value.State.ParsingActiveItems.Where(it => it.IsAtRecoveryPoint).Select(it => it.CreateCell()).ToList();
if (recovery_items.Any())
{
recoveryTable[node.State.Index, edge_int] = recovery_items;
}
}
}
}
private static void addEntry(Dictionary <SymbolChunk <SYMBOL_ENUM>, List <SymbolPrecedence <SYMBOL_ENUM> > > table,
SymbolPrecedence <SYMBOL_ENUM> entry,
StringRep <SYMBOL_ENUM> symbolsRep)
{
List <SymbolPrecedence <SYMBOL_ENUM> > list;
if (!table.TryGetValue(entry.Symbols, out list))
{
list = new List <SymbolPrecedence <SYMBOL_ENUM> >();
table.Add(entry.Symbols, list);
}
{
IEnumerable <SymbolPrecedence <SYMBOL_ENUM> > conflicts = list.Where(it => it.IsConflictingWith(entry)).ToArray();
if (conflicts.Any())
{
throw new ArgumentException("Duplicated precedence for " + entry.Symbols.ToString(symbolsRep));
}
}
list.Add(entry);
}
private Productions(
StringRep <SYMBOL_ENUM> symbolsRep,
IEnumerable <Production <SYMBOL_ENUM, TREE_NODE> > productions,
SYMBOL_ENUM eofSymbol, SYMBOL_ENUM syntaxErrorSymbol,
Action <string> addWarning)
{
this.SymbolsRep = symbolsRep;
this.productionsList = productions.ToList();
this.productions = productionsList.GroupBy(it => it.LhsNonTerminal).ToDictionary(it => it.Key, it => it.ToList());
this.EofSymbol = eofSymbol;
this.SyntaxErrorSymbol = syntaxErrorSymbol;
if (this.productions.Count == 0)
{
throw new ArgumentException("No productions.");
}
this.markingsMapping = productionsList.Select(it => it.MarkWith).Where(it => it != null).Distinct().ZipWithIndex().ToDictionary();
this.StartSymbol = productionsList.First().LhsNonTerminal;
nonTerminals = productionsList.Select(it => it.LhsNonTerminal).Distinct().ToList();
terminals = productionsList.Select(it => it.RhsSymbols)
.Flatten()
.Distinct()
.Where(it => !it.Equals(syntaxErrorSymbol) && !nonTerminals.Contains(it))
.ToList();
validate(addWarning);
int counter = 0;
foreach (Production <SYMBOL_ENUM, TREE_NODE> production in productionsList)
{
production.Attach(this, counter++);
}
}
public string ToString(StringRep <SYMBOL_ENUM> symbolsRep)
{
// lowering the case so we can search a string more effectively in DFA text file
string next_lookaheads = NextLookaheads.ToString(symbolsRep, verboseMode: false).ToLower();
string after_lookaheads = AfterLookaheads.ToString(symbolsRep, verboseMode: false).ToLower();
var source = new List <string>();
if (closureParents.Any())
{
source.AddRange(closureParents.Select(it => "c:" + it.IndexStr));
}
if (shiftParents.Any())
{
source.AddRange(shiftParents.Select(it => "s:" + it.IndexStr));
}
return(IndexStr + ") " + symbolsRep.Get(LhsSymbol) + " := "
+ (String.Join(" ", Production.RhsSymbols.Take(RhsSeenCount).Select(it => symbolsRep.Get(it)))
+ " . "
+ String.Join(" ", Production.RhsSymbols.Skip(RhsSeenCount).Select(it => symbolsRep.Get(it)))).Trim()
+ (next_lookaheads.Length > 0 ? "\t (n: " + next_lookaheads + " )" : "")
+ (after_lookaheads.Length > 0 ? "\t (a: " + after_lookaheads + " )" : "")
+ (source.Any() ? "\t <-- " + source.Join(" ") : ""));
}
public NaiveLanguageTools.Lexer.Lexer <TokenEnum, StateEnum> CreateLexer()
{
var symbols_rep = StringRep.Create(Tuple.Create(TokenEnum.Error, "Error"),
Tuple.Create(TokenEnum.EOF, "EOF"),
Tuple.Create(TokenEnum.STR, "STR"),
Tuple.Create(TokenEnum.LBRACE, "LBRACE"),
Tuple.Create(TokenEnum.LPAREN, "LPAREN"),
Tuple.Create(TokenEnum.RPAREN, "RPAREN"),
Tuple.Create(TokenEnum.RBRACE, "RBRACE"),
Tuple.Create(TokenEnum.EQ, "EQ"),
Tuple.Create(TokenEnum.SEMI, "SEMI"),
Tuple.Create(TokenEnum.COMMA, "COMMA"),
Tuple.Create(TokenEnum.ID, "ID"),
Tuple.Create(TokenEnum.proj, "proj"),
Tuple.Create(TokenEnum.pair, "pair"),
Tuple.Create(TokenEnum.dict, "dict"),
Tuple.Create(TokenEnum.list, "list"),
Tuple.Create(TokenEnum.id_list, "id_list"),
Tuple.Create(TokenEnum.str_list, "str_list"),
Tuple.Create(TokenEnum.dict_list, "dict_list"),
Tuple.Create(TokenEnum.__list___merged_dict_pair____, "__list___merged_dict_pair____"),
Tuple.Create(TokenEnum.__optList___merged_dict_pair____, "__optList___merged_dict_pair____"),
Tuple.Create(TokenEnum.__list___merged_id_list_ID____, "__list___merged_id_list_ID____"),
Tuple.Create(TokenEnum.__optList___merged_id_list_ID____, "__optList___merged_id_list_ID____"),
Tuple.Create(TokenEnum.__list___merged_str_list_STR____, "__list___merged_str_list_STR____"),
Tuple.Create(TokenEnum.__list___merged_dict_list_dict____, "__list___merged_dict_list_dict____"));
var states_rep = StringRep.Create(Tuple.Create(StateEnum.INIT, "INIT"),
Tuple.Create(StateEnum.STR, "STR"),
Tuple.Create(StateEnum.COMMENT, "COMMENT"));
var mre = new DfaTable(ConnectionTable.Create(0, new int [] { 6, 18, 18, 18, 18, 18, 18, 18, 18, 5, 2, 5, 5, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 5, 18, 8, 18, 18, 18, 18, 18, 15, 14, 3, 18, 10, 18, 18, 4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 18, 11, 18, 12, 18, 18, 18, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 18, 7, 18, 18, 9, 18, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1, 18, 13, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, -1, -1, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16 }), //0
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(0, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //1
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(14, false), Tuple.Create(20, false), Tuple.Create(26, false), Tuple.Create(28, false)), //2
ConnectionTable.Create(47, new int [] { 19 }, Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //3
ConnectionTable.Create(42, new int [] { 20, -1, -1, -1, -1, 21 }, Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //4
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(18, false), Tuple.Create(19, false), Tuple.Create(25, false), Tuple.Create(27, false)), //5
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(17, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //6
ConnectionTable.Create(0, new int [] { 22, 30, 30, 30, 30, 30, 30, 30, 30, 30, 23, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 27, 30, 30, 30, 24, 30, 30, 30, 30, 30, 30, 30, 25, 30, 30, 30, 30, 30, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, -1, -1, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28 }, Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //7
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(8, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //8
ConnectionTable.Create(48, new int [] { 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, -1, -1, -1, -1, -1, -1, -1, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, -1, -1, -1, -1, 31, -1, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31 }, Tuple.Create(7, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //9
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(6, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //10
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(5, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //11
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(4, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //12
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(3, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //13
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(2, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //14
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(1, false), Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //15
ConnectionTable.Create(160, new int [] { 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17 }), //16
ConnectionTable.Create(13, new int [] { 18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18 }), //17
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(18, false), Tuple.Create(25, false), Tuple.Create(27, false)), //18
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(23, false), Tuple.Create(24, false)), //19
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(22, false)), //20
ConnectionTable.Create(0, new int [] { 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 35, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, -1, -1, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }, Tuple.Create(21, false)), //21
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(15, false), Tuple.Create(16, false)), //22
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(13, false)), //23
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(12, false), Tuple.Create(16, false)), //24
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(11, false), Tuple.Create(16, false)), //25
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(10, false), Tuple.Create(16, false)), //26
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(9, false), Tuple.Create(16, false)), //27
ConnectionTable.Create(160, new int [] { 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29 }), //28
ConnectionTable.Create(13, new int [] { 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30 }), //29
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(16, false)), //30
ConnectionTable.Create(48, new int [] { 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, -1, -1, -1, -1, -1, -1, -1, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, -1, -1, -1, -1, 31, -1, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31 }, Tuple.Create(7, false)), //31
ConnectionTable.Create(160, new int [] { 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34 }), //32
ConnectionTable.Create(0, new int [] { 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 35, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, -1, -1, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }, Tuple.Create(21, false)), //33
ConnectionTable.Create(13, new int [] { 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33 }), //34
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(21, false))); //35
var lexer = new NaiveLanguageTools.Lexer.Lexer <TokenEnum, StateEnum>(symbols_rep, states_rep, StateEnum.INIT, TokenEnum.EOF, TokenEnum.Error, mre);
lexer.AddIdRule(0, "\"" + "{" + "\"", StringCaseComparison.Sensitive, TokenEnum.LBRACE, StateEnum.INIT);
lexer.AddIdRule(1, "\"" + "(" + "\"", StringCaseComparison.Sensitive, TokenEnum.LPAREN, StateEnum.INIT);
lexer.AddIdRule(2, "\"" + ")" + "\"", StringCaseComparison.Sensitive, TokenEnum.RPAREN, StateEnum.INIT);
lexer.AddIdRule(3, "\"" + "}" + "\"", StringCaseComparison.Sensitive, TokenEnum.RBRACE, StateEnum.INIT);
lexer.AddIdRule(4, "\"" + "=" + "\"", StringCaseComparison.Sensitive, TokenEnum.EQ, StateEnum.INIT);
lexer.AddIdRule(5, "\"" + ";" + "\"", StringCaseComparison.Sensitive, TokenEnum.SEMI, StateEnum.INIT);
lexer.AddIdRule(6, "\"" + "," + "\"", StringCaseComparison.Sensitive, TokenEnum.COMMA, StateEnum.INIT);
lexer.AddIdRule(7, "/" + @"[A-Za-z_0-9]+" + "/", StringCaseComparison.Sensitive, TokenEnum.ID, StateEnum.INIT);
lexer.AddIdAction(8, "\"" + "\"" + "\"", StringCaseComparison.Sensitive, match => { if (!string_error)
{
validateString(match, false);
}
lexer.PopState(); }, StateEnum.STR);
lexer.AddIdAction(9, "\"" + "\\b" + "\"", StringCaseComparison.Sensitive, match => { string_buf.Append("\b"); }, StateEnum.STR);
lexer.AddIdAction(10, "\"" + "\\t" + "\"", StringCaseComparison.Sensitive, match => { string_buf.Append("\t"); }, StateEnum.STR);
lexer.AddIdAction(11, "\"" + "\\n" + "\"", StringCaseComparison.Sensitive, match => { string_buf.Append("\n"); }, StateEnum.STR);
lexer.AddIdAction(12, "\"" + "\\f" + "\"", StringCaseComparison.Sensitive, match => { string_buf.Append("\f"); }, StateEnum.STR);
lexer.AddIdAction(13, "\"" + "\\\n" + "\"", StringCaseComparison.Sensitive, match => { string_buf.Append("\n"); }, StateEnum.STR);
lexer.AddIdAction(14, "\"" + "\n" + "\"", StringCaseComparison.Sensitive, match => { if (!string_error)
{
validateString(match, true);
}
lexer.PopState(); }, StateEnum.STR);
lexer.AddIdAction(15, "\"" + "\\\0" + "\"", StringCaseComparison.Sensitive, match => { if (!string_error)
{
string_error = true;
match.Value = "String contains escaped null character.";
match.Token = TokenEnum.Error;
}
}, StateEnum.STR);
lexer.AddIdAction(16, "/" + @"\\(.|\r)" + "/", StringCaseComparison.Sensitive, match => { string_buf.Append(match.Text.Substring(1)); }, StateEnum.STR);
lexer.AddIdAction(17, "\"" + "\0" + "\"", StringCaseComparison.Sensitive, match => { if (!string_error)
{
string_error = true;
match.Value = "String contains null character.";
match.Token = TokenEnum.Error;
}
}, StateEnum.STR);
lexer.AddIdAction(18, "/" + @".|\r" + "/", StringCaseComparison.Sensitive, match => { string_buf.Append(match.Text); }, StateEnum.STR);
lexer.AddIdAction(19, "/" + @"[ \f\r\t\u000b]" + "/", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.AddIdAction(20, "\"" + "\n" + "\"", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.AddIdAction(21, "/" + @"//(\r|.)*[\n]?" + "/", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.AddIdAction(22, "\"" + "/*" + "\"", StringCaseComparison.Sensitive, match => { lexer.PushState(StateEnum.COMMENT); }, StateEnum.INIT, StateEnum.COMMENT);
lexer.AddIdAction(23, "\"" + "*/" + "\"", StringCaseComparison.Sensitive, match => { lexer.PopState(); }, StateEnum.COMMENT);
lexer.AddIdAction(24, "\"" + "*/" + "\"", StringCaseComparison.Sensitive, match => { match.Value = "Unmatched */";
match.Token = TokenEnum.Error; }, StateEnum.INIT);
lexer.AddIdAction(25, "/" + @".|\r" + "/", StringCaseComparison.Sensitive, match => {}, StateEnum.COMMENT);
lexer.AddIdAction(26, "\"" + "\n" + "\"", StringCaseComparison.Sensitive, match => {}, StateEnum.COMMENT);
lexer.AddIdRule(27, "/" + @".|\r" + "/", StringCaseComparison.Sensitive, TokenEnum.Error, StateEnum.INIT);
lexer.AddIdRule(28, "\"" + "\n" + "\"", StringCaseComparison.Sensitive, TokenEnum.Error, StateEnum.INIT);
lexer.EofAction = match => { if (!lexer.IsValidEofState)
{
match.Value = "Invalid state at EOF";
match.Token = TokenEnum.Error;
}
else
{
match.Token = TokenEnum.EOF;
} };
return(lexer);
}
public Lexer <int, int> CreateLexer()
{
// For assembling string constants
LexPattern lex_pattern = null;
StringBuilder str_buf = null;
bool code_statement = false;
var lexer = new Lexer <int, int>(StringRep.CreateInt <Symbols>(), StringRep.CreateInt <States>(), States.GRAMMAR, Symbols.EOF, Symbols.Error);
string whitespace_pattern = "[ \r\n\f\t\u000b]+";
lexer.AddStringRule("using", Symbols.USING, States.GRAMMAR);
lexer.AddStringRule("namespace", Symbols.NAMESPACE, States.GRAMMAR);
lexer.AddStringAction("parser", match =>
{
match.Token = Symbols.PARSER;
lexer.PushState(States.FACTORY_SECTION);
},
States.GRAMMAR);
lexer.AddStringAction("lexer", match =>
{
match.Token = Symbols.LEXER;
lexer.PushState(States.FACTORY_SECTION);
}
,
States.GRAMMAR);
lexer.AddStringAction("options", match =>
{
match.Token = Symbols.OPTIONS;
lexer.PushState(States.OPTIONS_SECTION);
}, States.GRAMMAR);
lexer.AddStringAction(";", match =>
{
match.Token = Symbols.SEMI;
lexer.PopState();
}, States.OPTIONS_SECTION, States.FACTORY_SECTION);
lexer.AddStringRule("terminals", Symbols.TERMINALS, States.GRAMMAR);
lexer.AddStringRule("var", Symbols.VAR, States.GRAMMAR);
lexer.AddStringRule("types", Symbols.TYPES, States.GRAMMAR);
lexer.AddStringRule("patterns", Symbols.PATTERNS, States.GRAMMAR);
lexer.AddStringRule("tokens", Symbols.TOKENS, States.GRAMMAR);
lexer.AddStringRule(new[] { Symbols.TOKENS }, "int", Symbols.INT, States.GRAMMAR);
lexer.AddStringRule(new[] { Symbols.STATES }, "int", Symbols.INT, States.GRAMMAR);
lexer.AddStringRule(new[] { Symbols.LEXER }, "override", Symbols.OVERRIDE, States.FACTORY_SECTION);
lexer.AddStringRule(new[] { Symbols.PARSER }, "override", Symbols.OVERRIDE, States.FACTORY_SECTION);
lexer.AddStringRule("states", Symbols.STATES, States.GRAMMAR);
lexer.AddStringRule("precedence", Symbols.PRECEDENCE, States.GRAMMAR);
lexer.AddStringRule("parsing", Symbols.PARSING, States.GRAMMAR);
lexer.AddStringRule("scanning", Symbols.SCANNING, States.GRAMMAR);
lexer.AddStringRule("end", Symbols.END, States.GRAMMAR);
lexer.AddStringRule("%EOF", Symbols.EOF_ACTION, States.GRAMMAR);
lexer.AddRegexRule("%empty", Symbols.EMPTY, States.GRAMMAR);
lexer.AddRegexRule("%mark", Symbols.MARK, States.GRAMMAR);
lexer.AddRegexRule(IdentifierPattern, Symbols.IDENTIFIER, States.GRAMMAR, States.OPTIONS_SECTION, States.FACTORY_SECTION);
lexer.AddStringRule(";", Symbols.SEMI, States.GRAMMAR);
lexer.AddStringRule(":", Symbols.COLON, States.GRAMMAR, States.FACTORY_SECTION);
lexer.AddStringRule("=", Symbols.EQ, States.GRAMMAR);
lexer.AddStringRule("->", Symbols.RARROW, States.GRAMMAR);
lexer.AddStringRule("|", Symbols.PIPE, States.GRAMMAR);
lexer.AddStringRule("?", Symbols.QUESTION_MARK, States.GRAMMAR);
lexer.AddStringRule(".", Symbols.DOT, States.GRAMMAR, States.FACTORY_SECTION);
lexer.AddStringRule("[", Symbols.LBRACKET, States.GRAMMAR);
lexer.AddStringRule("]", Symbols.RBRACKET, States.GRAMMAR);
lexer.AddStringRule("(", Symbols.LPAREN, States.GRAMMAR);
lexer.AddStringRule(")", Symbols.RPAREN, States.GRAMMAR);
lexer.AddStringRule("<", Symbols.LANGLE, States.GRAMMAR, States.FACTORY_SECTION);
lexer.AddStringRule(">", Symbols.RANGLE, States.GRAMMAR, States.FACTORY_SECTION);
lexer.AddStringRule("*", Symbols.ASTERISK, States.GRAMMAR);
lexer.AddStringRule("+", Symbols.PLUS, States.GRAMMAR, States.OPTIONS_SECTION, States.FACTORY_SECTION);
lexer.AddStringRule("++", Symbols.PLUSPLUS, States.GRAMMAR);
lexer.AddStringRule("+?", Symbols.PLUS_OPT, States.GRAMMAR);
lexer.AddStringRule("-", Symbols.MINUS, States.GRAMMAR, States.OPTIONS_SECTION, States.FACTORY_SECTION);
lexer.AddStringRule("^", Symbols.ACCENT, States.GRAMMAR);
lexer.AddStringRule("#", Symbols.HASH, States.GRAMMAR);
lexer.AddStringRule("@", Symbols.AT, States.GRAMMAR);
// ----- strings and characters in code --------------------------------------------
// we are just rewriting input controlling when the string ends, so we can
// analyze real C# code properly (in short we want to know if we are in C# code, or in C# string)
lexer.AddStringAction(@"\\", match => str_buf.Append(match.Text),
States.STR_CODE, States.CHAR_CODE);
lexer.AddStringAction("\\\"", match => str_buf.Append(match.Text),
States.STR_CODE, States.CHAR_CODE);
lexer.AddStringAction("\\\'", match => str_buf.Append(match.Text),
States.STR_CODE, States.CHAR_CODE);
lexer.AddStringAction("\"\"", match => str_buf.Append(match.Text),
States.VERBATIM_STR_CODE);
lexer.AddStringAction("'", match => // start character in code
{
str_buf.Append(match.Text);
lexer.PushState(States.CHAR_CODE);
}, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
lexer.AddStringAction("\"", match => // start string in code
{
str_buf.Append(match.Text);
lexer.PushState(States.STR_CODE);
}, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
lexer.AddStringAction("@\"", match => // start verbatim string in code
{
str_buf.Append(match.Text);
lexer.PushState(States.VERBATIM_STR_CODE);
}, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
lexer.AddStringAction("\"", match => // end string in code
{
str_buf.Append(match.Text);
lexer.PopState();
}, States.STR_CODE, States.VERBATIM_STR_CODE);
lexer.AddStringAction("'", match => // end character in code
{
str_buf.Append(match.Text);
lexer.PopState();
}, States.CHAR_CODE);
// ----- string and regex common --------------------------------------------
lexer.AddRegexAction("\r|\n", match => { match.Token = Symbols.Error; match.Value = "New line not allowed inside a string/regex"; },
States.REGEX_GRAMMAR, States.STR_CODE, States.STR_GRAMMAR, States.CHAR_CODE);
// two backslashes
lexer.AddStringAction(@"\\", match => lex_pattern.AddSpecial(match.Text, @"\"),
States.STR_GRAMMAR, States.REGEX_GRAMMAR);
// ----- anything else for string in code ------------------------------------
lexer.AddRegexAction(".", match => str_buf.Append(match.Text), States.STR_CODE, States.VERBATIM_STR_CODE, States.CHAR_CODE);
// ----- string --------------------------------------------------------------
lexer.AddStringAction("\"", match => // start string in grammar
{
lex_pattern = new LexPattern(LexPattern.TypeEnum.String);
lexer.PushState(States.STR_GRAMMAR);
}, States.GRAMMAR);
lexer.AddStringAction("\"", match => // end string in grammar
{
match.Value = lex_pattern.SetStringComparison(StringCaseComparison.Sensitive);
lex_pattern = null;
match.Token = Symbols.STRING;
lexer.PopState();
}, States.STR_GRAMMAR);
lexer.AddStringAction("\"i", match => // end string in grammar
{
match.Value = lex_pattern.SetStringComparison(StringCaseComparison.Insensitive);
lex_pattern = null;
match.Token = Symbols.STRING;
lexer.PopState();
}, States.STR_GRAMMAR);
//http://msdn.microsoft.com/en-us/library/aa691087%28v=vs.71%29.aspx
//http://msdn.microsoft.com/en-us/library/aa664669%28v=vs.71%29.aspx
//http://blogs.msdn.com/b/csharpfaq/archive/2004/03/12/what-character-escape-sequences-are-available.aspx
lexer.AddStringAction(@"\0", match => lex_pattern.AddSpecial(match.Text, "\0"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\a", match => lex_pattern.AddSpecial(match.Text, "\a"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\b", match => lex_pattern.AddSpecial(match.Text, "\b"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\f", match => lex_pattern.AddSpecial(match.Text, "\f"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\n", match => lex_pattern.AddSpecial(match.Text, "\n"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\r", match => lex_pattern.AddSpecial(match.Text, "\r"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\t", match => lex_pattern.AddSpecial(match.Text, "\t"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\v", match => lex_pattern.AddSpecial(match.Text, "\v"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\'", match => lex_pattern.AddSpecial(match.Text, "\'"), States.STR_GRAMMAR);
lexer.AddStringAction(@"\""", match => lex_pattern.AddSpecial(match.Text, "\""), States.STR_GRAMMAR);
//http://msdn.microsoft.com/en-us/library/bb311038.aspx
lexer.AddRegexAction("\\\\x[0-9a-fA-F]{1,4}",
match => lex_pattern.AddHexCode(match.Text, match.Text.Substring(2)),
//States.STR_CODE,
States.STR_GRAMMAR);
lexer.AddRegexAction("\\\\u[0-9a-fA-F]{4}",
match => lex_pattern.AddHexCode(match.Text, match.Text.Substring(2)),
//States.STR_CODE,
States.STR_GRAMMAR);
lexer.AddRegexAction("\\\\U[0-9a-fA-F]{8}",
match => lex_pattern.AddHexCode(match.Text, match.Text.Substring(2)),
//States.STR_CODE,
States.STR_GRAMMAR);
lexer.AddRegexAction(@"\.", match => { match.Token = Symbols.Error; match.Value = "Unrecognized escape sequence \"" + match.Text.Substring(1) + "\""; }, States.STR_GRAMMAR);
lexer.AddStringAction(@"\", match => { match.Token = Symbols.Error; match.Value = "Empty escape sequence"; }, States.STR_GRAMMAR);
lexer.AddRegexAction(".", match => lex_pattern.Add(match.Text), States.STR_GRAMMAR);
// ----- regex --------------------------------------------------------------
lexer.AddStringAction("/", match => // regex start
{
lex_pattern = new LexPattern(LexPattern.TypeEnum.Regex);
lexer.PushState(States.REGEX_GRAMMAR);
}, States.GRAMMAR);
lexer.AddStringAction("/", match => // regex end
{
match.Value = lex_pattern.SetStringComparison(StringCaseComparison.Sensitive);
lex_pattern = null;
match.Token = Symbols.REGEX;
lexer.PopState();
}, States.REGEX_GRAMMAR);
lexer.AddStringAction("/i", match => // regex end
{
match.Value = lex_pattern.SetStringComparison(StringCaseComparison.Insensitive);
lex_pattern = null;
match.Token = Symbols.REGEX;
lexer.PopState();
}, States.REGEX_GRAMMAR);
// backslash and end-of-regex
lexer.AddStringAction(@"\/", match => lex_pattern.Add(match.Text.Substring(1)), States.REGEX_GRAMMAR);
lexer.AddRegexAction(".", match => lex_pattern.Add(match.Text), States.REGEX_GRAMMAR);
// ---- whitespaces --------------------------
// keep the variable for macro clean from whitespaces
lexer.AddRegexAction(new [] { Symbols.LMACRO }, whitespace_pattern, _ => { }, States.IN_CODE_MACRO);
lexer.AddRegexAction(whitespace_pattern, match => str_buf.Append(match.Text), States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
lexer.AddRegexAction(whitespace_pattern, _ => { }, States.GRAMMAR, States.OPTIONS_SECTION, States.FACTORY_SECTION);
// ----------- macros ----------------------
lexer.AddStringAction("$(", match =>
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
code_statement = false;
str_buf.Clear();
match.Token = Symbols.LMACRO;
lexer.PushState(States.IN_CODE_MACRO);
}, States.CODE_BLOCK, States.IN_CODE_MACRO);
lexer.AddStringAction(":", match =>
{
if (lexer.NestingCounter > 0)
{
str_buf.Append(match.Text);
}
else
{
// keeeping macro variable clean
if (str_buf.ToString().Trim().Length > 0)
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
}
str_buf.Clear();
code_statement = false;
match.Token = Symbols.COLON;
}
}, States.IN_CODE_MACRO);
lexer.AddStringAction("(", match =>
{
str_buf.Append(match.Text);
++lexer.NestingCounter;
}, States.IN_CODE_MACRO);
lexer.AddStringAction(")", match =>
{
if (lexer.NestingCounter > 0)
{
str_buf.Append(match.Text);
--lexer.NestingCounter;
}
else
{
// keeping macro variable clean
if (str_buf.ToString().Trim().Length > 0)
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
}
str_buf.Clear();
code_statement = false;
match.Token = Symbols.RMACRO;
lexer.PopState();
}
}, States.IN_CODE_MACRO);
// ----------- expressions ----------------------
// this is not 100% correct, because after COMMA can be LBRACE, so this is not CODE_EXPR but CODE_BLOCK
// so we have to fix this later
lexer.AddStringAction(new [] { Symbols.RARROW, Symbols.IDENTIFIER }, ",", match =>
{
match.Token = Symbols.COMMA;
lexer.PushState(States.CODE_EXPR);
str_buf = new StringBuilder();
code_statement = false;
}, States.GRAMMAR);
// not an expression, but it has to be below the more restricted COMMA rule
lexer.AddStringRule(",", Symbols.COMMA, States.GRAMMAR, States.OPTIONS_SECTION, States.FACTORY_SECTION);
lexer.AddStringAction(";", match =>
{
lexer.PopState();
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), false));
match.Token = Symbols.SEMI;
str_buf = null;
}, States.CODE_EXPR);
// ----------- code block ----------------------
lexer.AddStringAction("(", match => // start code block
{
match.Token = Symbols.LPAREN;
lexer.PushState(States.CODE_BLOCK);
str_buf = new StringBuilder();
code_statement = false;
}, States.FACTORY_SECTION);
lexer.AddStringAction("{", match => // start code block
{
match.Token = Symbols.LBRACE;
lexer.PushState(States.CODE_BLOCK);
str_buf = new StringBuilder();
code_statement = false;
}, States.GRAMMAR, States.FACTORY_SECTION);
lexer.AddStringAction(new [] { Symbols.RARROW, Symbols.IDENTIFIER, Symbols.COMMA },
"{", match => // start code block -- this is the correction of the previous too eager switch to CODE_EXPR
{
match.Token = Symbols.LBRACE;
lexer.PopState(); // remove the previous CODE_EXPR
lexer.PushState(States.CODE_BLOCK);
str_buf = new StringBuilder();
code_statement = false;
}, States.CODE_EXPR);
lexer.AddStringAction("{", match =>
{
str_buf.Append(match.Text);
lexer.PushState(States.CODE_BLOCK);
}, States.CODE_BLOCK);
lexer.AddStringAction("(", match =>
{
str_buf.Append(match.Text);
lexer.PushState(States.CODE_BLOCK);
}, States.CODE_BLOCK);
lexer.AddStringAction("}", match =>
{
lexer.PopState();
if (lexer.State != States.CODE_BLOCK)
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
str_buf = null;
match.Token = Symbols.RBRACE;
}
else
{
str_buf.Append(match.Text);
}
}, States.CODE_BLOCK);
lexer.AddStringAction(")", match =>
{
lexer.PopState();
if (lexer.State != States.CODE_BLOCK)
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
str_buf = null;
match.Token = Symbols.RPAREN;
}
else
{
str_buf.Append(match.Text);
}
}, States.CODE_BLOCK);
// identifier with dollar sign ("$") in front
lexer.AddRegexAction("\\" + CodePiece.PlaceholderSigil + IdentifierPattern, match =>
{
if (str_buf.Length > 0)
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
}
str_buf.Clear();
code_statement = false;
match.Value = match.Text.Substring(1);
match.Token = Symbols.CODE_PLACEHOLDER;
}, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
lexer.AddRegexAction(IdentifierPattern, match =>
{
if (str_buf.Length > 0)
{
lexer.PrependToken(Symbols.CODE_SNIPPET, new CodeSnippet(str_buf.ToString(), code_statement));
}
str_buf.Clear();
code_statement = false;
match.Token = Symbols.IDENTIFIER;
}, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
lexer.AddStringAction(";", match => { code_statement = true; str_buf.Append(match.Text); }, States.CODE_BLOCK);
lexer.AddRegexAction(".", match => str_buf.Append(match.Text), States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO);
// ---- comments -----------------------------------------
lexer.AddStringAction("/*", _ => lexer.PushState(States.COMMENT), States.GRAMMAR, States.OPTIONS_SECTION, States.CODE_BLOCK, States.COMMENT, States.CODE_EXPR, States.IN_CODE_MACRO, States.FACTORY_SECTION);
lexer.AddStringAction("*/", _ => lexer.PopState(), States.COMMENT);
lexer.AddRegexAction(".|\n|\r", _ => { }, States.COMMENT);
lexer.AddStringAction("*/", match => { match.Value = "Unmatched */"; match.Token = Symbols.Error; }, States.GRAMMAR, States.OPTIONS_SECTION, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO, States.FACTORY_SECTION);
// single-liners comment
lexer.AddRegexAction("//.*\n", _ => { }, States.GRAMMAR, States.OPTIONS_SECTION, States.CODE_BLOCK, States.CODE_EXPR, States.IN_CODE_MACRO, States.FACTORY_SECTION);
// -------------------------------------------------------
lexer.EofAction = match =>
{
if (!lexer.IsValidEofState)
{
match.Value = "Invalid state at EOF";
match.Token = Symbols.Error;
}
else
{
match.Token = Symbols.EOF;
}
};
return(lexer);
}
private static IEnumerable <Production <SYMBOL_ENUM, TREE_NODE> > unfoldErrorProductions_NOT_USED(StringRep <SYMBOL_ENUM> symbolsRep,
IEnumerable <Production <SYMBOL_ENUM, TREE_NODE> > productions,
SYMBOL_ENUM eofSymbol, SYMBOL_ENUM syntaxErrorSymbol)
{
var result = new Productions <SYMBOL_ENUM, TREE_NODE>(symbolsRep, productions, eofSymbol, syntaxErrorSymbol, s => { });
var new_productions = new List <Production <SYMBOL_ENUM, TREE_NODE> >();
// compute all non-terminals that serve as aliases for terminals
// alias is a symbol that can be substituted by single terminal, for example
// a := A | B (this is alias)
// a := A B (this is not an alias)
// we start with terminals, because they serve as aliases too (to themselves)
DynamicDictionary <SYMBOL_ENUM, HashSet <SYMBOL_ENUM> > term_aliases = result.Terminals
.Select(it => Tuple.Create(it, new HashSet <SYMBOL_ENUM>(new[] { it })))
.ToDefaultDynamicDictionary();
// this is not cover set algorithm!
while (true)
{
int count = term_aliases.Count;
foreach (SYMBOL_ENUM non_term in result.NonTerminals.Where(it => !term_aliases.ContainsKey(it)))
{
bool found = true;
foreach (Production <SYMBOL_ENUM, TREE_NODE> prod in result.FilterByLhs(non_term))
{
if (prod.RhsSymbols.Count != 1 || !term_aliases.ContainsKey(prod.RhsSymbols.Single()))
{
found = false;
break;
}
}
if (found)
{
term_aliases[non_term].AddRange(result.FilterByLhs(non_term).Select(it => term_aliases[it.RhsSymbols.Single()]).Flatten());
}
}
if (count == term_aliases.Count)
{
break;
}
}
// check the placement of error token in every error production
foreach (Production <SYMBOL_ENUM, TREE_NODE> prod in productions)
{
IEnumerable <SYMBOL_ENUM> error_symbols = prod.RhsSymbols.Where(it => it.Equals(result.SyntaxErrorSymbol));
if (error_symbols.Any())
{
new_productions.Add(prod);
}
else if (error_symbols.Count() > 1)
{
throw new ArgumentException("Only one syntax error token per production: " + prod.PositionDescription);
}
else
{
int idx = prod.RhsSymbols.IndexOf(result.SyntaxErrorSymbol);
if (idx != prod.RhsSymbols.Count - 2)
{
throw new ArgumentException("Syntax error token has to be next to last: " + prod.PositionDescription);
}
SYMBOL_ENUM recovery_symbol = prod.RhsSymbols[idx + 1];
if (!term_aliases.ContainsKey(recovery_symbol))
{
throw new ArgumentException("There has to be a terminal or alias non-terminal after syntax error token: " + prod.PositionDescription);
}
else if (result.NonTerminals.Contains(recovery_symbol))
{
foreach (SYMBOL_ENUM term in term_aliases[recovery_symbol])
{
new_productions.Add(new Production <SYMBOL_ENUM, TREE_NODE>(
symbolsRep,
prod.LhsNonTerminal,
prod.Recursive,
prod.RhsSymbols.SkipTail(1).Concat(term), // replacing aliased terminal
prod.UserAction,
prod.IdentityOuterFunctionParamIndex));
}
}
else
{
new_productions.Add(prod);
}
}
}
return(new_productions);
}
public NaiveLanguageTools.Lexer.Lexer <TokenEnum, StateEnum> CreateLexer()
{
var symbols_rep = StringRep.Create(Tuple.Create(TokenEnum.Error, "Error"),
Tuple.Create(TokenEnum.EOF, "EOF"),
Tuple.Create(TokenEnum.__term0, "__term0"),
Tuple.Create(TokenEnum.__term1, "__term1"),
Tuple.Create(TokenEnum.sin, "sin"),
Tuple.Create(TokenEnum.cos, "cos"),
Tuple.Create(TokenEnum.sqrt, "sqrt"),
Tuple.Create(TokenEnum.pi, "pi"),
Tuple.Create(TokenEnum.e, "e"),
Tuple.Create(TokenEnum.PLUS, "PLUS"),
Tuple.Create(TokenEnum.MINUS, "MINUS"),
Tuple.Create(TokenEnum.MULT, "MULT"),
Tuple.Create(TokenEnum.DIV, "DIV"),
Tuple.Create(TokenEnum.POWER, "POWER"),
Tuple.Create(TokenEnum.NUM, "NUM"),
Tuple.Create(TokenEnum.s, "s"),
Tuple.Create(TokenEnum.exp, "exp"));
var states_rep = StringRep.Create(Tuple.Create(StateEnum.INIT, "INIT"));
var mre = new DfaTable(ConnectionTable.Create(0, new int [] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 4, 1, 6, 7, 8, 4, 9, 4, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 12, 4, 4, 4, 4, 13, 4, 14, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 15, 4, 4, 16, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }), //0
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(0, false), Tuple.Create(14, false)), //1
ConnectionTable.Create(160, new int [] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }), //2
ConnectionTable.Create(128, new int [] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }), //3
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(14, false)), //4
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(13, false), Tuple.Create(14, false)), //5
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(1, false), Tuple.Create(14, false)), //6
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(9, false), Tuple.Create(14, false)), //7
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(7, false), Tuple.Create(14, false)), //8
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(8, false), Tuple.Create(14, false)), //9
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(10, false), Tuple.Create(14, false)), //10
ConnectionTable.Create(46, new int [] { 17, -1, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19 }, Tuple.Create(12, false), Tuple.Create(14, false)), //11
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(11, false), Tuple.Create(14, false)), //12
ConnectionTable.Create(111, new int [] { 20 }, Tuple.Create(14, false)), //13
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(6, false), Tuple.Create(14, false)), //14
ConnectionTable.Create(105, new int [] { 21 }, Tuple.Create(14, false)), //15
ConnectionTable.Create(105, new int [] { 23, -1, -1, -1, -1, -1, -1, -1, 22 }, Tuple.Create(14, false)), //16
ConnectionTable.Create(48, new int [] { 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }), //17
ConnectionTable.Create(46, new int [] { 17, -1, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19 }, Tuple.Create(12, false)), //18
ConnectionTable.Create(43, new int [] { 25, -1, 25, -1, -1, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26 }), //19
ConnectionTable.Create(115, new int [] { 27 }), //20
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(5, false)), //21
ConnectionTable.Create(114, new int [] { 28 }), //22
ConnectionTable.Create(110, new int [] { 29 }), //23
ConnectionTable.Create(48, new int [] { 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 19 }, Tuple.Create(12, false)), //24
ConnectionTable.Create(48, new int [] { 26, 26, 26, 26, 26, 26, 26, 26, 26, 26 }), //25
ConnectionTable.Create(48, new int [] { 26, 26, 26, 26, 26, 26, 26, 26, 26, 26 }, Tuple.Create(12, false)), //26
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(3, false)), //27
ConnectionTable.Create(116, new int [] { 30 }), //28
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(2, false)), //29
ConnectionTable.Create(-1, new int [] {}, Tuple.Create(4, false))); //30
var lexer = new NaiveLanguageTools.Lexer.Lexer <TokenEnum, StateEnum>(symbols_rep, states_rep, StateEnum.INIT, TokenEnum.EOF, TokenEnum.Error, mre);
lexer.AddIdRule(0, "\"" + "(" + "\"", StringCaseComparison.Sensitive, TokenEnum.__term0, StateEnum.INIT);
lexer.AddIdRule(1, "\"" + ")" + "\"", StringCaseComparison.Sensitive, TokenEnum.__term1, StateEnum.INIT);
lexer.AddIdRule(2, "\"" + "sin" + "\"", StringCaseComparison.Sensitive, TokenEnum.sin, StateEnum.INIT);
lexer.AddIdRule(3, "\"" + "cos" + "\"", StringCaseComparison.Sensitive, TokenEnum.cos, StateEnum.INIT);
lexer.AddIdRule(4, "\"" + "sqrt" + "\"", StringCaseComparison.Sensitive, TokenEnum.sqrt, StateEnum.INIT);
lexer.AddIdRule(5, "\"" + "pi" + "\"", StringCaseComparison.Sensitive, TokenEnum.pi, StateEnum.INIT);
lexer.AddIdRule(6, "\"" + "e" + "\"", StringCaseComparison.Sensitive, TokenEnum.e, StateEnum.INIT);
lexer.AddIdRule(7, "\"" + "+" + "\"", StringCaseComparison.Sensitive, TokenEnum.PLUS, StateEnum.INIT);
lexer.AddIdRule(8, "\"" + "-" + "\"", StringCaseComparison.Sensitive, TokenEnum.MINUS, StateEnum.INIT);
lexer.AddIdRule(9, "\"" + "*" + "\"", StringCaseComparison.Sensitive, TokenEnum.MULT, StateEnum.INIT);
lexer.AddIdRule(10, "\"" + "/" + "\"", StringCaseComparison.Sensitive, TokenEnum.DIV, StateEnum.INIT);
lexer.AddIdRule(11, "\"" + "^" + "\"", StringCaseComparison.Sensitive, TokenEnum.POWER, StateEnum.INIT);
lexer.AddIdAction(12, "/" + @"[0-9]+(\.[0-9]+)?([Ee][\-+]?[0-9]+)?" + "/", StringCaseComparison.Sensitive, match => { match.Token = TokenEnum.NUM;
match.Value = Convert.ToDouble(match.Text, System.Globalization.CultureInfo.InvariantCulture); }, StateEnum.INIT);
lexer.AddIdAction(13, "\"" + " " + "\"", StringCaseComparison.Sensitive, match => {}, StateEnum.INIT);
lexer.AddIdRule(14, "/" + @"." + "/", StringCaseComparison.Sensitive, TokenEnum.Error, StateEnum.INIT);
lexer.EofAction = (TokenMatch <TokenEnum> match) => match.Token = TokenEnum.EOF;
return(lexer);
}
public IEnumerable <string> Report(StringRep <SYMBOL_ENUM> symbolsRep)
{
return(productionsList.Select(prod => symbolsRep.Get(prod.LhsNonTerminal)
+ " := " + String.Join(" ", prod.RhsSymbols.Select(sym => symbolsRep.Get(sym)))));
}
private static IEnumerable <Production <SYMBOL_ENUM, TREE_NODE> > unfoldIdentityProductions(StringRep <SYMBOL_ENUM> symbolsRep,
IEnumerable <Production <SYMBOL_ENUM, TREE_NODE> > productions,
SYMBOL_ENUM eofSymbol, SYMBOL_ENUM syntaxErrorSymbol)
{
var used_aliases = new HashSet <SYMBOL_ENUM>();
while (true)
{
var result = new Productions <SYMBOL_ENUM, TREE_NODE>(symbolsRep, productions, eofSymbol, syntaxErrorSymbol, s => {});
List <Production <SYMBOL_ENUM, TREE_NODE> > new_productions = null;
CoverSets <SYMBOL_ENUM> cover_sets = new BuilderCoverSets <SYMBOL_ENUM, TREE_NODE>(result, lookaheadWidth: 1).ComputeCoverSets();
foreach (SYMBOL_ENUM non_term in result.NonTerminals)
{
if (!used_aliases.Contains(non_term) &&
!cover_sets.IsRecursive(non_term) &&
result.isAlias(non_term))
{
used_aliases.Add(non_term);
// todo: this won't work for multi-param alias rules
SYMBOL_ENUM[] expansions = result.FilterByLhs(non_term).Select(it => it.RhsSymbols.Single()).ToArray();
new_productions = new List <Production <SYMBOL_ENUM, TREE_NODE> >();
UnAliasing change = UnAliasing.NoChange;
foreach (Production <SYMBOL_ENUM, TREE_NODE> p in productions.Where(it => !it.LhsNonTerminal.Equals(non_term)))
{
new_productions.AddRange(
unAliasProductionRhs(non_term, expansions, p, result.StartSymbol, syntaxErrorSymbol, ref change)
.Select(rhs => new Production <SYMBOL_ENUM, TREE_NODE>(
symbolsRep,
p.LhsNonTerminal,
p.Recursive,
rhs,
p.UserAction,
// todo: this won't work for multi-param alias rules
p.IdentityOuterFunctionParamIndex)));
// we need to break right-away, otherwise "change" variable could be changed in next expansion to "expanded"
if (change == UnAliasing.Forbidden)
{
break;
}
}
if (change == UnAliasing.Expansion)
{
break;
}
else
{
new_productions = null;
}
}
}
// all non terminals checked or productions were expanded
if (new_productions == null)
{
break;
}
else
{
productions = new_productions;
}
}
return(productions);
}
public StatesInfo(string name, ConstMode mode, IEnumerable <Tuple <bool, string> > states)
: base(name, mode)
{
this.states = states.ToList();
this.StrRep = StringRep.Create <int>(this.states.ZipWithIndex().Select(it => Tuple.Create(it.Item3, it.Item2)));
}
