public LexerState GroupTokens(IEnumerable <TokenEntry> tokens) { var state = new LexerState(); GroupTokens(tokens, 0, state); SetNone(state, false); return(state); }
public string EmitState(LexerState state, bool root, IEnumerable<string> defaults = null) { if (table.KeywordTokens.Contains(state.Token)) { //defaults } string childTemplates; if (state.Children.Any()) { childTemplates = state.Children .Select(x => { var isCode = !string.IsNullOrEmpty(x.Code); string body = ""; if (isCode) body += x.Code; if (x.NewMode != null) body += "mode = " + x.NewMode.ToString() + ";\r\n"; if (x.Token != "None") { body += "return {TokenType}." + x.Token.ToString() + ";\r\n"; } else body += "\r\nbreak;\r\n"; var t = Properties.Resources.CaseTemplate .Replace("{{Cases}}", CreateCases(EncodeChar(x.State))) .Replace("{{Body}}", body) .Replace("{{States}}", x.Children.Any() ? EmitState(x, false) : ""); return t; }) .Aggregate((x, y) => x + "\r\n" + y); } else { return ""; } if (defaults != null && defaults.Any()) childTemplates += "\r\ndefault:\r\n" + defaults.Aggregate((x, y) => x + "\r\n" + y) + "\r\nbreak;\r\n"; var template = !root ? Properties.Resources.StateTemplate : Properties.Resources.RootStateTemplate; return template.Replace("{{States}}", childTemplates); }
private void SetNone(LexerState state, bool ancestralExitState) { if (ancestralExitState && state.Token == "Unknown") { state.Token = "None"; } else if (state.Token != "Unknown" && state.Token != "None") { ancestralExitState = true; } foreach (var child in state.Children) { SetNone(child, ancestralExitState); } }
public string EmitState(LexerState state, bool root, IEnumerable <string> defaults = null) { if (table.KeywordTokens.Contains(state.Token)) { //defaults } string childTemplates; if (state.Children.Any()) { childTemplates = state.Children .Select(x => { var isCode = !string.IsNullOrEmpty(x.Code); string body = ""; if (isCode) { body += x.Code; } if (x.NewMode != null) { body += "mode = " + x.NewMode.ToString() + ";\r\n"; } if (x.Token != "None") { body += "return {TokenType}." + x.Token.ToString() + ";\r\n"; } else { body += "\r\nbreak;\r\n"; } var t = Properties.Resources.CaseTemplate .Replace("{{Char}}", EncodeChar(x.State)) .Replace("{{Body}}", body) .Replace("{{States}}", x.Children.Any() ? EmitState(x, false) : ""); return(t); }) .Aggregate((x, y) => x + "\r\n" + y); } else { return(""); } if (defaults != null && defaults.Any()) { childTemplates += "\r\ndefault:\r\n" + defaults.Aggregate((x, y) => x + "\r\n" + y) + "\r\nbreak;\r\n"; } var template = !root ? Properties.Resources.StateTemplate : Properties.Resources.RootStateTemplate; return(template.Replace("{{States}}", childTemplates)); }
private void SetNone(LexerState state, bool ancestralExitState) { if (ancestralExitState && state.Token == "Unknown") state.Token = "None"; else if (state.Token != "Unknown" && state.Token != "None") ancestralExitState = true; foreach (var child in state.Children) { SetNone(child, ancestralExitState); } }
public void GroupTokens(IEnumerable<TokenEntry> tokens, int groupIndex, LexerState state) { var groups = tokens .Where(x => x.Lexeme != null && groupIndex < x.Lexeme.Length) .GroupBy(x => x.Lexeme[groupIndex]); foreach (var group in groups) { var groupState = new LexerState(); groupState.State = group.Key; var returnToken = group.SingleOrDefault(x => x.Lexeme.Length - 1 == groupIndex); if (returnToken != null && returnToken.Lexeme != null) { groupState.Token = returnToken.TokenType; groupState.Code = returnToken.Code; groupState.NewMode = returnToken.NewMode; } else { groupState.Token = "Unknown"; } state.Children.Add(groupState); GroupTokens(group, groupIndex + 1, groupState); } }
public LexerState GroupTokens(IEnumerable<TokenEntry> tokens) { var state = new LexerState(); GroupTokens(tokens, 0, state); SetNone(state, false); return state; }