public LexerState GroupTokens(IEnumerable <TokenEntry> tokens)
{
var state = new LexerState();
GroupTokens(tokens, 0, state);
SetNone(state, false);
return(state);
}
public string EmitState(LexerState state, bool root, IEnumerable<string> defaults = null)
{
if (table.KeywordTokens.Contains(state.Token))
{
//defaults
}
string childTemplates;
if (state.Children.Any())
{
childTemplates = state.Children
.Select(x =>
{
var isCode = !string.IsNullOrEmpty(x.Code);
string body = "";
if (isCode)
body += x.Code;
if (x.NewMode != null)
body += "mode = " + x.NewMode.ToString() + ";\r\n";
if (x.Token != "None")
{
body += "return {TokenType}." + x.Token.ToString() + ";\r\n";
}
else
body += "\r\nbreak;\r\n";
var t = Properties.Resources.CaseTemplate
.Replace("{{Cases}}", CreateCases(EncodeChar(x.State)))
.Replace("{{Body}}", body)
.Replace("{{States}}", x.Children.Any() ? EmitState(x, false) : "");
return t;
})
.Aggregate((x, y) => x + "\r\n" + y);
}
else
{
return "";
}
if (defaults != null && defaults.Any())
childTemplates += "\r\ndefault:\r\n" +
defaults.Aggregate((x, y) => x + "\r\n" + y) + "\r\nbreak;\r\n";
var template = !root ?
Properties.Resources.StateTemplate :
Properties.Resources.RootStateTemplate;
return template.Replace("{{States}}", childTemplates);
}
private void SetNone(LexerState state, bool ancestralExitState)
{
if (ancestralExitState && state.Token == "Unknown")
{
state.Token = "None";
}
else if (state.Token != "Unknown" && state.Token != "None")
{
ancestralExitState = true;
}
foreach (var child in state.Children)
{
SetNone(child, ancestralExitState);
}
}
public string EmitState(LexerState state, bool root, IEnumerable <string> defaults = null)
{
if (table.KeywordTokens.Contains(state.Token))
{
//defaults
}
string childTemplates;
if (state.Children.Any())
{
childTemplates = state.Children
.Select(x =>
{
var isCode = !string.IsNullOrEmpty(x.Code);
string body = "";
if (isCode)
{
body += x.Code;
}
if (x.NewMode != null)
{
body += "mode = " + x.NewMode.ToString() + ";\r\n";
}
if (x.Token != "None")
{
body += "return {TokenType}." + x.Token.ToString() + ";\r\n";
}
else
{
body += "\r\nbreak;\r\n";
}
var t = Properties.Resources.CaseTemplate
.Replace("{{Char}}", EncodeChar(x.State))
.Replace("{{Body}}", body)
.Replace("{{States}}", x.Children.Any() ? EmitState(x, false) : "");
return(t);
})
.Aggregate((x, y) => x + "\r\n" + y);
}
else
{
return("");
}
if (defaults != null && defaults.Any())
{
childTemplates += "\r\ndefault:\r\n" +
defaults.Aggregate((x, y) => x + "\r\n" + y) + "\r\nbreak;\r\n";
}
var template = !root ?
Properties.Resources.StateTemplate :
Properties.Resources.RootStateTemplate;
return(template.Replace("{{States}}", childTemplates));
}
private void SetNone(LexerState state, bool ancestralExitState)
{
if (ancestralExitState && state.Token == "Unknown")
state.Token = "None";
else if (state.Token != "Unknown" && state.Token != "None")
ancestralExitState = true;
foreach (var child in state.Children)
{
SetNone(child, ancestralExitState);
}
}
public void GroupTokens(IEnumerable<TokenEntry> tokens, int groupIndex, LexerState state)
{
var groups = tokens
.Where(x => x.Lexeme != null && groupIndex < x.Lexeme.Length)
.GroupBy(x => x.Lexeme[groupIndex]);
foreach (var group in groups)
{
var groupState = new LexerState();
groupState.State = group.Key;
var returnToken = group.SingleOrDefault(x => x.Lexeme.Length - 1 == groupIndex);
if (returnToken != null && returnToken.Lexeme != null)
{
groupState.Token = returnToken.TokenType;
groupState.Code = returnToken.Code;
groupState.NewMode = returnToken.NewMode;
}
else
{
groupState.Token = "Unknown";
}
state.Children.Add(groupState);
GroupTokens(group, groupIndex + 1, groupState);
}
}
public LexerState GroupTokens(IEnumerable<TokenEntry> tokens)
{
var state = new LexerState();
GroupTokens(tokens, 0, state);
SetNone(state, false);
return state;
}
