static ICollection <LRItem> _FillLRMove(this CfgDocument cfg, IEnumerable <LRItem> itemSet, object input, IProgress <CfgLalr1Progress> progress, ICollection <LRItem> result = null)
{
if (null == result)
{
result = new HashSet <LRItem>();
}
int i = 0;
foreach (var item in itemSet)
{
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.ComputingMove, i));
}
var next = item.RightIndex < item.Rule.Right.Count ? item.Rule.Right[item.RightIndex] : null;
if (item.RightIndex < item.Rule.Right.Count)
{
if (Equals(next, input))
{
var lri = new LRItem(item.Rule, item.RightIndex + 1);
result.Add(lri);
}
}
++i;
}
_FillLRClosureInPlace(cfg, progress, result);
return(result);
}
public static IList <CfgMessage> EliminateFirstFirstConflicts(this CfgDocument cfg)
{
var result = new List <CfgMessage>();
foreach (var nt in new List <string>(cfg.EnumNonTerminals()))
{
var rules = cfg.FillNonTerminalRules(nt);
var rights = new List <IList <string> >();
foreach (var rule in rules)
{
rights.Add(rule.Right);
}
while (true)
{
var pfx = rights.GetLongestCommonPrefix();
if (pfx.IsNullOrEmpty())
{
break;
}
// obv first first conflict
var nnt = _GetLeftFactorId(cfg, nt);
var suffixes = new List <IList <string> >();
foreach (var rule in rules)
{
if (rule.Right.StartsWith(pfx))
{
rights.Remove(rule.Right);
suffixes.Add(new List <string>(rule.Right.Range(pfx.Count)));
cfg.Rules.Remove(rule);
result.Add(new CfgMessage(CfgErrorLevel.Message, -1, string.Format("Removed rule {0} because it is part of a first-first conflict.", rule), rule.Line, rule.Column, rule.Position));
}
}
var newRule = new CfgRule(nt);
newRule.Right.AddRange(pfx);
newRule.Right.Add(nnt);
if (!cfg.Rules.Contains(newRule))
{
cfg.Rules.Add(newRule);
}
result.Add(new CfgMessage(CfgErrorLevel.Message, -1, string.Format("Added rule {0} to resolve first-first conflict.", newRule), 0, 0, 0));
foreach (var suffix in suffixes)
{
newRule = new CfgRule(nnt);
newRule.Right.AddRange(suffix);
if (!cfg.Rules.Contains(newRule))
{
cfg.Rules.Add(newRule);
}
result.Add(new CfgMessage(CfgErrorLevel.Message, -1, string.Format("Added rule {0} to resolve first-first conflict.", newRule), 0, 0, 0));
}
_SetAttribute(cfg, nnt, "collapsed", true);
}
}
return(result);
}
public static IList <CfgMessage> TryWriteClassTo(CfgDocument cfg, string name, string @namespace, string language, IProgress <CfgLL1Progress> progress, TextWriter writer)
{
var result = new List <CfgMessage>();
if (string.IsNullOrEmpty(language))
{
language = "cs";
}
var cdp = CodeDomProvider.CreateProvider(language);
CodeTypeDeclaration parser;
result.AddRange(_CreateParserClass(cfg, name, progress, out parser));
if (null != parser)
{
var opts = new CodeGeneratorOptions();
opts.BlankLinesBetweenMembers = false;
if (string.IsNullOrEmpty(@namespace))
{
cdp.GenerateCodeFromType(parser, writer, opts);
}
else
{
var cns = new CodeNamespace(@namespace);
cns.Types.Add(parser);
cdp.GenerateCodeFromNamespace(cns, writer, opts);
}
}
return(result);
}
static Lrfa _ToLrfa(CfgDocument cfg, IProgress <CfgLalr1Progress> progress)
{
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.ComputingStates, 0));
}
// TODO: this takes a long time sometimes
var map = new Dictionary <ICollection <LRItem>, Lrfa>(_LRItemSetComparer.Default);
// create an augmented grammar - add rule {start} -> [[StartId]]
var start = new CfgRule(cfg.GetAugmentedStartId(cfg.StartSymbol), new string[] { cfg.StartSymbol });
var cl = new HashSet <LRItem>();
cl.Add(new LRItem(start, 0));
_FillLRClosureInPlace(cfg, progress, cl);
var lrfa = new Lrfa(true, cl);
var items = cl.Count;
map.Add(cl, lrfa);
var done = false;
var oc = 0;
while (!done)
{
done = true;
var arr = map.Keys.ToArray();
for (var i = 0; i < arr.Length; ++i)
{
var itemSet = arr[i];
foreach (var item in itemSet)
{
var next = item.RightIndex < item.Rule.Right.Count ? item.Rule.Right[item.RightIndex] : null;
if (item.RightIndex < item.Rule.Right.Count)
{
var n = _FillLRMove(cfg, itemSet, next, progress);
if (!_ContainsItemSet(map.Keys, n))
{
done = false;
var npda = new Lrfa(true, n);
map.Add(n, npda);
items += n.Count;
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.ComputingConfigurations, items));
}
}
map[itemSet].Transitions[next] = map[n];
}
}
}
if (!done)
{
oc = map.Count;
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.ComputingStates, oc));
}
}
}
return(lrfa);
}
static int _Lalr1Tree(string[] args)
{
if (1 > args.Length || args.Length > 2)
{
_PrintUsageLalr1Tree();
return(1);
}
var pckspec = args[0];
string pck;
using (var sr = File.OpenText(pckspec))
pck = sr.ReadToEnd();
var cfg = CfgDocument.Parse(pck);
var lex = LexDocument.Parse(pck);
var tokenizer = lex.ToTokenizer(
(1 < args.Length) ? (TextReaderEnumerable) new FileReaderEnumerable(args[1]) :
new ConsoleReaderEnumerable(),
cfg.EnumSymbols(), new _TokenizerConsoleProgress());
var parser = cfg.ToLalr1Parser(tokenizer, new _Lalr1ConsoleProgress());
Console.Error.WriteLine();
parser.ShowHidden = true;
while (LRNodeType.EndDocument != parser.NodeType)
{
Console.WriteLine(parser.ParseReductions());
}
return(1);
}
public static Lalr1Parser ToLalr1Parser(this CfgDocument cfg, IEnumerable <Token> tokenizer, IProgress <CfgLalr1Progress> progress = null)
{
Lalr1Parser parser;
var res = TryToLalr1Parser(cfg, out parser, tokenizer, progress);
CfgException.ThrowIfErrors(res);
return(parser);
}
public static CfgLalr1ParseTable ToLalr1ParseTable(this CfgDocument cfg, IProgress <CfgLalr1Progress> progress = null)
{
CfgLalr1ParseTable result = null;
var msgs = TryToLalr1ParseTable(cfg, progress, out result);
CfgException.ThrowIfErrors(msgs);
return(result);
}
public Lalr1DebugParser(CfgDocument cfg, IEnumerable <Token> tokenizer, CfgLalr1ParseTable parseTable = null)
{
_cfg = cfg;
_PopulateAttrs();
_stack = new Stack <int>();
_tokenEnum = tokenizer.GetEnumerator();
_parseTable = parseTable ?? cfg.ToLalr1ParseTable();
_nodeType = LRNodeType.Initial;
}
public static IList <CfgMessage> TryValidateLalr1(this CfgDocument cfg, IList <CfgMessage> result = null)
{
if (null == result)
{
result = new List <CfgMessage>();
}
_TryValidateAttributesLalr1(cfg, result);
_TryValidateRulesLalr1(cfg, result);
return(result);
}
/// <summary>
/// Constructs a new instance of the parser
/// </summary>
/// <param name="parseTable">The parse table to use</param>
/// <param name="tokenizer">The tokenizer to use </param>
/// <param name="startSymbol">The start symbol</param>
public LL1DebugParser(CfgDocument cfg,
IEnumerable <Token> tokenizer)
{
_cfg = cfg;
_PopulateAttrs();
_parseTable = cfg.ToLL1ParseTable();
_stack = new Stack <string>();
_errorToken.Symbol = null;
Restart(tokenizer);
}
public static IList <CfgLL1Conflict> FillLL1Conflicts(this CfgDocument cfg, IList <CfgLL1Conflict> result = null)
{
if (null == result)
{
result = new List <CfgLL1Conflict>();
}
// build a temporary parse table to check for conflicts
var predict = cfg.FillPredict();
var follows = cfg.FillFollows();
foreach (var nt in cfg.EnumNonTerminals())
{
var d = new Dictionary <string, CfgRule>();
foreach (var f in predict[nt])
{
if (null != f.Symbol)
{
CfgRule r;
if (d.TryGetValue(f.Symbol, out r) && r != f.Rule)
{
var cf = new CfgLL1Conflict(CfgLL1ConflictKind.FirstFirst, r, f.Rule, f.Symbol);
if (!result.Contains(cf))
{
result.Add(cf);
}
}
else
{
d.Add(f.Symbol, f.Rule);
}
}
else
{
foreach (var ff in follows[nt])
{
CfgRule r;
if (d.TryGetValue(ff, out r) && r != f.Rule)
{
var cf = new CfgLL1Conflict(CfgLL1ConflictKind.FirstFollows, r, f.Rule, ff);
if (!result.Contains(cf))
{
result.Add(cf);
}
}
else
{
d.Add(ff, f.Rule);
}
}
}
}
}
return(result);
}
static string _GetLeftFactorId(CfgDocument cfg, string s)
{
var i = 2;
var ss = string.Concat(s, "part");
while (cfg.IsSymbol(ss))
{
ss = string.Concat(s, "part", i.ToString());
++i;
}
return(ss);
}
static string _GetRightAssocId(CfgDocument cfg, string s)
{
var i = 2;
var ss = string.Concat(s, "rightassoc");
while (cfg.IsSymbol(ss))
{
ss = string.Concat(s, "rightassoc", i.ToString());
++i;
}
return(ss);
}
public static IList <CfgMessage> EliminateFirstFollowsConflicts(this CfgDocument cfg)
{
var result = new List <CfgMessage>();
var conflicts = cfg.FillLL1Conflicts();
for (int ic = conflicts.Count, i = 0; i < ic; ++i)
{
var conflict = conflicts[i];
if (CfgLL1ConflictKind.FirstFollows == conflict.Kind)
{
if (conflict.Rule1.IsNil || conflict.Rule2.IsNil)
{
var rule = conflict.Rule1.IsNil ? conflict.Rule1 : conflict.Rule2;
// we might be able to do something about this.
var refs = cfg.FillReferencesToSymbol(rule.Left);
var ntr = cfg.FillNonTerminalRules(rule.Left);
for (int jc = refs.Count, j = 0; j < jc; ++j)
{
for (int kc = ntr.Count, k = 0; k < kc; ++k)
{
var ntrr = ntr[k];
var r = refs[j];
var rr = new CfgRule(r.Left, r.Right.Replace(rule.Left, ntrr.Right));
if (!cfg.Rules.Contains(rr))
{
cfg.Rules.Add(rr);
}
result.Add(new CfgMessage(CfgErrorLevel.Message, -1, string.Concat("Added rule ", rr.ToString(), " to resolve first-follows conflict."), rr.Line, rr.Column, rr.Position));
}
result.Add(new CfgMessage(CfgErrorLevel.Message, -1, string.Concat("Removed rule ", refs[j].ToString(), " to resolve first-follows conflict."), refs[j].Line, refs[j].Column, refs[j].Position));
cfg.Rules.Remove(refs[j]);
}
for (int jc = ntr.Count, j = 0; j < jc; ++j)
{
cfg.Rules.Remove(ntr[j]);
result.Add(new CfgMessage(CfgErrorLevel.Message, -1, string.Concat("Removed rule ", ntr[j].ToString(), " to resolve first-follows conflict."), ntr[j].Line, ntr[j].Column, ntr[j].Position));
}
}
}
}
return(result);
}
static void _SetAttribute(CfgDocument cfg, string symbol, string name, object value)
{
CfgAttributeList attrs;
if (!cfg.AttributeSets.TryGetValue(symbol, out attrs))
{
attrs = new CfgAttributeList();
cfg.AttributeSets.Add(symbol, attrs);
}
var i = attrs.IndexOf(name);
if (0 > i)
{
attrs.Add(new CfgAttribute(name, value));
}
else
{
attrs[i].Value = value;
}
}
static void _FillLRClosureInPlace(CfgDocument cfg, IProgress <CfgLalr1Progress> progress, ICollection <LRItem> result)
{
var done = false;
while (!done)
{
done = true;
var l = result.ToArray();
for (var i = 0; i < l.Length; i++)
{
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.ComputingClosure, i));
}
var item = l[i];
var next = item.RightIndex < item.Rule.Right.Count ? item.Rule.Right[item.RightIndex] : null;
if (item.RightIndex < item.Rule.Right.Count)
{
if (cfg.IsNonTerminal(next))
{
for (int jc = cfg.Rules.Count, j = 0; j < jc; ++j)
{
var r = cfg.Rules[j];
if (r.Left == next)
{
var lri = new LRItem(r, 0);
if (!result.Contains(lri))
{
done = false;
result.Add(lri);
}
}
}
}
}
}
}
}
public static void Transform(CfgDocument cfg, TextWriter output)
{
var terms = cfg.FillTerminals();
for (int ic = terms.Count, i = 0; i < ic; ++i)
{
var t = terms[i];
if ("#ERROR" != t && "#EOS" != t)
{
output.WriteLine(string.Concat("%token ", t));
}
}
output.WriteLine("%%");
foreach (var nt in cfg.EnumNonTerminals())
{
var nt2 = _EscId(nt);
var rules = cfg.FillNonTerminalRules(nt);
if (0 < rules.Count) // sanity check
{
output.Write(nt2);
var delim = ":";
for (int ic = rules.Count, i = 0; i < ic; ++i)
{
var rule = rules[i];
output.Write(delim);
for (int jc = rule.Right.Count, j = 0; j < jc; ++j)
{
output.Write(" ");
output.Write(_EscId(rule.Right[j]));
}
delim = Environment.NewLine + "\t|";
}
output.WriteLine(";");
}
}
}
public static void Transform(TextReader input, TextWriter output) => Transform(CfgDocument.ReadFrom(input), output);
public static IList <CfgMessage> TryToLalr1ParseTable(this CfgDocument cfg, IProgress <CfgLalr1Progress> progress, out CfgLalr1ParseTable parseTable)
{
var result = new List <CfgMessage>();
var start = cfg.GetAugmentedStartId(cfg.StartSymbol);
var lrfa = _ToLrfa(cfg, progress);
var trnsCfg = _ToLRTransitionGrammar(cfg, lrfa, progress);
trnsCfg.RebuildCache();
var closure = new List <Lrfa>();
parseTable = new CfgLalr1ParseTable();
var itemSets = new List <ICollection <LRItem> >();
lrfa.FillClosure(closure);
var i = 0;
foreach (var p in closure)
{
itemSets.Add(p.AcceptSymbol);
parseTable.Add(new Dictionary <string, (int RuleOrStateId, string Left, string[] Right)>());
++i;
}
i = 0;
foreach (var p in closure)
{
foreach (var trn in p.Transitions)
{
var idx = closure.IndexOf(trn.Value);
parseTable[i].Add(
trn.Key,
(idx, null, null)
);
}
foreach (var item in p.AcceptSymbol)
{
if (Equals(item.Rule.Left, start) && item.RightIndex == item.Rule.Right.Count)
{
parseTable[i].Add(
"#EOS",
(-1, null, null));
break;
}
}
++i;
}
var follows = trnsCfg.FillFollows();
// work on our reductions now
var map = new Dictionary <CfgRule, ICollection <string> >(_TransitionMergeRuleComparer.Default);
foreach (var rule in trnsCfg.Rules)
{
ICollection <string> f;
if (!map.TryGetValue(rule, out f))
{
map.Add(rule, follows[rule.Left]);
}
else
{
foreach (var o in follows[rule.Left])
{
if (!f.Contains(o))
{
f.Add(o);
}
}
}
}
var j = 0;
foreach (var mapEntry in map)
{
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.ComputingReductions, j));
}
var rule = mapEntry.Key;
var lr = _LrtSymbol.Parse(rule.Right[rule.Right.Count - 1]);
var left = _LrtSymbol.Parse(rule.Left).Id;
var right = new List <string>();
foreach (var s in rule.Right)
{
right.Add(_LrtSymbol.Parse(s).Id);
}
var newRule = new CfgRule(left, right);
if (!Equals(left, start))
{
foreach (var f in mapEntry.Value)
{
// build the rule data
var rr = new string[newRule.Right.Count];
for (var ri = 0; ri < rr.Length; ri++)
{
rr[ri] = newRule.Right[ri];
}
var iid = _LrtSymbol.Parse(f).Id;
(int RuleOrStateId, string Left, string[] Right)tuple;
var rid = cfg.Rules.IndexOf(newRule);
var newTuple = (RuleOrStateId : rid, Left : newRule.Left, Right : rr);
// this gets rid of duplicate entries which crop up in the table
if (!parseTable[lr.To].TryGetValue(iid, out tuple))
{
parseTable[lr.To].Add(_LrtSymbol.Parse(f).Id,
newTuple);
}
else
{
// TODO: Verify this - may need the dragon book
if (null == tuple.Right)
{
var nr = cfg.Rules[rid];
var msg = new CfgMessage(CfgErrorLevel.Warning, -1, string.Format("Shift-Reduce conflict on rule {0}, token {1}", nr, iid), nr.Line, nr.Column, nr.Position);
if (!result.Contains(msg))
{
result.Add(msg);
}
}
else
{
if (rid != newTuple.RuleOrStateId)
{
var nr = cfg.Rules[rid];
var msg = new CfgMessage(CfgErrorLevel.Error, -1, string.Format("Reduce-Reduce conflict on rule {0}, token {1}", nr, iid), nr.Line, nr.Column, nr.Position);
if (!result.Contains(msg))
{
result.Add(msg);
}
}
}
}
}
}
++j;
}
return(result);
}
public static IList <CfgMessage> TryToLalr1ParseTable(this CfgDocument cfg, out CfgLalr1ParseTable parseTable) => TryToLalr1ParseTable(cfg, null, out parseTable);
static int _Lalr1Gen(string[] args)
{
string specFile = null;
string outFile = null;
string @namespace = null;
string @class = null;
string language = null; // "c#";
var optIndex = -1;
for (var i = 0; i < args.Length; ++i)
{
if ("--help" == args[i] || "/?" == args[i] || "/help" == args[i])
{
Console.Error.Write("Usage: ");
_PrintUsageLalr1Gen();
return(0);
}
if (args[i].StartsWith("/"))
{
optIndex = i;
if (i == args.Length - 1)
{
Console.Error.Write("Usage: ");
_PrintUsageLL1Gen();
return(1);
}
switch (args[i])
{
case "/language":
++i;
language = args[i];
break;
case "/namespace":
++i;
@namespace = args[i];
break;
case "/class":
++i;
@class = args[i];
break;
default:
Console.Error.Write("Usage: ");
_PrintUsageLL1Gen();
return(1);
}
}
else
{
if (-1 != optIndex)
{
Console.Error.Write("Usage: ");
_PrintUsageLL1Gen();
return(1);
}
if (0 == i)
{
specFile = args[i];
}
else if (1 == i)
{
outFile = args[i];
}
else
{
Console.Error.Write("Usage: ");
_PrintUsageLL1Gen();
return(1);
}
}
}
TextReader inp = null;
TextWriter outp = null;
try
{
if (null == specFile)
{
inp = Console.In;
}
else
{
inp = new StreamReader(specFile);
}
if (null == outFile)
{
outp = Console.Out;
}
else
{
outp = new StreamWriter(outFile);
}
var buf = inp.ReadToEnd();
var cfg = CfgDocument.Parse(buf);
var hasErrors = false;
foreach (var msg in cfg.TryValidateLalr1())
{
Console.Error.WriteLine(msg);
if (CfgErrorLevel.Error == msg.ErrorLevel)
{
hasErrors = true;
}
}
if (!hasErrors)
{
if (string.IsNullOrEmpty(@class))
{
if (null != outFile)
{
@class = Path.GetFileNameWithoutExtension(outFile);
}
else if (null == @class && 0 < cfg.Rules.Count)
{
@class = string.Concat(cfg.StartSymbol, "Parser");
}
else
{
@class = "Parser";
}
}
Lalr1ParserCodeGenerator.WriteClassTo(cfg, @class, @namespace, language, outp);
return(0);
}
return(1);
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.Message);
return(1);
}
finally
{
if (null != inp)
{
inp.Close();
}
if (null != outp)
{
outp.Close();
}
}
}
static IList <CfgMessage> _TryValidateAttributesLalr1(CfgDocument cfg, IList <CfgMessage> result)
{
if (null == result)
{
result = new List <CfgMessage>();
}
string start = null;
foreach (var attrs in cfg.AttributeSets)
{
if (!cfg.IsSymbol(attrs.Key))
{
// hidden rules should never be in the grammar
// so warnings about them not being in the grammar
// are suppressed.
var i = attrs.Value.IndexOf("hidden");
if (!(-1 < i && attrs.Value[i].Value is bool && ((bool)attrs.Value[i].Value)))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat("Attributes declared on a symbol \"", attrs.Key, "\" that is not in the grammar"), attrs.Value[0].Line, attrs.Value[0].Column, attrs.Value[0].Position));
}
}
foreach (var attr in attrs.Value)
{
string s;
var p = string.Concat("On \"", attrs.Key, "\": ");
switch (attr.Name)
{
case "start":
if (!(attr.Value is bool))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "start attribute expects a bool value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
if (null != start)
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "start attribute was already specified on \"", start, "\" and this declaration will be ignored"), attr.Line, attr.Column, attr.Position));
}
else
{
start = attrs.Key;
}
continue;
case "hidden":
if (!(attr.Value is bool))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "hidden attribute expects a bool value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
continue;
case "terminal":
if (!(attr.Value is bool))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "terminal attribute expects a bool value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
continue;
case "collapsed":
if (!(attr.Value is bool))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "collapse attribute expects a bool value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
continue;
case "substitute":
s = attr.Value as string;
if (!(attr.Value is string))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "substitute attribute expects a string value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
else if (string.IsNullOrEmpty(s))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "substitute attribute expects a non-empty string value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
else if (!cfg.IsSymbol(s))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "substitute attribute expects a symbol reference and will be ignored"), attr.Line, attr.Column, attr.Position));
}
continue;
case "blockEnd":
if (cfg.IsNonTerminal(attrs.Key))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "blockEnd attribute cannot be specified on a non-terminal and will be ignored"), attr.Line, attr.Column, attr.Position));
}
else
{
s = attr.Value as string;
if (!(attr.Value is string))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "blockEnd attribute expects a string value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
else if (string.IsNullOrEmpty(s))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "blockEnd attribute expects a non-empty string value and will be ignored"), attr.Line, attr.Column, attr.Position));
}
}
continue;
case "followsConflict":
s = attr.Value as string;
switch (s)
{
case "error":
case "first":
case "last":
break;
default:
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "followsError attribute expects \"error\", \"first\", or \"last\" and will revert to \"error\"."), attr.Line, attr.Column, attr.Position));
break;
}
continue;
}
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat(p, "Unknown attribute \"", attr.Name, "\" will be ignored"), attr.Line, attr.Column, attr.Position));
}
}
if (null == start)
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat("start attribute was not specified and the first non-terminal in the grammar (\"", cfg.StartSymbol, "\") will be used"), 0, 0, 0));
}
return(result);
}
static int _Fagen(string[] args)
{
string specFile = null;
string outFile = null;
string @namespace = null;
string @class = null;
string language = null; // "c#";
var optIndex = -1;
for (var i = 0; i < args.Length; ++i)
{
if ("--help" == args[i] || "/?" == args[i] || "/help" == args[i])
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(0);
}
if (args[i].StartsWith("/"))
{
optIndex = i;
if (i == args.Length - 1)
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(1);
}
switch (args[i])
{
case "/language":
++i;
language = args[i];
break;
case "/namespace":
++i;
@namespace = args[i];
break;
case "/class":
++i;
@class = args[i];
break;
default:
_PrintUsageFAGen();
return(1);
}
}
else
{
if (-1 != optIndex)
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(1);
}
if (0 == i)
{
specFile = args[i];
}
else if (1 == i)
{
outFile = args[i];
}
else
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(1);
}
}
}
TextReader inp = null;
TextWriter outp = null;
try
{
if (null == specFile)
{
inp = Console.In;
}
else
{
inp = new StreamReader(specFile);
}
if (null == outFile)
{
outp = Console.Out;
}
else
{
outp = new StreamWriter(outFile);
}
var buf = inp.ReadToEnd();
var lex = LexDocument.Parse(buf);
var cfg = CfgDocument.Parse(buf);
var syms = cfg.FillSymbols();
if (string.IsNullOrEmpty(@class))
{
if (null != outFile)
{
@class = Path.GetFileNameWithoutExtension(outFile);
}
else if (0 < syms.Count)
{
foreach (var attrs in lex.AttributeSets)
{
var i = attrs.Value.IndexOf("start");
if (-1 < i && attrs.Value[i].Value is bool && (bool)attrs.Value[i].Value)
{
@class = string.Concat(attrs.Key, "Tokenizer");
break;
}
}
if (null == @class)
{
@class = string.Concat(syms[0], "Tokenizer");
}
}
}
TokenizerCodeGenerator.WriteClassTo(lex, syms, @class, @namespace, language, new _TokenizerConsoleProgress(), outp);
Console.Error.WriteLine();
outp.Flush();
return(0);
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.Message);
return(1);
}
finally
{
inp.Close();
outp.Close();
}
}
public static IList <CfgMessage> TryWriteClassTo(CfgDocument cfg, string name, string @namespace, string language, TextWriter writer) => TryWriteClassTo(cfg, name, @namespace, language, null, writer);
static CfgDocument _ToLRTransitionGrammar(CfgDocument cfg, Lrfa lrfa, IProgress <CfgLalr1Progress> progress)
{
var result = new CfgDocument();
var closure = new List <Lrfa>();
var itemSets = new List <ICollection <LRItem> >();
lrfa.FillClosure(closure);
foreach (var p in closure)
{
itemSets.Add(p.AcceptSymbol);
}
_LrtSymbol start = null;
int j = 0;
foreach (var p in closure)
{
if (null != progress)
{
progress.Report(new CfgLalr1Progress(CfgLalr1Status.CreatingLookaheadGrammar, j));
}
int si = itemSets.IndexOf(p.AcceptSymbol, _LRItemSetComparer.Default);
foreach (var item in p.AcceptSymbol)
{
if (0 == item.RightIndex)
{
var next = item.RightIndex < item.Rule.Right.Count ? item.Rule.Right[item.RightIndex] : null;
var rule = item.Rule;
if (item.RightIndex < item.Rule.Right.Count)
{
int dst = -1;
Lrfa dsts;
if (p.Transitions.ContainsKey(rule.Left))
{
dsts = p.Transitions[rule.Left];
dst = itemSets.IndexOf(dsts.AcceptSymbol, _LRItemSetComparer.Default);
}
_LrtSymbol left = new _LrtSymbol(si, rule.Left, dst);
if (null == start)
{
start = left;
}
var right = new List <string>();
var pc = p;
foreach (var sym in rule.Right)
{
int s1 = itemSets.IndexOf(pc.AcceptSymbol, _LRItemSetComparer.Default);
var pt = pc.Transitions[sym];
int s2 = itemSets.IndexOf(pt.AcceptSymbol, _LRItemSetComparer.Default);
_LrtSymbol n = new _LrtSymbol(s1, sym, s2);
right.Add(n.ToString());
pc = pt;
}
result.Rules.Add(new CfgRule(left.ToString(), right));
}
}
}
++j;
}
result.StartSymbol = start.ToString();
return(result);
}
static IList <CfgMessage> _CreateParserClass(CfgDocument cfg, string name, IProgress <CfgLL1Progress> progress, out CodeTypeDeclaration parserClass)
{
var result = new List <CfgMessage>();
var sm = new Dictionary <string, int>();
var ii = 0;
var syms = new List <string>();
cfg.FillSymbols(syms);
foreach (var sym in syms)
{
sm.Add(sym, ii);
++ii;
}
CfgLL1ParseTable pt;
result.AddRange(cfg.TryToLL1ParseTable(progress, out pt));
var hasErrors = false;
foreach (var msg in result)
{
if (CfgErrorLevel.Error == msg.ErrorLevel)
{
hasErrors = true;
break;
}
}
if (!hasErrors)
{
var ipt = pt.ToArray(syms);
var nodeFlags = new int[syms.Count];
for (var i = 0; i < nodeFlags.Length; ++i)
{
var o = cfg.GetAttribute(syms[i], "hidden", false);
if (o is bool && (bool)o)
{
nodeFlags[i] |= 2;
}
o = cfg.GetAttribute(syms[i], "collapsed", false);
if (o is bool && (bool)o)
{
nodeFlags[i] |= 1;
}
}
var substitutions = new int[syms.Count];
for (var i = 0; i < substitutions.Length; i++)
{
var s = cfg.GetAttribute(syms[i], "substitute", null) as string;
if (!string.IsNullOrEmpty(s) && cfg.IsSymbol(s) && s != syms[i])
{
substitutions[i] = cfg.GetIdOfSymbol(s);
}
else
{
substitutions[i] = -1;
}
}
var attrSets = new KeyValuePair <string, object> [syms.Count][];
for (ii = 0; ii < attrSets.Length; ii++)
{
CfgAttributeList attrs;
if (cfg.AttributeSets.TryGetValue(syms[ii], out attrs))
{
attrSets[ii] = new KeyValuePair <string, object> [attrs.Count];
var j = 0;
foreach (var attr in attrs)
{
attrSets[ii][j] = new KeyValuePair <string, object>(attr.Name, attr.Value);
++j;
}
}
else
{
attrSets[ii] = null;
}
}
parserClass = new CodeTypeDeclaration();
parserClass.Name = name;
parserClass.Attributes = MemberAttributes.FamilyOrAssembly;
parserClass.BaseTypes.Add(typeof(LL1TableParser));
CodeMemberField f;
foreach (var s in syms)
{
if (null != s)
{
f = new CodeMemberField();
f.Attributes = MemberAttributes.Const | MemberAttributes.Public;
f.Name = s.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_");
f.Type = new CodeTypeReference(typeof(int));
f.InitExpression = CodeDomUtility.Serialize(cfg.GetIdOfSymbol(s));
parserClass.Members.Add(f);
}
}
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_Symbols";
f.Type = new CodeTypeReference(typeof(string[]));
f.InitExpression = CodeDomUtility.Serialize(syms.ToArray());
parserClass.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_ParseTable";
f.Type = new CodeTypeReference(typeof(int[][][]));
f.InitExpression = CodeDomUtility.Serialize(ipt);
parserClass.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_InitCfg";
f.Type = new CodeTypeReference(typeof(int[]));
f.InitExpression = CodeDomUtility.Serialize(new int[] { cfg.GetIdOfSymbol(cfg.StartSymbol), cfg.FillNonTerminals().Count });
parserClass.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_NodeFlags";
f.Type = new CodeTypeReference(typeof(int[]));
f.InitExpression = CodeDomUtility.Serialize(nodeFlags);
parserClass.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_Substitutions";
f.Type = new CodeTypeReference(typeof(int[]));
f.InitExpression = CodeDomUtility.Serialize(substitutions);
parserClass.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_AttributeSets";
f.Type = new CodeTypeReference(attrSets.GetType());
f.InitExpression = CodeDomUtility.Serialize(attrSets);
parserClass.Members.Add(f);
var ctor = new CodeConstructor();
ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <Token>), "tokenizer"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(parserClass.Name), "_ParseTable"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(parserClass.Name), "_InitCfg"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(parserClass.Name), "_Symbols"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(parserClass.Name), "_NodeFlags"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(parserClass.Name), "_Substitutions"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(parserClass.Name), "_AttributeSets"),
new CodeArgumentReferenceExpression("tokenizer")
});
ctor.Attributes = MemberAttributes.Public;
parserClass.Members.Add(ctor);
ctor = new CodeConstructor();
ctor.ChainedConstructorArgs.AddRange(new CodeExpression[] { new CodePrimitiveExpression(null) });
ctor.Attributes = MemberAttributes.Public;
parserClass.Members.Add(ctor);
}
else
{
parserClass = null;
}
return(result);
}
public static IList <CfgMessage> TryToLalr1Parser(this CfgDocument cfg, out Lalr1Parser parser, IEnumerable <Token> tokenizer = null, IProgress <CfgLalr1Progress> progress = null)
{
CfgLalr1ParseTable parseTable;
var result = TryToLalr1ParseTable(cfg, progress, out parseTable);
var syms = new List <string>();
cfg.FillSymbols(syms);
var nodeFlags = new int[syms.Count];
for (var i = 0; i < nodeFlags.Length; i++)
{
var o = cfg.GetAttribute(syms[i], "hidden", false);
if (o is bool && (bool)o)
{
nodeFlags[i] |= 2;
}
o = cfg.GetAttribute(syms[i], "collapsed", false);
if (o is bool && (bool)o)
{
nodeFlags[i] |= 1;
}
}
var substitutions = new int[syms.Count];
for (var i = 0; i < substitutions.Length; i++)
{
var s = cfg.GetAttribute(syms[i], "substitute", null) as string;
if (!string.IsNullOrEmpty(s) && cfg.IsSymbol(s) && s != syms[i])
{
substitutions[i] = cfg.GetIdOfSymbol(s);
}
else
{
substitutions[i] = -1;
}
}
var attrSets = new KeyValuePair <string, object> [syms.Count][];
for (var i = 0; i < attrSets.Length; i++)
{
CfgAttributeList attrs;
if (cfg.AttributeSets.TryGetValue(syms[i], out attrs))
{
attrSets[i] = new KeyValuePair <string, object> [attrs.Count];
var j = 0;
foreach (var attr in attrs)
{
attrSets[i][j] = new KeyValuePair <string, object>(attr.Name, attr.Value);
++j;
}
}
else
{
attrSets[i] = null; // new KeyValuePair<string, object>[0];
}
}
var ss = cfg.GetIdOfSymbol(cfg.StartSymbol);
var ntc = cfg.FillNonTerminals().Count;
parser = new Lalr1TableParser(parseTable.ToArray(syms), syms.ToArray(), nodeFlags, substitutions, attrSets, tokenizer);
return(result);
}
public static void WriteClassTo(CfgDocument cfg, string name, string @namespace, string language, TextWriter writer) => WriteClassTo(cfg, name, @namespace, language, null, writer);
static IList <CfgMessage> _TryValidateRulesLalr1(CfgDocument cfg, IList <CfgMessage> result)
{
if (null == result)
{
result = new List <CfgMessage>();
}
var ic = cfg.Rules.Count;
if (0 == ic)
{
result.Add(new CfgMessage(CfgErrorLevel.Error, -1, "Grammar has no rules", 0, 0, 0));
}
var dups = new HashSet <CfgRule>();
for (var i = 0; i < ic; ++i)
{
var rule = cfg.Rules[i];
if (rule.Left.IsNullOrEmpty())
{
result.Add(new CfgMessage(CfgErrorLevel.Error, -1, string.Concat("Rule has empty left hand side:", rule.ToString()), rule.Line, rule.Column, rule.Position));
}
else if ("#ERROR" == rule.Left || "#EOS" == rule.Left)
{
result.Add(new CfgMessage(CfgErrorLevel.Error, -1, string.Concat("Rule has reserved terminal on left hand side: ", rule.ToString()), rule.Line, rule.Column, rule.Position));
}
for (int jc = rule.Right.Count, j = 0; j > jc; ++j)
{
if (rule.Right[j].IsNullOrEmpty())
{
result.Add(new CfgMessage(CfgErrorLevel.Error, -1, string.Concat("Rule has empty symbols on the right hand side:", rule.ToString()), rule.Line, rule.Column, rule.Position));
}
else if ("#ERROR" == rule.Right[j] || "#EOS" == rule.Right[j])
{
result.Add(new CfgMessage(CfgErrorLevel.Error, -1, string.Concat("Rule has reserved terminal on right hand side:", rule.ToString()), rule.Line, rule.Column, rule.Position));
}
}
for (var j = 0; j < ic; ++j)
{
if (i != j && cfg.Rules[j] == rule && dups.Add(rule))
{
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat("Duplicate rule:", rule.ToString()), rule.Line, rule.Column, rule.Position));
}
}
}
var closure = cfg.FillClosure(cfg.StartSymbol);
var syms = cfg.FillSymbols();
ic = syms.Count;
for (var i = 0; i < ic; ++i)
{
var sym = syms[i];
if (!closure.Contains(sym))
{
var found = false;
if (!cfg.IsNonTerminal(sym))
{
if ("#EOS" == sym || "#ERROR" == sym || (bool)cfg.GetAttribute(sym, "hidden", false))
{
found = true;
}
}
if (!found)
{
var rl = cfg.FillNonTerminalRules(sym);
if (0 < rl.Count)
{
var r = rl[0];
result.Add(new CfgMessage(CfgErrorLevel.Warning, -1, string.Concat("Unreachable symbol \"", sym, "\""), r.Line, r.Column, r.Position));
}
}
}
}
// checking for conflicts here is way too time prohibitive for LALR(1) so we skip it
return(result);
}
public static void WriteClassTo(CfgDocument cfg, string name, string @namespace, string language, IProgress <CfgLL1Progress> progress, TextWriter writer)
{
var msgs = TryWriteClassTo(cfg, name, @namespace, language, progress, writer);
CfgException.ThrowIfErrors(msgs);
}
