public static IList <IMessage> TryCreateGenerationInfo(XbnfDocument document, out XbnfGenerationInfo genInfo)
{
var includes = new XbnfImportList();
_GatherIncludes(document, includes);
var incs = new List <XbnfDocument>();
incs.Add(document);
foreach (var inc in includes)
{
incs.Add(inc.Document);
}
var doc = XbnfDocument.Merge(incs);
var cfg = new CfgDocument();
return(_TryToGenInfo(doc, cfg, out genInfo));
}
public _TermPriorityComparer(CfgDocument cfg)
{
_cfg = cfg;
}
static IList <IMessage> _TryToGenInfo(XbnfDocument document, CfgDocument cfg, out XbnfGenerationInfo genInfo)
{
genInfo = default(XbnfGenerationInfo);
var hasErrors = false;
var result = new List <IMessage>();
var syms = new HashSet <string>();
// gather the attributes and production names
for (int ic = document.Productions.Count, i = 0; i < ic; ++i)
{
var p = document.Productions[i];
if (!syms.Add(p.Name))
{
result.Add(new XbnfMessage(ErrorLevel.Error, -1, string.Format("Duplicate production {0} defined.", p.Name), p.Line, p.Column, p.Position, document.FileOrUrl));
hasErrors = true;
}
if (0 < p.Attributes.Count)
{
CfgAttributeList attrs;
if (!cfg.AttributeSets.TryGetValue(p.Name, out attrs))
{
attrs = new CfgAttributeList();
cfg.AttributeSets.Add(p.Name, attrs);
}
for (int jc = p.Attributes.Count, j = 0; j < jc; ++j)
{
var attr = p.Attributes[j];
attrs.Add(new CfgAttribute(attr.Name, attr.Value));
}
}
}
// use a list dictionary to keep these in order
var tmap = new ListDictionary <XbnfExpression, string>();
var attrSets = new Dictionary <string, XbnfAttributeList>();
var rules = new List <KeyValuePair <string, IList <string> > >();
// below are scratch
var working = new HashSet <XbnfExpression>();
var done = new HashSet <XbnfExpression>();
// now get the terminals and their ids, declaring if necessary
for (int ic = document.Productions.Count, i = 0; i < ic; ++i)
{
var p = document.Productions[i];
if (p.IsTerminal)
{
string name;
if (!tmap.TryGetValue(p.Expression, out name))
{
tmap.Add(p.Expression, p.Name);
}
else
{
if (name != p.Name)
{
result.Add(new CfgMessage(ErrorLevel.Error, -1, string.Format("{0} attempts to redefine terminal {1}", name, p.Name), p.Line, p.Column, p.Position, document.FileOrUrl));
hasErrors = true;
}
}
done.Add(p.Expression);
}
else
{
_VisitFetchTerminals(p.Expression, working);
}
}
if (hasErrors)
{
return(result);
}
foreach (var term in working)
{
if (!done.Contains(term))
{
var newId = _GetImplicitTermId(syms);
var found = false;
var prod = document.GetProductionForExpression(term);
if (null != prod)
{
found = true;
// recycle this symbol
newId = prod.Name;
}
if (!found)
{
document.Productions.Add(new XbnfProduction(newId, term));
}
tmap.Add(term, newId);
}
}
// tmap now contains ALL of our terminal definitions from all of our imports
// now we can use tmap and syms to help solve the rest of our productions
var ntd = new Dictionary <string, IList <IList <string> > >();
for (int ic = document.Productions.Count, i = 0; i < ic; ++i)
{
var p = document.Productions[i];
if (!p.IsTerminal)
{
var dys = _GetDysjunctions(document, syms, tmap, attrSets, rules, p, p.Expression);
IList <IList <string> > odys;
if (ntd.TryGetValue(p.Name, out odys))
{
result.Add(new XbnfMessage(ErrorLevel.Error, -1, string.Format("The {0} production was specified more than once", p.Name), p.Line, p.Column, p.Position, document.FileOrUrl));
hasErrors = true;
}
ntd.Add(p.Name, dys);
if (hasErrors)
{
return(result);
}
}
}
// now that we've done that, build the rest of our attributes
foreach (var sattrs in attrSets)
{
CfgAttributeList attrs;
if (!cfg.AttributeSets.TryGetValue(sattrs.Key, out attrs))
{
attrs = new CfgAttributeList();
cfg.AttributeSets.Add(sattrs.Key, attrs);
}
for (int jc = sattrs.Value.Count, j = 0; j < jc; ++j)
{
var attr = sattrs.Value[j];
attrs.Add(new CfgAttribute(attr.Name, attr.Value));
}
}
// now write our main rules
foreach (var nt in ntd)
{
foreach (var l in nt.Value)
{
cfg.Rules.Add(new CfgRule(nt.Key, l));
}
}
// build our secondary rules
foreach (var rule in rules)
{
cfg.Rules.Add(new CfgRule(rule.Key, rule.Value));
}
if (hasErrors)
{
return(result);
}
genInfo.Xbnf = document;
genInfo.TerminalMap = tmap;
genInfo.Cfg = cfg;
cfg.RebuildCache();
return(result);
}
CfgDocument _LRFAToLRExtendedGrammar(_LRFA lrfa, IProgress <CfgLRProgress> progress)
{
var result = new CfgDocument();
var closure = new List <_LRFA>();
var itemSets = new List <_LR0ItemSet>();
lrfa.FillClosure(closure);
foreach (var p in closure)
{
itemSets.Add(p.Accept);
}
_LRExtendedSymbol start = null;
int j = 0;
foreach (var p in closure)
{
if (null != progress)
{
progress.Report(new CfgLRProgress(CfgLRStatus.CreatingLRExtendedGrammar, j));
}
int si = itemSets.IndexOf(p.Accept);
foreach (var item in p.Accept.Items)
{
if (0 == item.RightIndex)
{
var next = item.Next;
if (!item.IsEnd || item.IsNil)
{
int dst = -1;
_LRFA dsts;
if (p.Transitions.ContainsKey(item.Left))
{
dsts = p.Transitions[item.Left];
dst = itemSets.IndexOf(dsts.Accept);
}
_LRExtendedSymbol left = new _LRExtendedSymbol(si, item.Left, dst);
if (null == start)
{
start = left;
}
var right = new List <string>();
var pc = p;
foreach (var sym in item.Right)
{
int s1 = itemSets.IndexOf(pc.Accept);
var pt = pc.Transitions[sym];
int s2 = itemSets.IndexOf(pt.Accept);
_LRExtendedSymbol n = new _LRExtendedSymbol(s1, sym, s2);
right.Add(n.ToString());
pc = pt;
}
result.Rules.Add(new CfgRule(left.ToString(), right));
}
}
}
++j;
}
result.StartSymbol = start.ToString();
return(result);
}
/*static int Test()
* {
* string input;
* using (var sr = new StreamReader(@"..\..\data2.json"))
* input = sr.ReadToEnd();
* var tokenizer = new JsonTokenizer(input);
* var xbnf = XbnfDocument.ReadFrom(@"..\..\json.xbnf");
* XbnfGenerationInfo info;
* XbnfConvert.TryCreateGenerationInfo(xbnf, out info);
* int ts;
* var symbols =XbnfConvert.GetSymbolTable(info, out ts);
* CfgGlrParseTable parseTable;
* info.Cfg.RebuildCache();
* info.Cfg.TryToGlrParseTable(out parseTable, LRTableKind.Lalr1);
* var errorSentinels = new List<int>();
* var i = 0;
* var parseAttributes = new ParseAttribute[symbols.Count][];
* foreach(var attrs in info.Cfg.AttributeSets)
* {
* var id = symbols.IndexOf(attrs.Key);
* int jc = attrs.Value.Count;
* parseAttributes[id] = new ParseAttribute[jc];
* for(var j=0;j<jc;++j)
* {
* var attr = attrs.Value[j];
* parseAttributes[id][j] = new ParseAttribute(attr.Name, attr.Value);
* if ("errorSentinel" == attr.Name && attr.Value is bool && ((bool)attr.Value))
* errorSentinels.Add(id);
*
* }
++i;
* }
* for (i = 0; i < parseAttributes.Length; i++)
* if (null == parseAttributes[i])
* parseAttributes[i] = new ParseAttribute[0];
*
* var parser = new GlrTableParser(parseTable.ToArray(symbols), symbols.ToArray(), parseAttributes, errorSentinels.ToArray(),tokenizer);
* foreach (var pt in parser.ParseReductions(false, true, false))
* {
* Console.WriteLine(pt.ToString("t"));
* }
* return 0;
* }*/
public static int Run(string[] args, TextReader stdin, TextWriter stdout, TextWriter stderr)
{
int result = 0;
TextWriter output = null;
string inputfile = null;
string outputfile = null;
string rolexfile = null;
string codenamespace = null;
string codelanguage = null;
string codeclass = null;
string yaccfile = null;
bool verbose = false;
bool noshared = false;
bool ifstale = false;
bool fast = false;
bool noparser = false;
try
{
if (0 == args.Length)
{
_PrintUsage(stderr);
return(-1);
}
if (args[0].StartsWith("/"))
{
throw new ArgumentException("Missing input file.");
}
// process the command line args
inputfile = args[0];
for (var i = 1; i < args.Length; ++i)
{
switch (args[i])
{
case "/namespace":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
codenamespace = args[i];
break;
case "/class":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
codeclass = args[i];
break;
case "/language":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
codelanguage = args[i];
break;
case "/output":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
outputfile = args[i];
break;
case "/yacc":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
yaccfile = args[i];
break;
case "/ifstale":
ifstale = true;
break;
case "/fast":
fast = true;
break;
case "/noparser":
noparser = true;
break;
case "/noshared":
noshared = true;
break;
case "/verbose":
verbose = true;
break;
case "/rolex":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
rolexfile = args[i];
break;
default:
throw new ArgumentException(string.Format("Unknown switch {0}", args[i]));
}
}
if (null != outputfile && noparser)
{
throw new ArgumentException("<noparser> and <ouputfile> cannot both be specified.", "outputfile");
}
if (null == codeclass)
{
if (null != outputfile)
{
codeclass = Path.GetFileNameWithoutExtension(outputfile);
}
else
{
codeclass = Path.GetFileNameWithoutExtension(inputfile);
}
}
// override the options with our document's options
var doc = XbnfDocument.ReadFrom(inputfile);
var oi = -1;
oi = doc.Options.IndexOf("outputfile");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
outputfile = s;
if ("" == outputfile)
{
outputfile = null;
}
}
// if it's specified in the doc we need to make it doc relative
if (null != outputfile)
{
if (!Path.IsPathRooted(outputfile))
{
var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
outputfile = Path.GetFullPath(Path.Combine(dir, outputfile));
}
}
}
oi = doc.Options.IndexOf("rolexfile");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
rolexfile = s;
if ("" == rolexfile)
{
rolexfile = null;
}
}
// if it's specified in the doc we need to make it doc relative
if (null != rolexfile)
{
if (!Path.IsPathRooted(rolexfile))
{
var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
rolexfile = Path.GetFullPath(Path.Combine(dir, rolexfile));
}
}
}
oi = doc.Options.IndexOf("yaccfile");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
rolexfile = s;
if ("" == yaccfile)
{
yaccfile = null;
}
}
// if it's specified in the doc we need to make it doc relative
if (null != yaccfile)
{
if (!Path.IsPathRooted(yaccfile))
{
var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
rolexfile = Path.GetFullPath(Path.Combine(dir, yaccfile));
}
}
}
oi = doc.Options.IndexOf("codenamespace");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
codenamespace = s;
}
}
oi = doc.Options.IndexOf("codelanguage");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (!string.IsNullOrEmpty(s))
{
codelanguage = s;
}
}
oi = doc.Options.IndexOf("codeclass");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
codeclass = s;
if ("" == codeclass)
{
if (null != outputfile)
{
codeclass = Path.GetFileNameWithoutExtension(outputfile);
}
else
{
codeclass = Path.GetFileNameWithoutExtension(inputfile);
}
}
}
}
oi = doc.Options.IndexOf("verbose");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
if (o is bool)
{
verbose = (bool)o;
}
}
oi = doc.Options.IndexOf("fast");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
if (o is bool)
{
fast = (bool)o;
}
}
if (fast && null != codelanguage)
{
throw new ArgumentException("<codelanguage> and <fast> cannot both be specified. The <fast> option is C# only.");
}
var stale = true;
if (ifstale)
{
stale = false;
if (!stale && null != rolexfile)
{
if (_IsStale(inputfile, rolexfile))
{
stale = true;
}
}
if (!stale && null != yaccfile)
{
if (_IsStale(inputfile, yaccfile))
{
stale = true;
}
}
if (!stale)
{
var files = XbnfDocument.GetResources(inputfile);
foreach (var s in files)
{
if (_IsStale(s, outputfile))
{
stale = true;
break;
}
}
}
// see if our exe has changed
if (!stale && null != outputfile && _IsStale(CodeBase, outputfile))
{
stale = true;
}
}
if (!stale)
{
stderr.WriteLine("Skipped building of the following because they were not stale:");
if (null != outputfile)
{
stderr.WriteLine("Output file: " + outputfile);
}
if (null != rolexfile)
{
stderr.WriteLine("Rolex file: " + rolexfile);
}
if (null != yaccfile)
{
stderr.WriteLine("YACC file: " + yaccfile);
}
}
else
{
stderr.WriteLine("{0} is building the following:", Name);
if (null != outputfile)
{
stderr.WriteLine("Output file: " + outputfile);
}
if (null != rolexfile)
{
stderr.WriteLine("Rolex file: " + rolexfile);
}
if (null != yaccfile)
{
stderr.WriteLine("YACC file: " + yaccfile);
}
if (string.IsNullOrEmpty(codelanguage))
{
if (!string.IsNullOrEmpty(outputfile))
{
codelanguage = Path.GetExtension(outputfile);
if (codelanguage.StartsWith("."))
{
codelanguage = codelanguage.Substring(1);
}
}
if (string.IsNullOrEmpty(codelanguage))
{
codelanguage = "cs";
}
}
var isLexerOnly = true;
if (doc.HasNonTerminalProductions)
{
isLexerOnly = false;
}
else
{
foreach (var include in doc.Includes)
{
if (include.Document.HasNonTerminalProductions)
{
isLexerOnly = false;
break;
}
}
}
// we need to prepare it by marking every terminal
// with an attribute if it isn't already. we use
// "terminal" because it doesn't impact terminals
// in any way, but this way the CfgDocument can
// "see" them.
for (int ic = doc.Productions.Count, i = 0; i < ic; ++i)
{
var p = doc.Productions[i];
if (p.IsTerminal && 0 == p.Attributes.Count)
{
p.Attributes.Add(new XbnfAttribute("terminal", true));
}
}
XbnfGenerationInfo genInfo;
var msgs = XbnfConvert.TryCreateGenerationInfo(doc, out genInfo);
foreach (var msg in msgs)
{
if (verbose || ErrorLevel.Information != msg.ErrorLevel)
{
stderr.WriteLine(msg);
}
}
foreach (var msg in msgs)
{
if (msg.ErrorLevel == ErrorLevel.Error)
{
throw new Exception(msg.ToString());
}
}
CfgDocument primaryCfg = genInfo.Cfg;
doc = genInfo.Xbnf;
if (!isLexerOnly)
{
if (verbose)
{
stderr.WriteLine("Final grammar:");
stderr.WriteLine(primaryCfg.ToString());
stderr.WriteLine();
}
foreach (var msg in msgs)
{
if (msg.ErrorLevel == ErrorLevel.Error)
{
throw new Exception(msg.ToString());
}
}
if (!noparser)
{
var ccu = CodeGenerator.GenerateCompileUnit(genInfo, codeclass, codenamespace, fast);
ccu.Namespaces.Add(new CodeNamespace(codenamespace ?? ""));
var ccuNS = ccu.Namespaces[ccu.Namespaces.Count - 1];
var ccuShared = CodeGenerator.GenerateSharedCompileUnit(codenamespace);
ccu.ReferencedAssemblies.Add(typeof(TypeConverter).Assembly.GetName().ToString());
if (fast)
{
CD.CodeDomVisitor.Visit(ccu, (ctx) =>
{
var vd = ctx.Target as CodeVariableDeclarationStatement;
if (null != vd && CD.CodeDomResolver.IsNullOrVoidType(vd.Type))
{
vd.Type = C.Type("var");
}
}, CD.CodeDomVisitTargets.All & ~(CD.CodeDomVisitTargets.Expressions | CD.CodeDomVisitTargets.Comments | CD.CodeDomVisitTargets.Attributes | CD.CodeDomVisitTargets.Directives | CD.CodeDomVisitTargets.Types | CD.CodeDomVisitTargets.TypeRefs));
CD.CodeDomVisitor.Visit(ccuShared, (ctx) =>
{
var vd = ctx.Target as CodeVariableDeclarationStatement;
if (null != vd && CD.CodeDomResolver.IsNullOrVoidType(vd.Type))
{
vd.Type = C.Type("var");
}
}, CD.CodeDomVisitTargets.All & ~(CD.CodeDomVisitTargets.Expressions | CD.CodeDomVisitTargets.Comments | CD.CodeDomVisitTargets.Attributes | CD.CodeDomVisitTargets.Directives | CD.CodeDomVisitTargets.Types | CD.CodeDomVisitTargets.TypeRefs));
}
else
{
SlangPatcher.Patch(ccu, ccuShared);
var co = SlangPatcher.GetNextUnresolvedElement(ccu);
if (null != co)
{
stderr.WriteLine("Warning: Not all of the elements could be resolved. The generated code may not be correct in all languages.");
stderr.WriteLine(" Next unresolved: {0}", C.ToString(co).Trim());
}
}
if (!noshared)
{
CodeGenerator.ImportCompileUnit(ccuNS, ccuShared);
}
var prov = CodeDomProvider.CreateProvider(codelanguage);
if (null != outputfile)
{
var sw = new StreamWriter(outputfile);
sw.BaseStream.SetLength(0);
output = sw;
}
else
{
output = stdout;
}
var opts = new CodeGeneratorOptions();
opts.VerbatimOrder = true;
opts.BlankLinesBetweenMembers = false;
prov.GenerateCodeFromCompileUnit(ccu, output, opts);
output.Flush();
output.Close();
output = null;
}
}
else
{
stderr.WriteLine("{0} skipped parser generation because there are no non-terminals and no imports defined.", Name);
}
if (null != rolexfile)
{
var sw = new StreamWriter(rolexfile);
sw.BaseStream.SetLength(0);
output = sw;
output.WriteLine(XbnfConvert.ToRolexSpec(genInfo));
output.Flush();
output.Close();
output = null;
}
if (null != yaccfile)
{
var sw = new StreamWriter(yaccfile);
sw.BaseStream.SetLength(0);
output = sw;
output.WriteLine(genInfo.Cfg.ToString("y"));
output.Flush();
output.Close();
output = null;
}
}
}
#if !DEBUG
catch (Exception ex)
{
result = _ReportError(ex, stderr);
}
#endif
finally
{
stderr.Close();
stdout.Close();
if (outputfile != null && null != output)
{
output.Close();
}
}
return(result);
}