public static CodeCompileUnit GenerateCompileUnit(XbnfGenerationInfo genInfo, string codeclass, string codenamespace, bool fast)
{
var result = new CodeCompileUnit();
var ns = new CodeNamespace();
if (!string.IsNullOrEmpty(codenamespace))
{
ns.Name = codenamespace;
}
result.Namespaces.Add(ns);
ns.Imports.Add(new CodeNamespaceImport("System.Collections.Generic"));
var td = C.Class(codeclass, false);
td.CustomAttributes.Add(GeneratedCodeAttribute);
td.BaseTypes.Add(C.Type("GlrTableParser"));
// GlrTableParser(int[][][][] parseTable,string[] symbolTable, ParseAttribute[][] attributes,int[] errorSentinels, IEnumerable<Token> tokenizer,int maxErrorCount)
td.Members.Add(C.Field(C.Type(typeof(int[][][][])), "ParseTable", MemberAttributes.FamilyAndAssembly | MemberAttributes.Static));
td.Members.Add(C.Field(C.Type(typeof(string[])), "SymbolTable", MemberAttributes.FamilyAndAssembly | MemberAttributes.Static));
td.Members.Add(C.Field(C.Type(C.Type("ParseAttribute", 1), 1), "ParseAttributes", MemberAttributes.FamilyAndAssembly | MemberAttributes.Static));
td.Members.Add(C.Field(C.Type(typeof(int[])), "ErrorSentinels", MemberAttributes.FamilyAndAssembly | MemberAttributes.Static));
//td.Members.Add(C.Field(C.Type(typeof(int[])), "NodeFlags", MemberAttributes.FamilyAndAssembly | MemberAttributes.Static));
var et = C.Type("IEnumerable");
et.TypeArguments.Add(C.Type("Token"));
var ctor = C.Ctor(MemberAttributes.Public, C.Param(et, "tokenizer"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
C.FieldRef(C.TypeRef(codeclass), "ParseTable"),
C.FieldRef(C.TypeRef(codeclass), "SymbolTable"),
C.FieldRef(C.TypeRef(codeclass), "ParseAttributes"),
C.FieldRef(C.TypeRef(codeclass), "ErrorSentinels"),
//C.FieldRef(C.TypeRef(codeclass), "NodeFlags"),
C.ArgRef("tokenizer"),
C.FieldRef(C.TypeRef(typeof(int)), "MaxValue")
});
td.Members.Add(ctor);
ctor = C.Ctor(MemberAttributes.Public, C.Param(et, "tokenizer"), C.Param(typeof(int), "maxErrorCount"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
C.FieldRef(C.TypeRef(codeclass), "ParseTable"),
C.FieldRef(C.TypeRef(codeclass), "SymbolTable"),
C.FieldRef(C.TypeRef(codeclass), "ParseAttributes"),
C.FieldRef(C.TypeRef(codeclass), "ErrorSentinels"),
//C.FieldRef(C.TypeRef(codeclass), "NodeFlags"),
C.ArgRef("tokenizer"),
C.ArgRef("maxErrorCount")
});
td.Members.Add(ctor);
ns.Types.Add(td);
CfgGlrParseTable pt;
genInfo.Cfg.TryToGlrParseTable(out pt);
int ts;
var syms = XbnfConvert.GetSymbolTable(genInfo, out ts);
(C.GetByName("ParseTable", td.Members) as CodeMemberField).InitExpression = C.Literal(pt.ToArray(syms));
(C.GetByName("SymbolTable", td.Members) as CodeMemberField).InitExpression = C.Literal(syms.ToArray());
(C.GetByName("ParseAttributes", td.Members) as CodeMemberField).InitExpression = _SerializeParseAttributes(genInfo, syms);
(C.GetByName("ErrorSentinels", td.Members) as CodeMemberField).InitExpression = _SerializeErrorSentinels(genInfo, syms);
//(C.GetByName("NodeFlags", td.Members) as CodeMemberField).InitExpression = _SerializeNodeFlags(genInfo,syms);
foreach (var code in genInfo.Xbnf.Code)
{
td.Members.AddRange(SlangParser.ParseMembers(code.Value, code.Line, code.Column, code.Position));
}
var hasChangeType = false;
var hasEvalAny = false;
foreach (var prod in genInfo.Xbnf.Productions)
{
if (null != prod.Action)
{
var hasEA = false;
var hasCT = false;
_GenerateAction(genInfo, syms, td, prod, fast, out hasCT, out hasEA);
if (hasCT)
{
hasChangeType = true;
}
if (hasEA)
{
hasEvalAny = true;
}
}
}
var consts = new string[syms.Count];
for (int ic = syms.Count, i = 0; i < ic; ++i)
{
var s = syms[i];
if ("#ERROR" == s)
{
s = "ErrorSymbol";
}
else if ("#EOS" == s)
{
s = "EosSymbol";
}
s = _MakeSafeName(s);
s = _MakeUniqueMember(td, s);
consts[i] = s;
td.Members.Add(C.Field(typeof(int), s, MemberAttributes.Const | MemberAttributes.Public, C.Literal(i)));
}
if (hasChangeType)
{
var m = C.Method(C.Type(typeof(object)), "_ChangeType", MemberAttributes.Static | MemberAttributes.Private, C.Param(typeof(object), "obj"), C.Param(typeof(Type), "type"));
m.Statements.Add(C.Var(typeof(TypeConverter), "typeConverter", C.Invoke(C.TypeRef(typeof(TypeDescriptor)), "GetConverter", C.ArgRef("obj"))));
// if(null!=typeConverter || !typeConverter.CanConvertTo(type))
m.Statements.Add(C.If(C.Or(C.IdentEq(C.Null, C.VarRef("typeConverter")), C.Not(C.Invoke(C.VarRef("typeConverter"), "CanConvertTo", C.ArgRef("type")))),
C.Return(C.Invoke(C.TypeRef(typeof(Convert)), "ChangeType", C.ArgRef("obj"), C.ArgRef("type")))
));
m.Statements.Add(C.Return(C.Invoke(C.VarRef("typeConverter"), "ConvertTo", C.ArgRef("obj"), C.ArgRef("type"))));
td.Members.Add(m);
}
if (hasEvalAny)
{
var sid = C.PropRef(C.ArgRef("node"), "SymbolId");
var m = C.Method(typeof(object), "_EvaluateAny", MemberAttributes.Private | MemberAttributes.Static, C.Param(C.Type("ParseNode"), "node"), C.Param(typeof(object), "state"));
for (int ic = genInfo.Xbnf.Productions.Count, i = 0; i < ic; ++i)
{
var p = genInfo.Xbnf.Productions[i];
if (!p.IsCollapsed && !p.IsHidden)
{
var sidcmp = syms.IndexOf(p.Name);
var sidcf = C.FieldRef(C.TypeRef(td.Name), consts[sidcmp]);
var cnd = C.If(C.Eq(sid, sidcf));
if (!p.IsTerminal)
{
cnd.TrueStatements.Add(C.Return(C.Invoke(C.TypeRef(td.Name), string.Concat("Evaluate", p.Name), C.ArgRef("node"), C.ArgRef("state"))));
}
else
{
cnd.TrueStatements.Add(C.Return(C.PropRef(C.ArgRef("node"), "Value")));
}
m.Statements.Add(cnd);
}
}
m.Statements.Add(C.Return(C.Null));
td.Members.Add(m);
}
return(result);
}
/*static int Test()
* {
* string input;
* using (var sr = new StreamReader(@"..\..\data2.json"))
* input = sr.ReadToEnd();
* var tokenizer = new JsonTokenizer(input);
* var xbnf = XbnfDocument.ReadFrom(@"..\..\json.xbnf");
* XbnfGenerationInfo info;
* XbnfConvert.TryCreateGenerationInfo(xbnf, out info);
* int ts;
* var symbols =XbnfConvert.GetSymbolTable(info, out ts);
* CfgGlrParseTable parseTable;
* info.Cfg.RebuildCache();
* info.Cfg.TryToGlrParseTable(out parseTable, LRTableKind.Lalr1);
* var errorSentinels = new List<int>();
* var i = 0;
* var parseAttributes = new ParseAttribute[symbols.Count][];
* foreach(var attrs in info.Cfg.AttributeSets)
* {
* var id = symbols.IndexOf(attrs.Key);
* int jc = attrs.Value.Count;
* parseAttributes[id] = new ParseAttribute[jc];
* for(var j=0;j<jc;++j)
* {
* var attr = attrs.Value[j];
* parseAttributes[id][j] = new ParseAttribute(attr.Name, attr.Value);
* if ("errorSentinel" == attr.Name && attr.Value is bool && ((bool)attr.Value))
* errorSentinels.Add(id);
*
* }
++i;
* }
* for (i = 0; i < parseAttributes.Length; i++)
* if (null == parseAttributes[i])
* parseAttributes[i] = new ParseAttribute[0];
*
* var parser = new GlrTableParser(parseTable.ToArray(symbols), symbols.ToArray(), parseAttributes, errorSentinels.ToArray(),tokenizer);
* foreach (var pt in parser.ParseReductions(false, true, false))
* {
* Console.WriteLine(pt.ToString("t"));
* }
* return 0;
* }*/
public static int Run(string[] args, TextReader stdin, TextWriter stdout, TextWriter stderr)
{
int result = 0;
TextWriter output = null;
string inputfile = null;
string outputfile = null;
string rolexfile = null;
string codenamespace = null;
string codelanguage = null;
string codeclass = null;
string yaccfile = null;
bool verbose = false;
bool noshared = false;
bool ifstale = false;
bool fast = false;
bool noparser = false;
try
{
if (0 == args.Length)
{
_PrintUsage(stderr);
return(-1);
}
if (args[0].StartsWith("/"))
{
throw new ArgumentException("Missing input file.");
}
// process the command line args
inputfile = args[0];
for (var i = 1; i < args.Length; ++i)
{
switch (args[i])
{
case "/namespace":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
codenamespace = args[i];
break;
case "/class":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
codeclass = args[i];
break;
case "/language":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
codelanguage = args[i];
break;
case "/output":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
outputfile = args[i];
break;
case "/yacc":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
yaccfile = args[i];
break;
case "/ifstale":
ifstale = true;
break;
case "/fast":
fast = true;
break;
case "/noparser":
noparser = true;
break;
case "/noshared":
noshared = true;
break;
case "/verbose":
verbose = true;
break;
case "/rolex":
if (args.Length - 1 == i) // check if we're at the end
{
throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
}
++i; // advance
rolexfile = args[i];
break;
default:
throw new ArgumentException(string.Format("Unknown switch {0}", args[i]));
}
}
if (null != outputfile && noparser)
{
throw new ArgumentException("<noparser> and <ouputfile> cannot both be specified.", "outputfile");
}
if (null == codeclass)
{
if (null != outputfile)
{
codeclass = Path.GetFileNameWithoutExtension(outputfile);
}
else
{
codeclass = Path.GetFileNameWithoutExtension(inputfile);
}
}
// override the options with our document's options
var doc = XbnfDocument.ReadFrom(inputfile);
var oi = -1;
oi = doc.Options.IndexOf("outputfile");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
outputfile = s;
if ("" == outputfile)
{
outputfile = null;
}
}
// if it's specified in the doc we need to make it doc relative
if (null != outputfile)
{
if (!Path.IsPathRooted(outputfile))
{
var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
outputfile = Path.GetFullPath(Path.Combine(dir, outputfile));
}
}
}
oi = doc.Options.IndexOf("rolexfile");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
rolexfile = s;
if ("" == rolexfile)
{
rolexfile = null;
}
}
// if it's specified in the doc we need to make it doc relative
if (null != rolexfile)
{
if (!Path.IsPathRooted(rolexfile))
{
var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
rolexfile = Path.GetFullPath(Path.Combine(dir, rolexfile));
}
}
}
oi = doc.Options.IndexOf("yaccfile");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
rolexfile = s;
if ("" == yaccfile)
{
yaccfile = null;
}
}
// if it's specified in the doc we need to make it doc relative
if (null != yaccfile)
{
if (!Path.IsPathRooted(yaccfile))
{
var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
rolexfile = Path.GetFullPath(Path.Combine(dir, yaccfile));
}
}
}
oi = doc.Options.IndexOf("codenamespace");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
codenamespace = s;
}
}
oi = doc.Options.IndexOf("codelanguage");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (!string.IsNullOrEmpty(s))
{
codelanguage = s;
}
}
oi = doc.Options.IndexOf("codeclass");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
var s = o as string;
if (null != s)
{
codeclass = s;
if ("" == codeclass)
{
if (null != outputfile)
{
codeclass = Path.GetFileNameWithoutExtension(outputfile);
}
else
{
codeclass = Path.GetFileNameWithoutExtension(inputfile);
}
}
}
}
oi = doc.Options.IndexOf("verbose");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
if (o is bool)
{
verbose = (bool)o;
}
}
oi = doc.Options.IndexOf("fast");
if (-1 < oi)
{
var o = doc.Options[oi].Value;
if (o is bool)
{
fast = (bool)o;
}
}
if (fast && null != codelanguage)
{
throw new ArgumentException("<codelanguage> and <fast> cannot both be specified. The <fast> option is C# only.");
}
var stale = true;
if (ifstale)
{
stale = false;
if (!stale && null != rolexfile)
{
if (_IsStale(inputfile, rolexfile))
{
stale = true;
}
}
if (!stale && null != yaccfile)
{
if (_IsStale(inputfile, yaccfile))
{
stale = true;
}
}
if (!stale)
{
var files = XbnfDocument.GetResources(inputfile);
foreach (var s in files)
{
if (_IsStale(s, outputfile))
{
stale = true;
break;
}
}
}
// see if our exe has changed
if (!stale && null != outputfile && _IsStale(CodeBase, outputfile))
{
stale = true;
}
}
if (!stale)
{
stderr.WriteLine("Skipped building of the following because they were not stale:");
if (null != outputfile)
{
stderr.WriteLine("Output file: " + outputfile);
}
if (null != rolexfile)
{
stderr.WriteLine("Rolex file: " + rolexfile);
}
if (null != yaccfile)
{
stderr.WriteLine("YACC file: " + yaccfile);
}
}
else
{
stderr.WriteLine("{0} is building the following:", Name);
if (null != outputfile)
{
stderr.WriteLine("Output file: " + outputfile);
}
if (null != rolexfile)
{
stderr.WriteLine("Rolex file: " + rolexfile);
}
if (null != yaccfile)
{
stderr.WriteLine("YACC file: " + yaccfile);
}
if (string.IsNullOrEmpty(codelanguage))
{
if (!string.IsNullOrEmpty(outputfile))
{
codelanguage = Path.GetExtension(outputfile);
if (codelanguage.StartsWith("."))
{
codelanguage = codelanguage.Substring(1);
}
}
if (string.IsNullOrEmpty(codelanguage))
{
codelanguage = "cs";
}
}
var isLexerOnly = true;
if (doc.HasNonTerminalProductions)
{
isLexerOnly = false;
}
else
{
foreach (var include in doc.Includes)
{
if (include.Document.HasNonTerminalProductions)
{
isLexerOnly = false;
break;
}
}
}
// we need to prepare it by marking every terminal
// with an attribute if it isn't already. we use
// "terminal" because it doesn't impact terminals
// in any way, but this way the CfgDocument can
// "see" them.
for (int ic = doc.Productions.Count, i = 0; i < ic; ++i)
{
var p = doc.Productions[i];
if (p.IsTerminal && 0 == p.Attributes.Count)
{
p.Attributes.Add(new XbnfAttribute("terminal", true));
}
}
XbnfGenerationInfo genInfo;
var msgs = XbnfConvert.TryCreateGenerationInfo(doc, out genInfo);
foreach (var msg in msgs)
{
if (verbose || ErrorLevel.Information != msg.ErrorLevel)
{
stderr.WriteLine(msg);
}
}
foreach (var msg in msgs)
{
if (msg.ErrorLevel == ErrorLevel.Error)
{
throw new Exception(msg.ToString());
}
}
CfgDocument primaryCfg = genInfo.Cfg;
doc = genInfo.Xbnf;
if (!isLexerOnly)
{
if (verbose)
{
stderr.WriteLine("Final grammar:");
stderr.WriteLine(primaryCfg.ToString());
stderr.WriteLine();
}
foreach (var msg in msgs)
{
if (msg.ErrorLevel == ErrorLevel.Error)
{
throw new Exception(msg.ToString());
}
}
if (!noparser)
{
var ccu = CodeGenerator.GenerateCompileUnit(genInfo, codeclass, codenamespace, fast);
ccu.Namespaces.Add(new CodeNamespace(codenamespace ?? ""));
var ccuNS = ccu.Namespaces[ccu.Namespaces.Count - 1];
var ccuShared = CodeGenerator.GenerateSharedCompileUnit(codenamespace);
ccu.ReferencedAssemblies.Add(typeof(TypeConverter).Assembly.GetName().ToString());
if (fast)
{
CD.CodeDomVisitor.Visit(ccu, (ctx) =>
{
var vd = ctx.Target as CodeVariableDeclarationStatement;
if (null != vd && CD.CodeDomResolver.IsNullOrVoidType(vd.Type))
{
vd.Type = C.Type("var");
}
}, CD.CodeDomVisitTargets.All & ~(CD.CodeDomVisitTargets.Expressions | CD.CodeDomVisitTargets.Comments | CD.CodeDomVisitTargets.Attributes | CD.CodeDomVisitTargets.Directives | CD.CodeDomVisitTargets.Types | CD.CodeDomVisitTargets.TypeRefs));
CD.CodeDomVisitor.Visit(ccuShared, (ctx) =>
{
var vd = ctx.Target as CodeVariableDeclarationStatement;
if (null != vd && CD.CodeDomResolver.IsNullOrVoidType(vd.Type))
{
vd.Type = C.Type("var");
}
}, CD.CodeDomVisitTargets.All & ~(CD.CodeDomVisitTargets.Expressions | CD.CodeDomVisitTargets.Comments | CD.CodeDomVisitTargets.Attributes | CD.CodeDomVisitTargets.Directives | CD.CodeDomVisitTargets.Types | CD.CodeDomVisitTargets.TypeRefs));
}
else
{
SlangPatcher.Patch(ccu, ccuShared);
var co = SlangPatcher.GetNextUnresolvedElement(ccu);
if (null != co)
{
stderr.WriteLine("Warning: Not all of the elements could be resolved. The generated code may not be correct in all languages.");
stderr.WriteLine(" Next unresolved: {0}", C.ToString(co).Trim());
}
}
if (!noshared)
{
CodeGenerator.ImportCompileUnit(ccuNS, ccuShared);
}
var prov = CodeDomProvider.CreateProvider(codelanguage);
if (null != outputfile)
{
var sw = new StreamWriter(outputfile);
sw.BaseStream.SetLength(0);
output = sw;
}
else
{
output = stdout;
}
var opts = new CodeGeneratorOptions();
opts.VerbatimOrder = true;
opts.BlankLinesBetweenMembers = false;
prov.GenerateCodeFromCompileUnit(ccu, output, opts);
output.Flush();
output.Close();
output = null;
}
}
else
{
stderr.WriteLine("{0} skipped parser generation because there are no non-terminals and no imports defined.", Name);
}
if (null != rolexfile)
{
var sw = new StreamWriter(rolexfile);
sw.BaseStream.SetLength(0);
output = sw;
output.WriteLine(XbnfConvert.ToRolexSpec(genInfo));
output.Flush();
output.Close();
output = null;
}
if (null != yaccfile)
{
var sw = new StreamWriter(yaccfile);
sw.BaseStream.SetLength(0);
output = sw;
output.WriteLine(genInfo.Cfg.ToString("y"));
output.Flush();
output.Close();
output = null;
}
}
}
#if !DEBUG
catch (Exception ex)
{
result = _ReportError(ex, stderr);
}
#endif
finally
{
stderr.Close();
stdout.Close();
if (outputfile != null && null != output)
{
output.Close();
}
}
return(result);
}
