static int _Lalr1Tree(string[] args)
{
if (1 > args.Length || args.Length > 2)
{
_PrintUsageLalr1Tree();
return(1);
}
var pckspec = args[0];
string pck;
using (var sr = File.OpenText(pckspec))
pck = sr.ReadToEnd();
var cfg = CfgDocument.Parse(pck);
var lex = LexDocument.Parse(pck);
var tokenizer = lex.ToTokenizer(
(1 < args.Length) ? (TextReaderEnumerable) new FileReaderEnumerable(args[1]) :
new ConsoleReaderEnumerable(),
cfg.EnumSymbols(), new _TokenizerConsoleProgress());
var parser = cfg.ToLalr1Parser(tokenizer, new _Lalr1ConsoleProgress());
Console.Error.WriteLine();
parser.ShowHidden = true;
while (LRNodeType.EndDocument != parser.NodeType)
{
Console.WriteLine(parser.ParseReductions());
}
return(1);
}
public static void Transform(LexDocument lex, TextWriter output)
{
output.WriteLine("%%");
for (int ic = lex.Rules.Count, i = 0; i < ic; ++i)
{
var rule = lex.Rules[i];
var o = lex.GetAttribute(rule.Left, "hidden", false);
if (o is bool && (bool)o)
{
var s = string.Concat(rule.Right, "\t;");
var be = lex.GetAttribute(rule.Left, "blockEnd", null) as string;
if (!string.IsNullOrEmpty(be))
{
s = string.Concat(s, " /* TODO: implement blockend */");
}
output.WriteLine(s);
}
else
{
var s = string.Concat(rule.Right, "\treturn ", string.Concat(_EscId(rule.Left), ";"));
var be = lex.GetAttribute(rule.Left, "blockEnd", null) as string;
if (!string.IsNullOrEmpty(be))
{
s = string.Concat(s, " /* TODO: implement blockend */");
}
output.WriteLine(s);
}
}
}
// rips the lex info out of the document so it can
// paste it back later.
// technically, we no longer need this since we have access
// to the CfgDocument from here, but we'll keep it so that
// we can pull it out without the dependency
static IList <string> _RipSymbols(LexDocument l, string inp)
{
var result = new List <string>();
var terms = new List <string>();
var terms2 = new List <string>();
var sr = new StringReader(inp);
string line;
while (null != (line = sr.ReadLine()))
{
line = line.Trim();
var i = line.IndexOf("//");
var j = line.IndexOf('\'');
if (-1 < i && (0 > j || j > i)) // remove the comment
{
line = line.Substring(0, i);
}
var sbid = new StringBuilder();
for (i = 0; i < line.Length; i++)
{
var ch = line[i];
if (_NotIdentifierChars.Contains(ch))
{
break;
}
sbid.Append(ch);
}
while (i < line.Length && char.IsWhiteSpace(line[i]))
{
++i;
}
if (i < line.Length && '-' == line[i])
{
if (!result.Contains(sbid.ToString()))
{
result.Add(sbid.ToString());
}
}
else if (i < line.Length && '=' == line[i])
{
var o = l.GetAttribute(sbid.ToString(), "hidden", null);
if (o is bool && (bool)o)
{
if (!terms.Contains(sbid.ToString()) && !terms2.Contains(sbid.ToString()))
{
terms2.Add(sbid.ToString());
}
}
else if (!terms.Contains(sbid.ToString()) && !terms2.Contains(sbid.ToString()))
{
terms.Add(sbid.ToString());
}
}
}
result.AddRange(terms);
result.AddRange(terms2);
result.Add("#EOS");
result.Add("#ERROR");
return(result);
}
public static void WriteClassTo(LexDocument lex, IList <string> symbolTable, string name, string @namespace, string language, IProgress <FAProgress> progress, TextWriter writer)
{
if (string.IsNullOrEmpty(language))
{
language = "cs";
}
var cdp = CodeDomProvider.CreateProvider(language);
var tokenizer = _CreateTokenizerClass(lex, symbolTable, name, progress);
var opts = new CodeGeneratorOptions();
opts.BlankLinesBetweenMembers = false;
if (string.IsNullOrEmpty(@namespace))
{
cdp.GenerateCodeFromType(tokenizer, writer, opts);
}
else
{
var cns = new CodeNamespace(@namespace);
cns.Types.Add(tokenizer);
cdp.GenerateCodeFromNamespace(cns, writer, opts);
}
}
static CodeTypeDeclaration _CreateTokenizerClass(LexDocument lex, IList <string> symbolTable, string name, IProgress <FAProgress> progress)
{
var lexer = lex.ToLexer(progress);
var ii = 0;
var syms = new List <string>(symbolTable);
var bes = new string[syms.Count];
for (ii = 0; ii < bes.Length; ii++)
{
bes[ii] = lex.GetAttribute(syms[ii], "blockEnd", null) as string;
}
lexer = lexer.ToDfa(progress);
//lexer.TrimDuplicates(progress);
var dfaTable = lexer.ToArray(syms);
var result = new CodeTypeDeclaration();
var tt = new List <string>();
for (int ic = lex.Rules.Count, i = 0; i < ic; ++i)
{
var t = lex.Rules[i].Left;
if (!tt.Contains(t))
{
tt.Add(t);
}
}
tt.Add("#EOS");
tt.Add("#ERROR");
for (int ic = syms.Count, i = 0; i < ic; ++i)
{
if (!tt.Contains(syms[i]))
{
syms[i] = null;
}
}
result.Name = name;
result.BaseTypes.Add(typeof(TableTokenizer));
result.Attributes = MemberAttributes.FamilyOrAssembly;
CodeMemberField f;
foreach (var t in tt)
{
if (null != t)
{
f = new CodeMemberField();
f.Attributes = MemberAttributes.Const | MemberAttributes.Public;
f.Name = t.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_");
f.Type = new CodeTypeReference(typeof(int));
f.InitExpression = CodeDomUtility.Serialize(syms.IndexOf(t));
result.Members.Add(f);
}
}
f = new CodeMemberField();
f.Name = "_Symbols";
f.Type = new CodeTypeReference(typeof(string[]));
f.Attributes = MemberAttributes.Static;
var arr = new string[syms.Count];
syms.CopyTo(arr, 0);
f.InitExpression = CodeDomUtility.Serialize(arr);
result.Members.Add(f);
f = new CodeMemberField();
f.Name = "_BlockEnds";
f.Type = new CodeTypeReference(typeof(string[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(bes);
result.Members.Add(f);
f = new CodeMemberField();
f.Name = "_DfaTable";
f.Type = new CodeTypeReference(typeof(CharDfaEntry[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(dfaTable);
result.Members.Add(f);
var ctor = new CodeConstructor();
ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <char>), "input"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_DfaTable"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_Symbols"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_BlockEnds"),
new CodeArgumentReferenceExpression("input")
});
ctor.Attributes = MemberAttributes.Public;
result.Members.Add(ctor);
return(result);
}
public static void WriteClassTo(LexDocument lex, IList <string> symbolTable, string name, string @namespace, string language, TextWriter writer) => WriteClassTo(lex, symbolTable, name, @namespace, language, null, writer);
static int _Fagen(string[] args)
{
string specFile = null;
string outFile = null;
string @namespace = null;
string @class = null;
string language = null; // "c#";
var optIndex = -1;
for (var i = 0; i < args.Length; ++i)
{
if ("--help" == args[i] || "/?" == args[i] || "/help" == args[i])
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(0);
}
if (args[i].StartsWith("/"))
{
optIndex = i;
if (i == args.Length - 1)
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(1);
}
switch (args[i])
{
case "/language":
++i;
language = args[i];
break;
case "/namespace":
++i;
@namespace = args[i];
break;
case "/class":
++i;
@class = args[i];
break;
default:
_PrintUsageFAGen();
return(1);
}
}
else
{
if (-1 != optIndex)
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(1);
}
if (0 == i)
{
specFile = args[i];
}
else if (1 == i)
{
outFile = args[i];
}
else
{
Console.Error.Write("Usage: ");
_PrintUsageFAGen();
return(1);
}
}
}
TextReader inp = null;
TextWriter outp = null;
try
{
if (null == specFile)
{
inp = Console.In;
}
else
{
inp = new StreamReader(specFile);
}
if (null == outFile)
{
outp = Console.Out;
}
else
{
outp = new StreamWriter(outFile);
}
var buf = inp.ReadToEnd();
var lex = LexDocument.Parse(buf);
var cfg = CfgDocument.Parse(buf);
var syms = cfg.FillSymbols();
if (string.IsNullOrEmpty(@class))
{
if (null != outFile)
{
@class = Path.GetFileNameWithoutExtension(outFile);
}
else if (0 < syms.Count)
{
foreach (var attrs in lex.AttributeSets)
{
var i = attrs.Value.IndexOf("start");
if (-1 < i && attrs.Value[i].Value is bool && (bool)attrs.Value[i].Value)
{
@class = string.Concat(attrs.Key, "Tokenizer");
break;
}
}
if (null == @class)
{
@class = string.Concat(syms[0], "Tokenizer");
}
}
}
TokenizerCodeGenerator.WriteClassTo(lex, syms, @class, @namespace, language, new _TokenizerConsoleProgress(), outp);
Console.Error.WriteLine();
outp.Flush();
return(0);
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.Message);
return(1);
}
finally
{
inp.Close();
outp.Close();
}
}
public static void Transform(TextReader input, TextWriter output) => Transform(LexDocument.ReadFrom(input), output);