public override CharFA ToFA(EbnfDocument parent, Cfg cfg)
{
string sym = "";
if (null != parent)
{
sym = parent.GetContainingIdForExpression(this);
}
if (null == Right)
{
if (null == Left)
{
return(null);
}
var fa = Left.ToFA(parent, cfg);
fa.FirstAcceptingState.AcceptingSymbol = sym;
return(fa);
}
else if (null == Left)
{
var fa = Right.ToFA(parent, cfg);
fa.FirstAcceptingState.AcceptingSymbol = sym;
return(fa);
}
return(CharFA.Concat(new CharFA[] { Left.ToFA(parent, cfg), Right.ToFA(parent, cfg) }, sym));
}
public override CharFA ToFA(EbnfDocument parent, Cfg cfg)
{
if (null == Expression)
{
return(null);
}
return(CharFA.Optional(Expression.ToFA(parent, cfg), (null == parent) ? "" : parent.GetContainingIdForExpression(this)));
}
public DebugTokenizer(Cfg cfg, CharFA lexer, IEnumerable <char> input)
{
_cfg = cfg;
_lexer = lexer;
_input = input;
// we use the blockEnd attribute in the lexer to enable things like block comments and XML CDATA sections
_PopulateAttrs();
}
/// <summary>
/// Constructs a new instance of the parser
/// </summary>
/// <param name="parseTable">The parse table to use</param>
/// <param name="tokenizer">The tokenizer to use </param>
/// <param name="startSymbol">The start symbol</param>
public DebugLL1Parser(Cfg cfg,
IEnumerable <Token> tokenizer)
{
_cfg = cfg;
_PopulateAttrs();
_parseTable = cfg.ToLL1ParseTable();
_stack = new Stack <string>();
_errorToken.Symbol = null;
Restart(tokenizer);
}
public static void WriteParserClassTo(Cfg cfg, string name, string language, TextWriter writer)
{
if (string.IsNullOrEmpty(language))
{
language = "cs";
}
var cdp = CodeDomProvider.CreateProvider(language);
var parser = _CreateParserClass(cfg, name);
var opts = new CodeGeneratorOptions();
opts.BlankLinesBetweenMembers = false;
cdp.GenerateCodeFromType(parser, writer, opts);
}
public override IList <IList <string> > ToDisjunctions(EbnfDocument parent, Cfg cfg)
{
if (string.IsNullOrEmpty(Symbol))
{
throw new InvalidOperationException("The ref expression was nil.");
}
var l = new List <IList <string> >();
var ll = new List <string>();
l.Add(ll);
ll.Add(Symbol);
return(l);
}
public override CharFA ToFA(EbnfDocument parent, Cfg cfg)
{
if (null == Expression)
{
return(null);
}
var result = CharFA.Repeat(Expression.ToFA(parent, cfg), (null == parent) ? "" : parent.GetContainingIdForExpression(this));
if (IsOptional)
{
result = CharFA.Optional(result);
}
return(result);
}
public override CharFA ToFA(EbnfDocument parent, Cfg cfg)
{
if (null == parent)
{
throw new InvalidOperationException("The FA cannot be retrieved from a symbol reference without the parent document.");
}
var expr = parent.Productions[Symbol].Expression;
if (null != expr)
{
return(expr.ToFA(parent, cfg));
}
return(null);
}
public override IList <IList <string> > ToDisjunctions(EbnfDocument parent, Cfg cfg)
{
var l = new List <IList <string> >();
if (null != Expression)
{
l.AddRange(Expression.ToDisjunctions(parent, cfg));
var ll = new List <string>();
if (!l.Contains(ll, OrderedCollectionEqualityComparer <string> .Default))
{
l.Add(ll);
}
}
return(l);
}
public override IList <IList <string> > ToDisjunctions(EbnfDocument parent, Cfg cfg)
{
foreach (var prod in parent.Productions)
{
if (Equals(prod.Value.Expression, this))
{
var l = new List <IList <string> >();
var ll = new List <string>();
l.Add(ll);
ll.Add(prod.Key);
return(l);
}
}
throw new InvalidOperationException("The terminal was not declared.");
}
public CharFA ToLexer(Cfg cfg)
{
var result = new CharFA();
foreach (var prod in Productions)
{
var exp = prod.Value.Expression;
CharFA fa = null;
if (prod.Value.IsTerminal)
{
fa = prod.Value.Expression.ToFA(this, cfg);
result.EpsilonTransitions.Add(fa);
}
}
return(result);
}
public override IList <IList <string> > ToDisjunctions(EbnfDocument parent, Cfg cfg)
{
var l = new List <IList <string> >();
if (null == Right)
{
if (null == Left)
{
return(l);
}
foreach (var ll in Left.ToDisjunctions(parent, cfg))
{
l.Add(new List <string>(ll));
}
return(l);
}
else if (null == Left)
{
foreach (var ll in Right.ToDisjunctions(parent, cfg))
{
l.Add(new List <string>(ll));
}
return(l);
}
foreach (var ll in Left.ToDisjunctions(parent, cfg))
{
foreach (var ll2 in Right.ToDisjunctions(parent, cfg))
{
var ll3 = new List <string>();
ll3.AddRange(ll);
ll3.AddRange(ll2);
if (!l.Contains(ll3, OrderedCollectionEqualityComparer <string> .Default))
{
l.Add(ll3);
}
}
}
return(l);
}
public override CharFA ToFA(EbnfDocument parent, Cfg cfg)
{
string sym = "";
if (null != parent)
{
sym = parent.GetContainingIdForExpression(this);
}
if (null == Right)
{
if (null == Left)
{
return(null);
}
return(CharFA.Optional(Left.ToFA(parent, cfg), sym));
}
else if (null == Left)
{
return(CharFA.Optional(Right.ToFA(parent, cfg), sym));
}
return(CharFA.Or(new CharFA[] { Left.ToFA(parent, cfg), Right.ToFA(parent, cfg) }, sym));
}
public override IList <IList <string> > ToDisjunctions(EbnfDocument parent, Cfg cfg)
{
string sid = null;
var sr = Expression as EbnfRefExpression;
if (null != parent && null != sr)
{
sid = string.Concat(sr.Symbol, "list");
}
if (string.IsNullOrEmpty(sid))
{
var cc = Expression as EbnfConcatExpression;
if (null != cc)
{
sr = cc.Right as EbnfRefExpression;
if (null != sr)
{
sid = string.Concat(sr.Symbol, "listtail");
}
}
}
if (string.IsNullOrEmpty(sid))
{
sid = "implicitlist";
}
var _listId = cfg.GetUniqueId(sid);
var attrs = new AttributeSet();
attrs.Add("collapsed", true);
cfg.AttributeSets.Add(_listId, attrs);
var expr =
new EbnfOrExpression(
new EbnfConcatExpression(
new EbnfRefExpression(_listId), Expression), Expression);
//if (IsOptional)
// expr = new EbnfOrExpression(expr, null);
foreach (var nt in expr.ToDisjunctions(parent, cfg))
{
CfgRule r = new CfgRule();
r.Left = _listId;
foreach (var s in nt)
{
if (1 < r.Right.Count && null == s)
{
continue;
}
r.Right.Add(s);
}
if (!cfg.Rules.Contains(r))
{
cfg.Rules.Add(r);
}
}
if (!IsOptional)
{
return(new List <IList <string> >(new IList <string>[] { new List <string>(new string[] { _listId }) }));
}
else
{
var result = new List <IList <string> >();
result.Add(new List <string>(new string[] { _listId }));
result.Add(new List <string>());
return(result);
}
}
public override CharFA ToFA(EbnfDocument parent, Cfg cfg)
{
return(CharFA.Literal(Value, (null == parent) ? "" : parent.GetIdForExpression(this)));
}
static CodeTypeDeclaration _CreateTokenizerClass(EbnfDocument ebnf, Cfg cfg, string name)
{
var lexer = ebnf.ToLexer(cfg);
var sm = new Dictionary <string, int>();
var ii = 0;
var syms = new List <string>();
cfg.FillSymbols(syms);
var tt = new List <string>(syms);
for (int jc = tt.Count, j = 0; j < jc; ++j)
{
if (cfg.IsNonTerminal(tt[j]))
{
tt[j] = null;
}
}
foreach (var sym in syms)
{
sm.Add(sym, ii);
++ii;
}
var bes = new string[syms.Count];
for (ii = 0; ii < bes.Length; ii++)
{
bes[ii] = cfg.AttributeSets.GetAttribute(syms[ii], "blockEnd", null) as string;
}
var dfaTable = lexer.ToDfaTable(sm);
var result = new CodeTypeDeclaration();
result.Name = name;
result.BaseTypes.Add(typeof(TableTokenizer));
result.Attributes = MemberAttributes.FamilyOrAssembly;
CodeMemberField f;
foreach (var t in tt)
{
if (null != t)
{
f = new CodeMemberField();
f.Attributes = MemberAttributes.Const | MemberAttributes.Public;
f.Name = t.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_");
f.Type = new CodeTypeReference(typeof(int));
f.InitExpression = CodeDomUtility.Serialize(cfg.GetIdOfSymbol(t));
result.Members.Add(f);
}
}
f = new CodeMemberField();
f.Name = "_Symbols";
f.Type = new CodeTypeReference(typeof(string[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(tt.ToArray());
result.Members.Add(f);
f = new CodeMemberField();
f.Name = "_BlockEnds";
f.Type = new CodeTypeReference(typeof(string[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(bes);
result.Members.Add(f);
f = new CodeMemberField();
f.Name = "_DfaTable";
f.Type = new CodeTypeReference(typeof(CharDfaEntry[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(dfaTable);
result.Members.Add(f);
var ctor = new CodeConstructor();
ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <char>), "input"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_DfaTable"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_Symbols"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_BlockEnds"),
new CodeArgumentReferenceExpression("input")
});
ctor.Attributes = MemberAttributes.Public;
result.Members.Add(ctor);
return(result);
}
static CodeTypeDeclaration _CreateParserClass(Cfg cfg, string name)
{
var sm = new Dictionary <string, int>();
var ii = 0;
var syms = new List <string>();
cfg.FillSymbols(syms);
foreach (var sym in syms)
{
sm.Add(sym, ii);
++ii;
}
var pt = cfg.ToLL1ParseTable();
var ipt = pt.ToLL1Array(syms);
var nodeFlags = new int[syms.Count];
for (var i = 0; i < nodeFlags.Length; ++i)
{
var o = cfg.AttributeSets.GetAttribute(syms[i], "hidden", false);
if (o is bool && (bool)o)
{
nodeFlags[i] |= 2;
}
o = cfg.AttributeSets.GetAttribute(syms[i], "collapsed", false);
if (o is bool && (bool)o)
{
nodeFlags[i] |= 1;
}
}
var attrSets = new KeyValuePair <string, object> [syms.Count][];
for (ii = 0; ii < attrSets.Length; ii++)
{
AttributeSet attrs;
if (cfg.AttributeSets.TryGetValue(syms[ii], out attrs))
{
attrSets[ii] = new KeyValuePair <string, object> [attrs.Count];
var j = 0;
foreach (var attr in attrs)
{
attrSets[ii][j] = new KeyValuePair <string, object>(attr.Key, attr.Value);
++j;
}
}
else
{
attrSets[ii] = null; // new KeyValuePair<string, object>[0];
}
}
var result = new CodeTypeDeclaration();
result.Name = name;
result.Attributes = MemberAttributes.FamilyOrAssembly;
result.BaseTypes.Add(typeof(LL1TableParser));
CodeMemberField f;
foreach (var s in syms)
{
if (null != s)
{
f = new CodeMemberField();
f.Attributes = MemberAttributes.Const | MemberAttributes.Public;
f.Name = s.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_");
f.Type = new CodeTypeReference(typeof(int));
f.InitExpression = CodeDomUtility.Serialize(cfg.GetIdOfSymbol(s));
result.Members.Add(f);
}
}
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_Symbols";
f.Type = new CodeTypeReference(typeof(string[]));
f.InitExpression = CodeDomUtility.Serialize(syms.ToArray());
result.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_ParseTable";
f.Type = new CodeTypeReference(typeof(int[][][]));
f.InitExpression = CodeDomUtility.Serialize(ipt);
result.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_InitCfg";
f.Type = new CodeTypeReference(typeof(int[]));
f.InitExpression = CodeDomUtility.Serialize(new int[] { cfg.GetIdOfSymbol(cfg.StartSymbol), cfg.FillNonTerminals().Count });
result.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_NodeFlags";
f.Type = new CodeTypeReference(typeof(int[]));
f.InitExpression = CodeDomUtility.Serialize(nodeFlags);
result.Members.Add(f);
f = new CodeMemberField();
f.Attributes = MemberAttributes.Static;
f.Name = "_AttributeSets";
f.Type = new CodeTypeReference(attrSets.GetType());
f.InitExpression = CodeDomUtility.Serialize(attrSets);
result.Members.Add(f);
var ctor = new CodeConstructor();
ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <Token>), "tokenizer"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_ParseTable"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_InitCfg"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_Symbols"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_NodeFlags"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_AttributeSets"),
new CodeArgumentReferenceExpression("tokenizer")
});
ctor.Attributes = MemberAttributes.Public;
result.Members.Add(ctor);
return(result);
}
public abstract CharFA ToFA(EbnfDocument parent, Cfg cfg);
public abstract IList <IList <string> > ToDisjunctions(EbnfDocument parent, Cfg cfg);