public IEnumerable <Token> ToTokenizer(Cfg cfg, IEnumerable <char> input)
{
var lexer = ToLexer(cfg);
var sm = new Dictionary <string, int>();
var ii = 0;
var syms = new List <string>();
cfg.FillSymbols(syms);
var tt = new List <string>(syms);
for (int jc = tt.Count, j = 0; j < jc; ++j)
{
if (cfg.IsNonTerminal(tt[j]))
{
tt[j] = null;
}
}
foreach (var sym in syms)
{
sm.Add(sym, ii);
++ii;
}
var bes = new string[syms.Count];
for (ii = 0; ii < bes.Length; ii++)
{
bes[ii] = cfg.AttributeSets.GetAttribute(syms[ii], "blockEnd", null) as string;
}
var dfaTable = lexer.ToDfaTable(sm);
return(new TableTokenizer(dfaTable, tt.ToArray(), bes, input));
}
void _PopulateAttrs()
{
var syms = _cfg.FillSymbols();
_symbolIds = new Dictionary <string, int>();
for (int ic = syms.Count, i = 0; i < ic; ++i)
{
_symbolIds.Add(syms[i], i);
}
_hidden = new HashSet <string>();
_collapsed = new HashSet <string>();
object o;
foreach (var sattr in _cfg.AttributeSets)
{
// make sure "hidden" is only applied to terminals.
if (!_cfg.IsNonTerminal(sattr.Key) && sattr.Value.TryGetValue("hidden", out o) && o is bool && (bool)o)
{
_hidden.Add(sattr.Key);
}
if (sattr.Value.TryGetValue("collapsed", out o) && o is bool && (bool)o)
{
_collapsed.Add(sattr.Key);
}
}
}
void _PopulateAttrs()
{
var syms = _cfg.FillSymbols();
_symbolIds = new Dictionary <string, int>();
for (int ic = syms.Count, i = 0; i < ic; ++i)
{
_symbolIds.Add(syms[i], i);
}
_blockEnds = new Dictionary <string, string>();
foreach (var s in _cfg.AttributeSets.Keys)
{
if (!_cfg.IsNonTerminal(s))
{
var be = _cfg.AttributeSets.GetAttribute(s, "blockEnd") as string;
if (null != be)
{
_blockEnds.Add(s, be);
}
}
}
}
static CodeTypeDeclaration _CreateTokenizerClass(EbnfDocument ebnf, Cfg cfg, string name)
{
var lexer = ebnf.ToLexer(cfg);
var sm = new Dictionary <string, int>();
var ii = 0;
var syms = new List <string>();
cfg.FillSymbols(syms);
var tt = new List <string>(syms);
for (int jc = tt.Count, j = 0; j < jc; ++j)
{
if (cfg.IsNonTerminal(tt[j]))
{
tt[j] = null;
}
}
foreach (var sym in syms)
{
sm.Add(sym, ii);
++ii;
}
var bes = new string[syms.Count];
for (ii = 0; ii < bes.Length; ii++)
{
bes[ii] = cfg.AttributeSets.GetAttribute(syms[ii], "blockEnd", null) as string;
}
var dfaTable = lexer.ToDfaTable(sm);
var result = new CodeTypeDeclaration();
result.Name = name;
result.BaseTypes.Add(typeof(TableTokenizer));
result.Attributes = MemberAttributes.FamilyOrAssembly;
CodeMemberField f;
foreach (var t in tt)
{
if (null != t)
{
f = new CodeMemberField();
f.Attributes = MemberAttributes.Const | MemberAttributes.Public;
f.Name = t.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_");
f.Type = new CodeTypeReference(typeof(int));
f.InitExpression = CodeDomUtility.Serialize(cfg.GetIdOfSymbol(t));
result.Members.Add(f);
}
}
f = new CodeMemberField();
f.Name = "_Symbols";
f.Type = new CodeTypeReference(typeof(string[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(tt.ToArray());
result.Members.Add(f);
f = new CodeMemberField();
f.Name = "_BlockEnds";
f.Type = new CodeTypeReference(typeof(string[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(bes);
result.Members.Add(f);
f = new CodeMemberField();
f.Name = "_DfaTable";
f.Type = new CodeTypeReference(typeof(CharDfaEntry[]));
f.Attributes = MemberAttributes.Static;
f.InitExpression = CodeDomUtility.Serialize(dfaTable);
result.Members.Add(f);
var ctor = new CodeConstructor();
ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <char>), "input"));
ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_DfaTable"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_Symbols"),
new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_BlockEnds"),
new CodeArgumentReferenceExpression("input")
});
ctor.Attributes = MemberAttributes.Public;
result.Members.Add(ctor);
return(result);
}