public static void Transform(LexDocument lex, TextWriter output) { output.WriteLine("%%"); for (int ic = lex.Rules.Count, i = 0; i < ic; ++i) { var rule = lex.Rules[i]; var o = lex.GetAttribute(rule.Left, "hidden", false); if (o is bool && (bool)o) { var s = string.Concat(rule.Right, "\t;"); var be = lex.GetAttribute(rule.Left, "blockEnd", null) as string; if (!string.IsNullOrEmpty(be)) { s = string.Concat(s, " /* TODO: implement blockend */"); } output.WriteLine(s); } else { var s = string.Concat(rule.Right, "\treturn ", string.Concat(_EscId(rule.Left), ";")); var be = lex.GetAttribute(rule.Left, "blockEnd", null) as string; if (!string.IsNullOrEmpty(be)) { s = string.Concat(s, " /* TODO: implement blockend */"); } output.WriteLine(s); } } }
// rips the lex info out of the document so it can // paste it back later. // technically, we no longer need this since we have access // to the CfgDocument from here, but we'll keep it so that // we can pull it out without the dependency static IList <string> _RipSymbols(LexDocument l, string inp) { var result = new List <string>(); var terms = new List <string>(); var terms2 = new List <string>(); var sr = new StringReader(inp); string line; while (null != (line = sr.ReadLine())) { line = line.Trim(); var i = line.IndexOf("//"); var j = line.IndexOf('\''); if (-1 < i && (0 > j || j > i)) // remove the comment { line = line.Substring(0, i); } var sbid = new StringBuilder(); for (i = 0; i < line.Length; i++) { var ch = line[i]; if (_NotIdentifierChars.Contains(ch)) { break; } sbid.Append(ch); } while (i < line.Length && char.IsWhiteSpace(line[i])) { ++i; } if (i < line.Length && '-' == line[i]) { if (!result.Contains(sbid.ToString())) { result.Add(sbid.ToString()); } } else if (i < line.Length && '=' == line[i]) { var o = l.GetAttribute(sbid.ToString(), "hidden", null); if (o is bool && (bool)o) { if (!terms.Contains(sbid.ToString()) && !terms2.Contains(sbid.ToString())) { terms2.Add(sbid.ToString()); } } else if (!terms.Contains(sbid.ToString()) && !terms2.Contains(sbid.ToString())) { terms.Add(sbid.ToString()); } } } result.AddRange(terms); result.AddRange(terms2); result.Add("#EOS"); result.Add("#ERROR"); return(result); }
static CodeTypeDeclaration _CreateTokenizerClass(LexDocument lex, IList <string> symbolTable, string name, IProgress <FAProgress> progress) { var lexer = lex.ToLexer(progress); var ii = 0; var syms = new List <string>(symbolTable); var bes = new string[syms.Count]; for (ii = 0; ii < bes.Length; ii++) { bes[ii] = lex.GetAttribute(syms[ii], "blockEnd", null) as string; } lexer = lexer.ToDfa(progress); //lexer.TrimDuplicates(progress); var dfaTable = lexer.ToArray(syms); var result = new CodeTypeDeclaration(); var tt = new List <string>(); for (int ic = lex.Rules.Count, i = 0; i < ic; ++i) { var t = lex.Rules[i].Left; if (!tt.Contains(t)) { tt.Add(t); } } tt.Add("#EOS"); tt.Add("#ERROR"); for (int ic = syms.Count, i = 0; i < ic; ++i) { if (!tt.Contains(syms[i])) { syms[i] = null; } } result.Name = name; result.BaseTypes.Add(typeof(TableTokenizer)); result.Attributes = MemberAttributes.FamilyOrAssembly; CodeMemberField f; foreach (var t in tt) { if (null != t) { f = new CodeMemberField(); f.Attributes = MemberAttributes.Const | MemberAttributes.Public; f.Name = t.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_"); f.Type = new CodeTypeReference(typeof(int)); f.InitExpression = CodeDomUtility.Serialize(syms.IndexOf(t)); result.Members.Add(f); } } f = new CodeMemberField(); f.Name = "_Symbols"; f.Type = new CodeTypeReference(typeof(string[])); f.Attributes = MemberAttributes.Static; var arr = new string[syms.Count]; syms.CopyTo(arr, 0); f.InitExpression = CodeDomUtility.Serialize(arr); result.Members.Add(f); f = new CodeMemberField(); f.Name = "_BlockEnds"; f.Type = new CodeTypeReference(typeof(string[])); f.Attributes = MemberAttributes.Static; f.InitExpression = CodeDomUtility.Serialize(bes); result.Members.Add(f); f = new CodeMemberField(); f.Name = "_DfaTable"; f.Type = new CodeTypeReference(typeof(CharDfaEntry[])); f.Attributes = MemberAttributes.Static; f.InitExpression = CodeDomUtility.Serialize(dfaTable); result.Members.Add(f); var ctor = new CodeConstructor(); ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <char>), "input")); ctor.BaseConstructorArgs.AddRange(new CodeExpression[] { new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_DfaTable"), new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_Symbols"), new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_BlockEnds"), new CodeArgumentReferenceExpression("input") }); ctor.Attributes = MemberAttributes.Public; result.Members.Add(ctor); return(result); }