Exemple #1
0
        static int _Lalr1Tree(string[] args)
        {
            if (1 > args.Length || args.Length > 2)
            {
                _PrintUsageLalr1Tree();
                return(1);
            }
            var    pckspec = args[0];
            string pck;

            using (var sr = File.OpenText(pckspec))
                pck = sr.ReadToEnd();

            var cfg = CfgDocument.Parse(pck);
            var lex = LexDocument.Parse(pck);

            var tokenizer = lex.ToTokenizer(
                (1 < args.Length) ? (TextReaderEnumerable) new FileReaderEnumerable(args[1]) :
                new ConsoleReaderEnumerable(),
                cfg.EnumSymbols(), new _TokenizerConsoleProgress());
            var parser = cfg.ToLalr1Parser(tokenizer, new _Lalr1ConsoleProgress());

            Console.Error.WriteLine();
            parser.ShowHidden = true;
            while (LRNodeType.EndDocument != parser.NodeType)
            {
                Console.WriteLine(parser.ParseReductions());
            }

            return(1);
        }
Exemple #2
0
 public static void Transform(LexDocument lex, TextWriter output)
 {
     output.WriteLine("%%");
     for (int ic = lex.Rules.Count, i = 0; i < ic; ++i)
     {
         var rule = lex.Rules[i];
         var o    = lex.GetAttribute(rule.Left, "hidden", false);
         if (o is bool && (bool)o)
         {
             var s  = string.Concat(rule.Right, "\t;");
             var be = lex.GetAttribute(rule.Left, "blockEnd", null) as string;
             if (!string.IsNullOrEmpty(be))
             {
                 s = string.Concat(s, " /* TODO: implement blockend */");
             }
             output.WriteLine(s);
         }
         else
         {
             var s  = string.Concat(rule.Right, "\treturn ", string.Concat(_EscId(rule.Left), ";"));
             var be = lex.GetAttribute(rule.Left, "blockEnd", null) as string;
             if (!string.IsNullOrEmpty(be))
             {
                 s = string.Concat(s, " /* TODO: implement blockend */");
             }
             output.WriteLine(s);
         }
     }
 }
Exemple #3
0
        // rips the lex info out of the document so it can
        // paste it back later.
        // technically, we no longer need this since we have access
        // to the CfgDocument from here, but we'll keep it so that
        // we can pull it out without the dependency
        static IList <string> _RipSymbols(LexDocument l, string inp)
        {
            var    result = new List <string>();
            var    terms  = new List <string>();
            var    terms2 = new List <string>();
            var    sr     = new StringReader(inp);
            string line;

            while (null != (line = sr.ReadLine()))
            {
                line = line.Trim();
                var i = line.IndexOf("//");
                var j = line.IndexOf('\'');
                if (-1 < i && (0 > j || j > i))                 // remove the comment
                {
                    line = line.Substring(0, i);
                }
                var sbid = new StringBuilder();
                for (i = 0; i < line.Length; i++)
                {
                    var ch = line[i];
                    if (_NotIdentifierChars.Contains(ch))
                    {
                        break;
                    }
                    sbid.Append(ch);
                }
                while (i < line.Length && char.IsWhiteSpace(line[i]))
                {
                    ++i;
                }
                if (i < line.Length && '-' == line[i])
                {
                    if (!result.Contains(sbid.ToString()))
                    {
                        result.Add(sbid.ToString());
                    }
                }
                else if (i < line.Length && '=' == line[i])
                {
                    var o = l.GetAttribute(sbid.ToString(), "hidden", null);
                    if (o is bool && (bool)o)
                    {
                        if (!terms.Contains(sbid.ToString()) && !terms2.Contains(sbid.ToString()))
                        {
                            terms2.Add(sbid.ToString());
                        }
                    }
                    else if (!terms.Contains(sbid.ToString()) && !terms2.Contains(sbid.ToString()))
                    {
                        terms.Add(sbid.ToString());
                    }
                }
            }
            result.AddRange(terms);
            result.AddRange(terms2);
            result.Add("#EOS");
            result.Add("#ERROR");
            return(result);
        }
Exemple #4
0
        public static void WriteClassTo(LexDocument lex, IList <string> symbolTable, string name, string @namespace, string language, IProgress <FAProgress> progress, TextWriter writer)
        {
            if (string.IsNullOrEmpty(language))
            {
                language = "cs";
            }
            var cdp       = CodeDomProvider.CreateProvider(language);
            var tokenizer = _CreateTokenizerClass(lex, symbolTable, name, progress);
            var opts      = new CodeGeneratorOptions();

            opts.BlankLinesBetweenMembers = false;
            if (string.IsNullOrEmpty(@namespace))
            {
                cdp.GenerateCodeFromType(tokenizer, writer, opts);
            }
            else
            {
                var cns = new CodeNamespace(@namespace);
                cns.Types.Add(tokenizer);
                cdp.GenerateCodeFromNamespace(cns, writer, opts);
            }
        }
Exemple #5
0
        static CodeTypeDeclaration _CreateTokenizerClass(LexDocument lex, IList <string> symbolTable, string name, IProgress <FAProgress> progress)
        {
            var lexer = lex.ToLexer(progress);
            var ii    = 0;
            var syms  = new List <string>(symbolTable);
            var bes   = new string[syms.Count];

            for (ii = 0; ii < bes.Length; ii++)
            {
                bes[ii] = lex.GetAttribute(syms[ii], "blockEnd", null) as string;
            }
            lexer = lexer.ToDfa(progress);
            //lexer.TrimDuplicates(progress);
            var dfaTable = lexer.ToArray(syms);
            var result   = new CodeTypeDeclaration();
            var tt       = new List <string>();

            for (int ic = lex.Rules.Count, i = 0; i < ic; ++i)
            {
                var t = lex.Rules[i].Left;
                if (!tt.Contains(t))
                {
                    tt.Add(t);
                }
            }
            tt.Add("#EOS");
            tt.Add("#ERROR");
            for (int ic = syms.Count, i = 0; i < ic; ++i)
            {
                if (!tt.Contains(syms[i]))
                {
                    syms[i] = null;
                }
            }
            result.Name = name;
            result.BaseTypes.Add(typeof(TableTokenizer));
            result.Attributes = MemberAttributes.FamilyOrAssembly;
            CodeMemberField f;

            foreach (var t in tt)
            {
                if (null != t)
                {
                    f                = new CodeMemberField();
                    f.Attributes     = MemberAttributes.Const | MemberAttributes.Public;
                    f.Name           = t.Replace("#", "_").Replace("'", "_").Replace("<", "_").Replace(">", "_");
                    f.Type           = new CodeTypeReference(typeof(int));
                    f.InitExpression = CodeDomUtility.Serialize(syms.IndexOf(t));
                    result.Members.Add(f);
                }
            }

            f            = new CodeMemberField();
            f.Name       = "_Symbols";
            f.Type       = new CodeTypeReference(typeof(string[]));
            f.Attributes = MemberAttributes.Static;
            var arr = new string[syms.Count];

            syms.CopyTo(arr, 0);
            f.InitExpression = CodeDomUtility.Serialize(arr);
            result.Members.Add(f);

            f                = new CodeMemberField();
            f.Name           = "_BlockEnds";
            f.Type           = new CodeTypeReference(typeof(string[]));
            f.Attributes     = MemberAttributes.Static;
            f.InitExpression = CodeDomUtility.Serialize(bes);
            result.Members.Add(f);

            f                = new CodeMemberField();
            f.Name           = "_DfaTable";
            f.Type           = new CodeTypeReference(typeof(CharDfaEntry[]));
            f.Attributes     = MemberAttributes.Static;
            f.InitExpression = CodeDomUtility.Serialize(dfaTable);
            result.Members.Add(f);

            var ctor = new CodeConstructor();

            ctor.Parameters.Add(new CodeParameterDeclarationExpression(typeof(IEnumerable <char>), "input"));
            ctor.BaseConstructorArgs.AddRange(new CodeExpression[] {
                new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_DfaTable"),
                new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_Symbols"),
                new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(result.Name), "_BlockEnds"),
                new CodeArgumentReferenceExpression("input")
            });
            ctor.Attributes = MemberAttributes.Public;
            result.Members.Add(ctor);
            return(result);
        }
Exemple #6
0
 public static void WriteClassTo(LexDocument lex, IList <string> symbolTable, string name, string @namespace, string language, TextWriter writer)
 => WriteClassTo(lex, symbolTable, name, @namespace, language, null, writer);
Exemple #7
0
        static int _Fagen(string[] args)
        {
            string specFile   = null;
            string outFile    = null;
            string @namespace = null;
            string @class     = null;
            string language   = null;          // "c#";
            var    optIndex   = -1;

            for (var i = 0; i < args.Length; ++i)
            {
                if ("--help" == args[i] || "/?" == args[i] || "/help" == args[i])
                {
                    Console.Error.Write("Usage: ");
                    _PrintUsageFAGen();
                    return(0);
                }
                if (args[i].StartsWith("/"))
                {
                    optIndex = i;
                    if (i == args.Length - 1)
                    {
                        Console.Error.Write("Usage: ");
                        _PrintUsageFAGen();
                        return(1);
                    }
                    switch (args[i])
                    {
                    case "/language":
                        ++i;
                        language = args[i];
                        break;

                    case "/namespace":
                        ++i;
                        @namespace = args[i];
                        break;

                    case "/class":
                        ++i;
                        @class = args[i];
                        break;

                    default:
                        _PrintUsageFAGen();
                        return(1);
                    }
                }
                else
                {
                    if (-1 != optIndex)
                    {
                        Console.Error.Write("Usage: ");
                        _PrintUsageFAGen();
                        return(1);
                    }
                    if (0 == i)
                    {
                        specFile = args[i];
                    }
                    else if (1 == i)
                    {
                        outFile = args[i];
                    }
                    else
                    {
                        Console.Error.Write("Usage: ");
                        _PrintUsageFAGen();

                        return(1);
                    }
                }
            }
            TextReader inp  = null;
            TextWriter outp = null;

            try
            {
                if (null == specFile)
                {
                    inp = Console.In;
                }
                else
                {
                    inp = new StreamReader(specFile);
                }
                if (null == outFile)
                {
                    outp = Console.Out;
                }
                else
                {
                    outp = new StreamWriter(outFile);
                }

                var buf  = inp.ReadToEnd();
                var lex  = LexDocument.Parse(buf);
                var cfg  = CfgDocument.Parse(buf);
                var syms = cfg.FillSymbols();
                if (string.IsNullOrEmpty(@class))
                {
                    if (null != outFile)
                    {
                        @class = Path.GetFileNameWithoutExtension(outFile);
                    }
                    else if (0 < syms.Count)
                    {
                        foreach (var attrs in lex.AttributeSets)
                        {
                            var i = attrs.Value.IndexOf("start");
                            if (-1 < i && attrs.Value[i].Value is bool && (bool)attrs.Value[i].Value)
                            {
                                @class = string.Concat(attrs.Key, "Tokenizer");
                                break;
                            }
                        }
                        if (null == @class)
                        {
                            @class = string.Concat(syms[0], "Tokenizer");
                        }
                    }
                }
                TokenizerCodeGenerator.WriteClassTo(lex, syms, @class, @namespace, language, new _TokenizerConsoleProgress(), outp);
                Console.Error.WriteLine();
                outp.Flush();
                return(0);
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine(ex.Message);
                return(1);
            }
            finally
            {
                inp.Close();
                outp.Close();
            }
        }
Exemple #8
0
 public static void Transform(TextReader input, TextWriter output)
 => Transform(LexDocument.ReadFrom(input), output);