Пример #1
0
        public static IList <IMessage> TryCreateGenerationInfo(XbnfDocument document, out XbnfGenerationInfo genInfo)
        {
            var includes = new XbnfImportList();

            _GatherIncludes(document, includes);
            var incs = new List <XbnfDocument>();

            incs.Add(document);
            foreach (var inc in includes)
            {
                incs.Add(inc.Document);
            }
            var doc = XbnfDocument.Merge(incs);
            var cfg = new CfgDocument();

            return(_TryToGenInfo(doc, cfg, out genInfo));
        }
Пример #2
0
 public _TermPriorityComparer(CfgDocument cfg)
 {
     _cfg = cfg;
 }
Пример #3
0
        static IList <IMessage> _TryToGenInfo(XbnfDocument document, CfgDocument cfg, out XbnfGenerationInfo genInfo)
        {
            genInfo = default(XbnfGenerationInfo);
            var hasErrors = false;
            var result    = new List <IMessage>();
            var syms      = new HashSet <string>();

            // gather the attributes and production names

            for (int ic = document.Productions.Count, i = 0; i < ic; ++i)
            {
                var p = document.Productions[i];
                if (!syms.Add(p.Name))
                {
                    result.Add(new XbnfMessage(ErrorLevel.Error, -1, string.Format("Duplicate production {0} defined.", p.Name), p.Line, p.Column, p.Position, document.FileOrUrl));
                    hasErrors = true;
                }
                if (0 < p.Attributes.Count)
                {
                    CfgAttributeList attrs;
                    if (!cfg.AttributeSets.TryGetValue(p.Name, out attrs))
                    {
                        attrs = new CfgAttributeList();
                        cfg.AttributeSets.Add(p.Name, attrs);
                    }
                    for (int jc = p.Attributes.Count, j = 0; j < jc; ++j)
                    {
                        var attr = p.Attributes[j];
                        attrs.Add(new CfgAttribute(attr.Name, attr.Value));
                    }
                }
            }

            // use a list dictionary to keep these in order
            var tmap     = new ListDictionary <XbnfExpression, string>();
            var attrSets = new Dictionary <string, XbnfAttributeList>();
            var rules    = new List <KeyValuePair <string, IList <string> > >();
            // below are scratch
            var working = new HashSet <XbnfExpression>();
            var done    = new HashSet <XbnfExpression>();

            // now get the terminals and their ids, declaring if necessary


            for (int ic = document.Productions.Count, i = 0; i < ic; ++i)
            {
                var p = document.Productions[i];
                if (p.IsTerminal)
                {
                    string name;
                    if (!tmap.TryGetValue(p.Expression, out name))
                    {
                        tmap.Add(p.Expression, p.Name);
                    }
                    else
                    {
                        if (name != p.Name)
                        {
                            result.Add(new CfgMessage(ErrorLevel.Error, -1, string.Format("{0} attempts to redefine terminal {1}", name, p.Name), p.Line, p.Column, p.Position, document.FileOrUrl));
                            hasErrors = true;
                        }
                    }
                    done.Add(p.Expression);
                }
                else
                {
                    _VisitFetchTerminals(p.Expression, working);
                }
            }

            if (hasErrors)
            {
                return(result);
            }
            foreach (var term in working)
            {
                if (!done.Contains(term))
                {
                    var newId = _GetImplicitTermId(syms);
                    var found = false;
                    var prod  = document.GetProductionForExpression(term);
                    if (null != prod)
                    {
                        found = true;
                        // recycle this symbol
                        newId = prod.Name;
                    }

                    if (!found)
                    {
                        document.Productions.Add(new XbnfProduction(newId, term));
                    }
                    tmap.Add(term, newId);
                }
            }
            // tmap now contains ALL of our terminal definitions from all of our imports
            // now we can use tmap and syms to help solve the rest of our productions

            var ntd = new Dictionary <string, IList <IList <string> > >();

            for (int ic = document.Productions.Count, i = 0; i < ic; ++i)
            {
                var p = document.Productions[i];


                if (!p.IsTerminal)
                {
                    var dys = _GetDysjunctions(document, syms, tmap, attrSets, rules, p, p.Expression);
                    IList <IList <string> > odys;
                    if (ntd.TryGetValue(p.Name, out odys))
                    {
                        result.Add(new XbnfMessage(ErrorLevel.Error, -1, string.Format("The {0} production was specified more than once", p.Name), p.Line, p.Column, p.Position, document.FileOrUrl));
                        hasErrors = true;
                    }

                    ntd.Add(p.Name, dys);
                    if (hasErrors)
                    {
                        return(result);
                    }
                }
            }
            // now that we've done that, build the rest of our attributes
            foreach (var sattrs in attrSets)
            {
                CfgAttributeList attrs;
                if (!cfg.AttributeSets.TryGetValue(sattrs.Key, out attrs))
                {
                    attrs = new CfgAttributeList();
                    cfg.AttributeSets.Add(sattrs.Key, attrs);
                }
                for (int jc = sattrs.Value.Count, j = 0; j < jc; ++j)
                {
                    var attr = sattrs.Value[j];
                    attrs.Add(new CfgAttribute(attr.Name, attr.Value));
                }
            }

            // now write our main rules
            foreach (var nt in ntd)
            {
                foreach (var l in nt.Value)
                {
                    cfg.Rules.Add(new CfgRule(nt.Key, l));
                }
            }
            // build our secondary rules
            foreach (var rule in rules)
            {
                cfg.Rules.Add(new CfgRule(rule.Key, rule.Value));
            }

            if (hasErrors)
            {
                return(result);
            }
            genInfo.Xbnf        = document;
            genInfo.TerminalMap = tmap;
            genInfo.Cfg         = cfg;
            cfg.RebuildCache();
            return(result);
        }
Пример #4
0
        CfgDocument _LRFAToLRExtendedGrammar(_LRFA lrfa, IProgress <CfgLRProgress> progress)
        {
            var result   = new CfgDocument();
            var closure  = new List <_LRFA>();
            var itemSets = new List <_LR0ItemSet>();

            lrfa.FillClosure(closure);
            foreach (var p in closure)
            {
                itemSets.Add(p.Accept);
            }

            _LRExtendedSymbol start = null;
            int j = 0;

            foreach (var p in closure)
            {
                if (null != progress)
                {
                    progress.Report(new CfgLRProgress(CfgLRStatus.CreatingLRExtendedGrammar, j));
                }

                int si = itemSets.IndexOf(p.Accept);

                foreach (var item in p.Accept.Items)
                {
                    if (0 == item.RightIndex)
                    {
                        var next = item.Next;

                        if (!item.IsEnd || item.IsNil)
                        {
                            int   dst = -1;
                            _LRFA dsts;
                            if (p.Transitions.ContainsKey(item.Left))
                            {
                                dsts = p.Transitions[item.Left];
                                dst  = itemSets.IndexOf(dsts.Accept);
                            }

                            _LRExtendedSymbol left = new _LRExtendedSymbol(si, item.Left, dst);
                            if (null == start)
                            {
                                start = left;
                            }
                            var right = new List <string>();
                            var pc    = p;
                            foreach (var sym in item.Right)
                            {
                                int s1 = itemSets.IndexOf(pc.Accept);
                                var pt = pc.Transitions[sym];
                                int s2 = itemSets.IndexOf(pt.Accept);
                                _LRExtendedSymbol n = new _LRExtendedSymbol(s1, sym, s2);
                                right.Add(n.ToString());
                                pc = pt;
                            }

                            result.Rules.Add(new CfgRule(left.ToString(), right));
                        }
                    }
                }
                ++j;
            }
            result.StartSymbol = start.ToString();
            return(result);
        }
Пример #5
0
        /*static int Test()
         * {
         *      string input;
         *      using (var sr = new StreamReader(@"..\..\data2.json"))
         *              input = sr.ReadToEnd();
         *      var tokenizer = new JsonTokenizer(input);
         *      var xbnf = XbnfDocument.ReadFrom(@"..\..\json.xbnf");
         *      XbnfGenerationInfo info;
         *      XbnfConvert.TryCreateGenerationInfo(xbnf, out info);
         *      int ts;
         *      var symbols =XbnfConvert.GetSymbolTable(info, out ts);
         *      CfgGlrParseTable parseTable;
         *      info.Cfg.RebuildCache();
         *      info.Cfg.TryToGlrParseTable(out parseTable, LRTableKind.Lalr1);
         *      var errorSentinels = new List<int>();
         *      var i = 0;
         *      var parseAttributes = new ParseAttribute[symbols.Count][];
         *      foreach(var attrs in info.Cfg.AttributeSets)
         *      {
         *              var id = symbols.IndexOf(attrs.Key);
         *              int jc = attrs.Value.Count;
         *              parseAttributes[id] = new ParseAttribute[jc];
         *              for(var j=0;j<jc;++j)
         *              {
         *                      var attr = attrs.Value[j];
         *                      parseAttributes[id][j] = new ParseAttribute(attr.Name, attr.Value);
         *                      if ("errorSentinel" == attr.Name && attr.Value is bool && ((bool)attr.Value))
         *                              errorSentinels.Add(id);
         *
         *              }
         ++i;
         *      }
         *      for (i = 0; i < parseAttributes.Length; i++)
         *              if (null == parseAttributes[i])
         *                      parseAttributes[i] = new ParseAttribute[0];
         *
         *      var parser = new GlrTableParser(parseTable.ToArray(symbols), symbols.ToArray(), parseAttributes, errorSentinels.ToArray(),tokenizer);
         *      foreach (var pt in parser.ParseReductions(false, true, false))
         *      {
         *              Console.WriteLine(pt.ToString("t"));
         *      }
         *      return 0;
         * }*/
        public static int Run(string[] args, TextReader stdin, TextWriter stdout, TextWriter stderr)
        {
            int        result = 0;
            TextWriter output = null;

            string inputfile     = null;
            string outputfile    = null;
            string rolexfile     = null;
            string codenamespace = null;
            string codelanguage  = null;
            string codeclass     = null;
            string yaccfile      = null;
            bool   verbose       = false;
            bool   noshared      = false;
            bool   ifstale       = false;
            bool   fast          = false;
            bool   noparser      = false;

            try
            {
                if (0 == args.Length)
                {
                    _PrintUsage(stderr);
                    return(-1);
                }
                if (args[0].StartsWith("/"))
                {
                    throw new ArgumentException("Missing input file.");
                }

                // process the command line args
                inputfile = args[0];
                for (var i = 1; i < args.Length; ++i)
                {
                    switch (args[i])
                    {
                    case "/namespace":
                        if (args.Length - 1 == i)                                 // check if we're at the end
                        {
                            throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
                        }
                        ++i;                                 // advance
                        codenamespace = args[i];
                        break;

                    case "/class":
                        if (args.Length - 1 == i)                                 // check if we're at the end
                        {
                            throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
                        }
                        ++i;                                 // advance
                        codeclass = args[i];
                        break;

                    case "/language":
                        if (args.Length - 1 == i)                                 // check if we're at the end
                        {
                            throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
                        }
                        ++i;                                 // advance
                        codelanguage = args[i];
                        break;

                    case "/output":
                        if (args.Length - 1 == i)                                 // check if we're at the end
                        {
                            throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
                        }
                        ++i;                                 // advance
                        outputfile = args[i];
                        break;

                    case "/yacc":
                        if (args.Length - 1 == i)                                 // check if we're at the end
                        {
                            throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
                        }
                        ++i;                                 // advance
                        yaccfile = args[i];
                        break;

                    case "/ifstale":
                        ifstale = true;
                        break;

                    case "/fast":
                        fast = true;
                        break;

                    case "/noparser":
                        noparser = true;
                        break;

                    case "/noshared":
                        noshared = true;
                        break;

                    case "/verbose":
                        verbose = true;
                        break;

                    case "/rolex":
                        if (args.Length - 1 == i)                                 // check if we're at the end
                        {
                            throw new ArgumentException(string.Format("The parameter \"{0}\" is missing an argument", args[i].Substring(1)));
                        }
                        ++i;                                 // advance
                        rolexfile = args[i];
                        break;

                    default:
                        throw new ArgumentException(string.Format("Unknown switch {0}", args[i]));
                    }
                }
                if (null != outputfile && noparser)
                {
                    throw new ArgumentException("<noparser> and <ouputfile> cannot both be specified.", "outputfile");
                }

                if (null == codeclass)
                {
                    if (null != outputfile)
                    {
                        codeclass = Path.GetFileNameWithoutExtension(outputfile);
                    }
                    else
                    {
                        codeclass = Path.GetFileNameWithoutExtension(inputfile);
                    }
                }


                // override the options with our document's options
                var doc = XbnfDocument.ReadFrom(inputfile);
                var oi  = -1;
                oi = doc.Options.IndexOf("outputfile");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    var s = o as string;
                    if (null != s)
                    {
                        outputfile = s;
                        if ("" == outputfile)
                        {
                            outputfile = null;
                        }
                    }
                    // if it's specified in the doc we need to make it doc relative
                    if (null != outputfile)
                    {
                        if (!Path.IsPathRooted(outputfile))
                        {
                            var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
                            outputfile = Path.GetFullPath(Path.Combine(dir, outputfile));
                        }
                    }
                }
                oi = doc.Options.IndexOf("rolexfile");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    var s = o as string;
                    if (null != s)
                    {
                        rolexfile = s;
                        if ("" == rolexfile)
                        {
                            rolexfile = null;
                        }
                    }
                    // if it's specified in the doc we need to make it doc relative
                    if (null != rolexfile)
                    {
                        if (!Path.IsPathRooted(rolexfile))
                        {
                            var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
                            rolexfile = Path.GetFullPath(Path.Combine(dir, rolexfile));
                        }
                    }
                }

                oi = doc.Options.IndexOf("yaccfile");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    var s = o as string;
                    if (null != s)
                    {
                        rolexfile = s;
                        if ("" == yaccfile)
                        {
                            yaccfile = null;
                        }
                    }
                    // if it's specified in the doc we need to make it doc relative
                    if (null != yaccfile)
                    {
                        if (!Path.IsPathRooted(yaccfile))
                        {
                            var dir = Path.GetDirectoryName(Path.GetFullPath(inputfile));
                            rolexfile = Path.GetFullPath(Path.Combine(dir, yaccfile));
                        }
                    }
                }


                oi = doc.Options.IndexOf("codenamespace");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    var s = o as string;
                    if (null != s)
                    {
                        codenamespace = s;
                    }
                }
                oi = doc.Options.IndexOf("codelanguage");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    var s = o as string;
                    if (!string.IsNullOrEmpty(s))
                    {
                        codelanguage = s;
                    }
                }
                oi = doc.Options.IndexOf("codeclass");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    var s = o as string;
                    if (null != s)
                    {
                        codeclass = s;
                        if ("" == codeclass)
                        {
                            if (null != outputfile)
                            {
                                codeclass = Path.GetFileNameWithoutExtension(outputfile);
                            }
                            else
                            {
                                codeclass = Path.GetFileNameWithoutExtension(inputfile);
                            }
                        }
                    }
                }
                oi = doc.Options.IndexOf("verbose");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    if (o is bool)
                    {
                        verbose = (bool)o;
                    }
                }
                oi = doc.Options.IndexOf("fast");
                if (-1 < oi)
                {
                    var o = doc.Options[oi].Value;
                    if (o is bool)
                    {
                        fast = (bool)o;
                    }
                }
                if (fast && null != codelanguage)
                {
                    throw new ArgumentException("<codelanguage> and <fast> cannot both be specified. The <fast> option is C# only.");
                }

                var stale = true;
                if (ifstale)
                {
                    stale = false;
                    if (!stale && null != rolexfile)
                    {
                        if (_IsStale(inputfile, rolexfile))
                        {
                            stale = true;
                        }
                    }
                    if (!stale && null != yaccfile)
                    {
                        if (_IsStale(inputfile, yaccfile))
                        {
                            stale = true;
                        }
                    }

                    if (!stale)
                    {
                        var files = XbnfDocument.GetResources(inputfile);
                        foreach (var s in files)
                        {
                            if (_IsStale(s, outputfile))
                            {
                                stale = true;
                                break;
                            }
                        }
                    }
                    // see if our exe has changed
                    if (!stale && null != outputfile && _IsStale(CodeBase, outputfile))
                    {
                        stale = true;
                    }
                }

                if (!stale)
                {
                    stderr.WriteLine("Skipped building of the following because they were not stale:");
                    if (null != outputfile)
                    {
                        stderr.WriteLine("Output file: " + outputfile);
                    }
                    if (null != rolexfile)
                    {
                        stderr.WriteLine("Rolex file: " + rolexfile);
                    }
                    if (null != yaccfile)
                    {
                        stderr.WriteLine("YACC file: " + yaccfile);
                    }
                }
                else
                {
                    stderr.WriteLine("{0} is building the following:", Name);
                    if (null != outputfile)
                    {
                        stderr.WriteLine("Output file: " + outputfile);
                    }
                    if (null != rolexfile)
                    {
                        stderr.WriteLine("Rolex file: " + rolexfile);
                    }
                    if (null != yaccfile)
                    {
                        stderr.WriteLine("YACC file: " + yaccfile);
                    }
                    if (string.IsNullOrEmpty(codelanguage))
                    {
                        if (!string.IsNullOrEmpty(outputfile))
                        {
                            codelanguage = Path.GetExtension(outputfile);
                            if (codelanguage.StartsWith("."))
                            {
                                codelanguage = codelanguage.Substring(1);
                            }
                        }
                        if (string.IsNullOrEmpty(codelanguage))
                        {
                            codelanguage = "cs";
                        }
                    }

                    var isLexerOnly = true;
                    if (doc.HasNonTerminalProductions)
                    {
                        isLexerOnly = false;
                    }
                    else
                    {
                        foreach (var include in doc.Includes)
                        {
                            if (include.Document.HasNonTerminalProductions)
                            {
                                isLexerOnly = false;
                                break;
                            }
                        }
                    }
                    // we need to prepare it by marking every terminal
                    // with an attribute if it isn't already. we use
                    // "terminal" because it doesn't impact terminals
                    // in any way, but this way the CfgDocument can
                    // "see" them.
                    for (int ic = doc.Productions.Count, i = 0; i < ic; ++i)
                    {
                        var p = doc.Productions[i];
                        if (p.IsTerminal && 0 == p.Attributes.Count)
                        {
                            p.Attributes.Add(new XbnfAttribute("terminal", true));
                        }
                    }

                    XbnfGenerationInfo genInfo;
                    var msgs = XbnfConvert.TryCreateGenerationInfo(doc, out genInfo);

                    foreach (var msg in msgs)
                    {
                        if (verbose || ErrorLevel.Information != msg.ErrorLevel)
                        {
                            stderr.WriteLine(msg);
                        }
                    }
                    foreach (var msg in msgs)
                    {
                        if (msg.ErrorLevel == ErrorLevel.Error)
                        {
                            throw new Exception(msg.ToString());
                        }
                    }
                    CfgDocument primaryCfg = genInfo.Cfg;
                    doc = genInfo.Xbnf;
                    if (!isLexerOnly)
                    {
                        if (verbose)
                        {
                            stderr.WriteLine("Final grammar:");
                            stderr.WriteLine(primaryCfg.ToString());
                            stderr.WriteLine();
                        }
                        foreach (var msg in msgs)
                        {
                            if (msg.ErrorLevel == ErrorLevel.Error)
                            {
                                throw new Exception(msg.ToString());
                            }
                        }
                        if (!noparser)
                        {
                            var ccu = CodeGenerator.GenerateCompileUnit(genInfo, codeclass, codenamespace, fast);
                            ccu.Namespaces.Add(new CodeNamespace(codenamespace ?? ""));
                            var ccuNS     = ccu.Namespaces[ccu.Namespaces.Count - 1];
                            var ccuShared = CodeGenerator.GenerateSharedCompileUnit(codenamespace);
                            ccu.ReferencedAssemblies.Add(typeof(TypeConverter).Assembly.GetName().ToString());

                            if (fast)
                            {
                                CD.CodeDomVisitor.Visit(ccu, (ctx) =>
                                {
                                    var vd = ctx.Target as CodeVariableDeclarationStatement;
                                    if (null != vd && CD.CodeDomResolver.IsNullOrVoidType(vd.Type))
                                    {
                                        vd.Type = C.Type("var");
                                    }
                                }, CD.CodeDomVisitTargets.All & ~(CD.CodeDomVisitTargets.Expressions | CD.CodeDomVisitTargets.Comments | CD.CodeDomVisitTargets.Attributes | CD.CodeDomVisitTargets.Directives | CD.CodeDomVisitTargets.Types | CD.CodeDomVisitTargets.TypeRefs));
                                CD.CodeDomVisitor.Visit(ccuShared, (ctx) =>
                                {
                                    var vd = ctx.Target as CodeVariableDeclarationStatement;
                                    if (null != vd && CD.CodeDomResolver.IsNullOrVoidType(vd.Type))
                                    {
                                        vd.Type = C.Type("var");
                                    }
                                }, CD.CodeDomVisitTargets.All & ~(CD.CodeDomVisitTargets.Expressions | CD.CodeDomVisitTargets.Comments | CD.CodeDomVisitTargets.Attributes | CD.CodeDomVisitTargets.Directives | CD.CodeDomVisitTargets.Types | CD.CodeDomVisitTargets.TypeRefs));
                            }
                            else
                            {
                                SlangPatcher.Patch(ccu, ccuShared);
                                var co = SlangPatcher.GetNextUnresolvedElement(ccu);
                                if (null != co)
                                {
                                    stderr.WriteLine("Warning: Not all of the elements could be resolved. The generated code may not be correct in all languages.");
                                    stderr.WriteLine("  Next unresolved: {0}", C.ToString(co).Trim());
                                }
                            }
                            if (!noshared)
                            {
                                CodeGenerator.ImportCompileUnit(ccuNS, ccuShared);
                            }

                            var prov = CodeDomProvider.CreateProvider(codelanguage);

                            if (null != outputfile)
                            {
                                var sw = new StreamWriter(outputfile);
                                sw.BaseStream.SetLength(0);
                                output = sw;
                            }
                            else
                            {
                                output = stdout;
                            }
                            var opts = new CodeGeneratorOptions();
                            opts.VerbatimOrder            = true;
                            opts.BlankLinesBetweenMembers = false;
                            prov.GenerateCodeFromCompileUnit(ccu, output, opts);
                            output.Flush();
                            output.Close();
                            output = null;
                        }
                    }
                    else
                    {
                        stderr.WriteLine("{0} skipped parser generation because there are no non-terminals and no imports defined.", Name);
                    }

                    if (null != rolexfile)
                    {
                        var sw = new StreamWriter(rolexfile);
                        sw.BaseStream.SetLength(0);
                        output = sw;
                        output.WriteLine(XbnfConvert.ToRolexSpec(genInfo));
                        output.Flush();
                        output.Close();
                        output = null;
                    }
                    if (null != yaccfile)
                    {
                        var sw = new StreamWriter(yaccfile);
                        sw.BaseStream.SetLength(0);
                        output = sw;
                        output.WriteLine(genInfo.Cfg.ToString("y"));
                        output.Flush();
                        output.Close();
                        output = null;
                    }
                }
            }
#if !DEBUG
            catch (Exception ex)
            {
                result = _ReportError(ex, stderr);
            }
#endif
            finally
            {
                stderr.Close();
                stdout.Close();
                if (outputfile != null && null != output)
                {
                    output.Close();
                }
            }
            return(result);
        }