Exemplo n.º 1
0
        public string GenerateGrammar(Grammar grammar)
        {
            string ret = "";

            var defs = GetThings(grammar.Root, ThingType.Def);
            var tokens = GetThings(grammar.Root, ThingType.Token);
            var texts = GetThings(grammar.Root, ThingType.Text);

            foreach (Thing def in defs)
            {
                ret += Environment.NewLine + def.Name + " :";

                ret = DefChidren(def, "", ref ret);
            }
            ret += Environment.NewLine;

            foreach (Thing def in tokens)
            {
                ret += Environment.NewLine + def.Name + " : " + def.Text;
            }
            ret += Environment.NewLine;

            foreach (Thing def in texts)
            {
                ret += Environment.NewLine + def.Name + " : " + def.Text;
            }

            return ret;
        }
Exemplo n.º 2
0
        public void ObjectRef()
        {
            var dot = new Token("dot", ".");
            var text = new Text("Text", ".+");

            var root = new Def("ObjectRef", text, new Optional(dot, text), new Optional(dot, text));

            Grammar grammar = new Grammar
            {
                StringQuote = '\'',
                Root = root
            };

            string actual = GenerateAndBuildParser(grammar, "abc");

            Assert.That(actual, Is.EqualTo(@"
            ObjectRef
             Text - abc"));

            actual = GenerateAndBuildParser(grammar, "abc.def");

            Assert.That(actual, Is.EqualTo(@"
            ObjectRef
             Text - abc
             Dot
             Text - def"));

            actual = GenerateAndBuildParser(grammar, "abc.def.ghi");

            Assert.That(actual, Is.EqualTo(@"
            ObjectRef
             Text - abc
             Dot
             Text - def
             Dot
             Text - ghi"));
        }
Exemplo n.º 3
0
        private string GenerateAndBuildParser(Grammar grammar, string text)
        {
            Generator generator = new Generator();
            Builder builder = new Builder();
            string lexerDef = generator.GenerateLexer(grammar);

            string parserDef = generator.GenerateParser(grammar);

            var assembly = builder.Build(lexerDef, parserDef);

            object parser = Activator.CreateInstance(assembly.GetType("Xxx.Parser"));
            object walker = Activator.CreateInstance(assembly.GetType("Xxx.Walker"));

            var node = parser.GetType().GetMethod("Parse").Invoke(parser, new object[] { text });
            return (string)walker.GetType().GetMethod("NodesToString").Invoke(walker, new[] { node });
        }
Exemplo n.º 4
0
        public void Table()
        {
            var _as = new Token("as");
            var _space = new Token("WhiteSpace", " ");
            var text = new Text("Text", ".+");

            var root = new Def("Table", text, new Optional(_as), new Optional(text));

            Grammar grammar = new Grammar
            {
                StringQuote = '\'',
                Root = root,
                IgnoreTokens = new[] { _space }
            };

            string actual = GenerateAndBuildParser(grammar, "abc");

            Assert.That(actual, Is.EqualTo(@"
            Table
             Text - abc"));

            actual = GenerateAndBuildParser(grammar, "abc ABC");

            Assert.That(actual, Is.EqualTo(@"
            Table
             Text - abc
             Text - ABC"));

            actual = GenerateAndBuildParser(grammar, "abc as ABC");

            Assert.That(actual, Is.EqualTo(@"
            Table
             Text - abc
             As
             Text - ABC"));
        }
Exemplo n.º 5
0
        public void StarOrObjectRef()
        {
            var dot = new Token("dot", ".");
            var text = new Text("Text", ".+");
            var star = new Token("star", "*");
            var _space = new Token("whitespace", " ");

            var objectRef = new Def("ObjectRef", text, new Optional(dot, text), new Optional(dot, text));
            var root = new Def("StarOrObjectRef", new OneOf(star, objectRef));

            Grammar grammar = new Grammar
            {
                StringQuote = '\'',
                Root = root,
                IgnoreTokens = new[] { _space }
            };

            string actual = GenerateAndBuildParser(grammar, "*");

            Assert.That(actual, Is.EqualTo(@""));
        }
Exemplo n.º 6
0
        public Grammar BuildGrammar(Node<GrammarGrammar.NodeType> parent)
        {
            Grammar grammar = new Grammar();

            var nameNode = parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Text);

            if (nameNode != null)
            {
                grammar.Name = nameNode.Text;
            }

            List<Token> punctuation = GetTokens(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Punctuation));
            List<Token> keywords = GetTokens(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Keywords));
            List<Token> texts = GetTokens(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Texts));

            grammar.Punctuation = punctuation.Select(x => new Token(x.Name, x.Text)).ToList();
            grammar.Keywords = keywords.Select(x => new Token(x.Name, x.Text)).ToList();
            grammar.Texts = texts.Select(x => new Token(x.Name, x.Text)).ToList();

            List<string> ignoreNames = GetNames(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Ignore));
            List<string> discardNames = GetNames(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Discard));

            List<Def> defs = new List<Def>();

            List<Node<GrammarGrammar.NodeType>> reversed = parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Defs).Children;

            reversed.Reverse();

            foreach (var def in reversed)
            {
                var textNode = def.Children.First(x => x.NodeType == GrammarGrammar.NodeType.Text);

                var deff = new Def(textNode.Text);
                defs.Add(deff);

                foreach (var part in def.Children.Where(x => x.NodeType == GrammarGrammar.NodeType.Part))
                {
                    Thing thing = null;
                    Thing thing2 = null;

                    var names = part.Children.Single(x => x.NodeType == GrammarGrammar.NodeType.Names);

                    if (names.Children.Last().NodeType == GrammarGrammar.NodeType.Plus
                        || part.Children.Last().NodeType == GrammarGrammar.NodeType.Plus)
                    {
                        thing = thing2 = new OneOrMore();
                    }
                    else if (names.Children.Last().NodeType == GrammarGrammar.NodeType.Star
                        || part.Children.Last().NodeType == GrammarGrammar.NodeType.Star)
                    {
                        thing = thing2 = new ZeroOrMore();
                    }

                    if (part.Children.First().NodeType == GrammarGrammar.NodeType.OpenSquare)
                    {
                        if (thing == null)
                        {
                            thing = thing2 = new Optional();
                        }
                        else
                        {
                            thing2 = new Optional();
                            thing.Children = new List<Thing>() { thing2 };
                        }
                    }


                    if (names.Children.Any(x => x.NodeType == GrammarGrammar.NodeType.Pipe))
                    {
                        if (thing == null)
                        {
                            thing = thing2 = new OneOf();
                        }
                        else
                        {
                            thing2 = new OneOf();
                            thing.Children = new List<Thing>() { thing2 };
                        }
                    }

                    List<Thing> things = new List<Thing>();

                    foreach (var name in names.Children.Where(x => x.NodeType == GrammarGrammar.NodeType.Text))
                    {
                        Thing childThing = null;

                        var token = grammar.Punctuation.SingleOrDefault(x => x.Name == name.Text);
                        if (token != null)
                        {
                            things.Add(token);
                        }
                        else
                        {
                            token = grammar.Keywords.SingleOrDefault(x => x.Name == name.Text);
                            if (token != null)
                            {
                                things.Add(token);
                            }
                            else
                            {
                                token = grammar.Texts.SingleOrDefault(x => x.Name == name.Text);
                                if (token != null)
                                {
                                    things.Add(token);
                                }
                                else
                                {
                                    var def2 = defs.SingleOrDefault(x => x.Name == name.Text);
                                    if (def2 != null)
                                    {
                                        things.Add(def2);
                                    }
                                    else
                                    {
                                        throw new Exception();
                                    }
                                }
                            }
                        }
                    }

                    if (thing == null)
                    {
                        deff.Children.AddRange(things);
                    }
                    else
                    {
                        thing2.Children = things;
                        deff.Children.Add(thing);
                    }
                }
            }

            grammar.Root = defs.Last();

            List<Token> ignoreTokens = new List<Token>();
            List<string> discardThings = new List<string>();

            ignoreTokens.AddRange(keywords.Where(x => ignoreNames.Contains(x.Name)));
            ignoreTokens.AddRange(punctuation.Where(x => ignoreNames.Contains(x.Name)));

            discardThings.AddRange(keywords.Where(x => discardNames.Contains(x.Name)).Select(x => x.Name));
            discardThings.AddRange(punctuation.Where(x => discardNames.Contains(x.Name)).Select(x => x.Name));
            discardThings.AddRange(defs.Where(x => discardNames.Contains(x.Name)).Select(x => x.Name));

            grammar.IgnoreTokens = ignoreTokens.ToArray();
            grammar.DiscardThings = discardThings.ToArray();

            grammar.StringQuote = '\'';
            return grammar;
        }
Exemplo n.º 7
0
        public string GenerateParser(Grammar grammar)
        {
            StringBuilder stringBuilder = new StringBuilder();

            var defs = GetThings(grammar.Root, ThingType.Def);
            var tokens = GetThings(grammar.Root, ThingType.Token);
            var texts = GetThings(grammar.Root, ThingType.Text);
            //tokens.AddRange(grammar.DiscardThings.Where(x => !tokens.Contains(x)));

            stringBuilder.Append(
                $@"using System;
using System.Collections.Generic;
using Parsing.Core;

namespace Xxx
{{
    public class {grammar.Name}Parser : ParserBase<TokenType, NodeType>
    {{
        private readonly List<string> _discardThings;

        public {grammar.Name}Parser() : base(new {grammar.Name}Lexer())
        {{
            _discardThings = new List<string>
            {{");

            foreach (var text in grammar.DiscardThings)
            {
                stringBuilder.Append($@"
                ""{text.ToIdentifier()}"",");
            }

            stringBuilder.Append($@"
            }};
        }}

        public override List<string> DiscardThings {{ get {{ return _discardThings; }} }}

        public override Node<NodeType> Root()
        {{
            return {grammar.Root.Name}(null);
        }}");

            foreach (Thing def in defs)
            {
                string parentOrChild = grammar.DiscardThings.All(x => x != def.Name)
                    ? "child"
                    : "parent";

                stringBuilder.Append($@"

        public Node<NodeType> {def.Name}(Node<NodeType> parent)
        {{");
                if (grammar.DiscardThings.All(x => x != def.Name))
                {
                    stringBuilder.Append($@"
            var {parentOrChild} = Add(parent, NodeType.{def.Name});");
                }

                GenerateParserDef(def, parentOrChild, stringBuilder, 0);

                stringBuilder.Append($@"

            return {parentOrChild};
        }}");
            }

            stringBuilder.Append(@"
    }

    public enum NodeType
    {");

            foreach (var def in defs.Where(x => !IsNullOrWhiteSpace(x.Name)).Select(x => x.Name).Distinct())
            {
                stringBuilder.Append(@"
        " + def + ",");
            }

            foreach (var text in texts.Where(x => !IsNullOrWhiteSpace(x.Name)).Select(x => x.Name.ToIdentifier()).Distinct())
            {
                stringBuilder.Append(@"
        " + text + ",");
            }

            foreach (var token in tokens.Where(x => !IsNullOrWhiteSpace(x.Name)).Select(x => x.Name.ToIdentifier()).Distinct())
            {
                stringBuilder.Append(@"
        " + token + ",");
            }

            stringBuilder.Append(@"
    }

    public class Walker
    {
        public string NodesToString(object node)
        {
            Node<NodeType> parent = node as Node<NodeType>;

            string ret = """";

            NodesToString(parent, ref ret, 0);

            return ret;
        }

        private void NodesToString(Node<NodeType> parent, ref string ret, int indent)
        {
            ret += Environment.NewLine + new string(' ', indent) + parent.NodeType +(String.IsNullOrWhiteSpace(parent.Text) ? """" : "" - "" + parent.Text);
            foreach (Node<NodeType> child in parent.Children)
            {
                NodesToString(child, ref ret, indent + 1);
            }
        }
    }
}");

            return stringBuilder.ToString();
        }
Exemplo n.º 8
0
        public string GenerateLexer(Grammar grammar)
        {
            StringBuilder stringBuilder = new StringBuilder();

            stringBuilder.Append($@"using System.Collections.Generic;
using Parsing.Core;

namespace Xxx
{{
    public class {grammar.Name}Lexer : LexerBase<TokenType>
    {{
        private readonly Dictionary<char, TokenType> _punctuation;
        private readonly Dictionary<string, TokenType> _keywords;
        private readonly Dictionary<string, TokenType> _texts;
        private readonly List<TokenType> _ignoreTokenTypes;
        private readonly char _stringQuote;

        public {grammar.Name}Lexer()
        {{
            _punctuation = new Dictionary<char, TokenType>
            {{");

            foreach (Token token in grammar.Punctuation)
            {
                stringBuilder.Append($@"
                {{ '{token.Text}', TokenType.{token.Name.ToIdentifier()} }},");
            }

            stringBuilder.Append($@"
            }};

            _keywords = new Dictionary<string, TokenType>
            {{");

            foreach (Token token in grammar.Keywords)
            {
                stringBuilder.Append($@"
                {{ ""{token.Text}"", TokenType.{token.Name.ToIdentifier()} }},");
            }

            stringBuilder.Append($@"
            }};

            _texts = new Dictionary<string, TokenType>
            {{");

            foreach (Token text in grammar.Texts)
            {
                stringBuilder.Append($@"
                {{ ""^{text.Text}$"", TokenType.{text.Name} }},");
            }

            stringBuilder.Append($@"
            }};

            _ignoreTokenTypes = new List<TokenType>
            {{");

            foreach (Token text in grammar.IgnoreTokens)
            {
                stringBuilder.Append($@"
                TokenType.{text.Name.ToIdentifier()},");
            }

            stringBuilder.Append($@"
            }};");

            stringBuilder.Append($@"
            _stringQuote = '{(grammar.StringQuote == '\'' ? "\\" : "") + grammar.StringQuote}';
        }}");

            stringBuilder.Append(@"

        public override TokenType EndOfFileTokenType { get { return TokenType.EndOfFile; } }
        public override TokenType StringTokenType { get { return TokenType.String; } }
        public override Dictionary<char, TokenType> Punctuation { get { return _punctuation; } }
        public override Dictionary<string, TokenType> KeyWords { get { return _keywords; } }
        public override Dictionary<string, TokenType> Texts { get { return _texts; } }
        public override List<TokenType> IgnoreTokenTypes { get { return _ignoreTokenTypes; } }
        public override char StringQuote { get { return _stringQuote; } }
    }
");


            stringBuilder.Append(@"

    public enum TokenType
    {
        EndOfFile,
        String,");

            foreach (var text in grammar.Punctuation)
            {
                stringBuilder.Append(@"
        " + text.Name.ToIdentifier() + ",");
            }

            foreach (var token in grammar.Keywords)
            {
                stringBuilder.Append(@"
        " + token.Name.ToIdentifier() + ",");
            }

            foreach (var token in grammar.Texts)
            {
                stringBuilder.Append(@"
        " + token.Name.ToIdentifier() + ",");
            }

            stringBuilder.Append(@"
    }
}");

            return stringBuilder.ToString();
        }
Exemplo n.º 9
0
        private void RefreshGrammar(Grammar grammar)
        {
            genGrammar.Text = grammar.Name;

            var defs = GetThings(grammar.Root, ThingType.Def);

            foreach (var def in defs)
            {
                RefreshGrammar(def);
            }
        }