private void AddElements(Grammar grammar, List<Element> elements, TreeNode parentNode) { foreach (Element element in elements) { var patternIdentifier = element as PatternIdentifier; var defIdentifier = element as DefIdentifier; var oneOf = element as OneOf; var allOf = element as AllOf; var optional = element as Optional; var oneOrMore = element as OneOrMore; var zeroOrMore = element as ZeroOrMore; TreeNode childNode; if (patternIdentifier != null) { childNode = new PatternIdentifierTreeNode(patternIdentifier); } else if (defIdentifier != null) { childNode = new DefIdentifierTreeNode(defIdentifier); AddElements(grammar, grammar.Defs.Single(x => x.Name == (element).Name).Elements, childNode); } else if (oneOf != null) { childNode = new OneOfTreeNode(oneOf); AddElements(grammar, oneOf.Elements.ToList(), childNode); } else if (allOf != null) { childNode = new AllOfTreeNode(allOf); AddElements(grammar, allOf.Elements.ToList(), childNode); } else if (optional != null) { childNode = new OptionalTreeNode(optional); AddElements(grammar, new List<Element> { optional.Element }, childNode); } else if (oneOrMore != null) { childNode = new OneOrMoreTreeNode(oneOrMore); AddElements(grammar, new List<Element> { oneOrMore.Element }, childNode); } else if (zeroOrMore != null) { childNode = new ZeroOrMoreTreeNode(zeroOrMore); AddElements(grammar, new List<Element> { zeroOrMore.Element }, childNode); } else { throw new Exception(); } parentNode.Nodes.Add(childNode); } }
private void AddDefNodes(Grammar grammar, TreeNode parentNode) { foreach (Def def in grammar.Defs) { TreeNode defNode = new TreeNode(def.Name); parentNode.Nodes.Add(defNode); AddElements(grammar, def.Elements, defNode); } }
public string BuildTokenType(Grammar grammar) { string ret = $@"namespace {grammar.Name} {{ public enum TokenType {{ EndOfFile,"; foreach (var pattern in grammar.Patterns.Select(x => x.Name.ToIdentifier()).Distinct()) { ret += $@" {pattern},"; } ret += @" } }"; return ret; }
private void BuildParser2(Grammar grammar, Element element, ref string ret) { PatternIdentifier patternIdentifier = element as PatternIdentifier; DefIdentifier defIdentifier = element as DefIdentifier; AllOf allOf = element as AllOf; OneOf oneOf = element as OneOf; Optional optional = element as Optional; if (patternIdentifier != null) { ret += $@" Consume(child, TokenType.{patternIdentifier.Name.ToIdentifier()}, NodeType.{patternIdentifier.Name.ToIdentifier()});"; } else if (defIdentifier != null) { ret += $@" {defIdentifier.Name}(child);"; } else if (oneOf != null) { if (oneOf.Elements.Count == 1) { patternIdentifier = oneOf.Elements[0] as PatternIdentifier; defIdentifier = oneOf.Elements[0] as DefIdentifier; if (patternIdentifier != null) { ret += $@" Consume(child, TokenType.{patternIdentifier.Name.ToIdentifier()}, NodeType.{patternIdentifier.Name.ToIdentifier()});"; } else if (defIdentifier != null) { ret += $@" {defIdentifier.Name}(child);"; } } else { bool first = true; foreach (var identifier in oneOf.Elements) { patternIdentifier = identifier as PatternIdentifier; ret += $@" {(first ? "" : "else ")}if ({GetAreTokenTypes(grammar, identifier)}) {{"; if (patternIdentifier != null) { ret += $@" Consume(child, TokenType.{patternIdentifier.Name.ToIdentifier()}, NodeType.{patternIdentifier.Name.ToIdentifier()});"; } else { ret += $@" {identifier.Name}(child);"; } ret += $@" }}"; first = false; } } } else if (allOf != null) { foreach (var identifier in allOf.Elements) { patternIdentifier = identifier as PatternIdentifier; if (patternIdentifier != null) { ret += $@" Consume(child, TokenType.{patternIdentifier.Name.ToIdentifier()}, NodeType.{patternIdentifier.Name.ToIdentifier()});"; } else { ret += $@" {identifier.Name}(child);"; } } } else if (optional != null) { BuildParser2(grammar, optional.Element, ref ret); } else { throw new Exception(); } }
private void GetTokenTypes(Grammar grammar, Element element, ref List<string> tokenTypes) { PatternIdentifier patternIdentifier = element as PatternIdentifier; DefIdentifier defIdentifier = element as DefIdentifier; //AllOf allOf = element as AllOf; //OneOf oneOf = element as OneOf; //OneOrMore oneOrMore = element as OneOrMore; //ZeroOrMore zeroOrMore = element as ZeroOrMore; //Optional optional = element as Optional; if (patternIdentifier != null) { tokenTypes.Add(patternIdentifier.Name.ToIdentifier()); } else if (defIdentifier != null) { Def def = grammar.Defs.Single(x => x.Name == defIdentifier.Name); foreach (var element1 in def.Elements) { GetTokenTypes(grammar, element1, ref tokenTypes); } } else { foreach (var identifier in element.Elements) { GetTokenTypes(grammar, identifier, ref tokenTypes); } } }
private List<string> GetTokenTypes(Grammar grammar, Element element) { List<string> tokenTypes = new List<string>(); GetTokenTypes(grammar, element, ref tokenTypes); return tokenTypes; }
private string GetAreTokenTypes(Grammar grammar, Element element) { List<string> tokenTypes = GetTokenTypes(grammar, element); List<List<string>> otherTokenTypes = new List<List<string>>(); var others = GetOthers(element); foreach (var element2 in others) { otherTokenTypes.Add(GetTokenTypes(grammar, element2)); } string[] names = { }; if (otherTokenTypes.Count == 0) { names = new[] { tokenTypes[0] }; } else { for (int i = 0; i < tokenTypes.Count; ++i) { names = tokenTypes.Take(i + 1).ToArray(); var tokenTypesString = String.Join(".", names); if (otherTokenTypes.All(x => String.Join(".", x.Take(i + 1)) != tokenTypesString)) { break; } } } return $"AreTokenTypes({ String.Join(", ", names.Select(x => $"TokenType.{x.ToIdentifier()}")) })"; }
public string BuildParser2(Grammar grammar) { string ret = $@"using System.Collections.Generic; using V2.Parsing.Core; namespace {grammar.Name} {{ public class Parser : ParserBase<TokenType, NodeType> {{ public Parser() : base(new Lexer()) {{ base.Discard = new List<string> {{"; foreach (var discard in grammar.Discards) { ret += $@" ""{discard.Name.ToIdentifier()}"","; } ret += $@" }}; }} public override Node<NodeType> Root() {{ Node<NodeType> root = new Node<NodeType>(null, NodeType.Root); return {grammar.Defs.First().Name}(root); }}"; foreach (var def in grammar.Defs) { ret += $@" public Node<NodeType> {def.Name}(Node<NodeType> parent) {{ var child = Add(parent, NodeType.{def.Name}); "; foreach (var element in def.Elements) { PatternIdentifier patternIdentifier = element as PatternIdentifier; DefIdentifier defIdentifier = element as DefIdentifier; Optional optional = element as Optional; OneOrMore oneOrMore = element as OneOrMore; ZeroOrMore zeroOrMore = element as ZeroOrMore; OneOf oneOf = element as OneOf; if (patternIdentifier != null) { ret += $@" Consume(child, TokenType.{patternIdentifier.Name.ToIdentifier()}, NodeType.{patternIdentifier.Name.ToIdentifier()});"; } else if (defIdentifier != null) { ret += $@" {defIdentifier.Name.ToIdentifier()}(child);"; } else if (optional != null) { ret += $@" if ({GetAreTokenTypes(grammar, element)}) {{"; BuildParser2(grammar, optional.Element, ref ret); ret += $@" }}"; } else if (zeroOrMore != null) { ret += $@" while ({GetAreTokenTypes(grammar, element)}) {{"; BuildParser2(grammar, zeroOrMore.Element, ref ret); ret += $@" }}"; } else if (oneOrMore != null) { ret += $@" do {{"; BuildParser2(grammar, oneOrMore.Element, ref ret); ret += $@" }} while ({GetAreTokenTypes(grammar, element)});"; } else if (oneOf != null) { ret += $@" "; bool first = true; foreach (var identifier in oneOf.Elements) { if (identifier is PatternIdentifier) { ret += $@" {(first ? "" : "else ")}if ({GetAreTokenTypes(grammar, identifier)}) {{ Consume(child, TokenType.{identifier.Name.ToIdentifier()}, NodeType.{identifier.Name.ToIdentifier()}); }}"; } else { ret += $@" {(first ? "" : "else ")}if ({GetAreTokenTypes(grammar, identifier)}) {{ {identifier.Name}(child); }}"; } first = false; } } else { throw new Exception(); } } ret += $@" return child; }}"; } ret += $@" }} }}"; return ret; }
public string BuildNodeType(Grammar grammar) { string ret = $@"namespace {grammar.Name} {{ public enum NodeType {{ Root,"; var names = grammar.Defs.Select(x => x.Name).ToList(); names.AddRange(grammar.Patterns .Select(x => x.Name.ToIdentifier())); foreach (var name in names.Distinct()) { ret += $@" {name},"; } ret += @" } }"; return ret; }
private List<Element> GetIdentifiers(Grammar grammar, List<Node<NodeType>> optionalIdents) { return optionalIdents.Select(x => grammar.Patterns.Any(y => y.Name == x.Text) ? new PatternIdentifier { Name = x.Text } as Element : new DefIdentifier { Name = x.Text }) .ToList(); }
private void SetParents(Grammar grammar) { foreach (Def def in grammar.Defs) { SetParents(def); } }
public Grammar BuildGrammar(Node<NodeType> root) { Grammar grammar = new Grammar { Name = root.Children.Single(x => x.NodeType == NodeType.Identifier).Text, CaseSensitive = root.Children.Any(x => x.NodeType == NodeType.CaseSensitive) }; var patternsNode = root.Children.SingleOrDefault(x => x.NodeType == NodeType.Patterns); if (patternsNode != null) { foreach (var patternNode in patternsNode.Children.Where(x => x.NodeType == NodeType.Pattern && x.Children.Count > 0)) { var children = patternNode.Children.Where(x => x.NodeType == NodeType.Identifier).ToArray(); var pattern = new Pattern { Name = children[0].Text, Texts = children.Skip(1).Select(x => x.Text).ToArray() }; grammar.Patterns.Add(pattern); } } var defsNode = root.Children.SingleOrDefault(x => x.NodeType == NodeType.Defs); if (defsNode != null) { foreach (var defNode in defsNode.Children.Where(x => x.NodeType == NodeType.Def && x.Children.Count > 0)) { var children = defNode.Children.Where(x => x.NodeType == NodeType.Identifier).ToArray(); var def = new Def { Name = children[0].Text }; grammar.Defs.Add(def); } foreach (var defNode in defsNode.Children.Where(x => x.NodeType == NodeType.Def)) { var def = grammar.Defs.Single(x => x.Name == defNode.Children.Single(y => y.NodeType == NodeType.Identifier).Text); foreach (var partNode in defNode.Children.Where(x => x.NodeType == NodeType.Part)) { Element element = null; if (partNode.Children.Count == 1 && partNode.Children[0].NodeType == NodeType.Optional) { var optionalIdents = partNode.Children[0].Children.SingleOrDefault(x => x.NodeType == NodeType.OptionalIdents); var requiredIdents = partNode.Children[0].Children.SingleOrDefault(x => x.NodeType == NodeType.RequiredIdents); Element subElement; if (optionalIdents != null) { subElement = new OneOf(GetIdentifiers(grammar, optionalIdents.Children.Where(x => x.NodeType != NodeType.Pipe).ToList())); } else if (requiredIdents != null) { subElement = new AllOf(GetIdentifiers(grammar, requiredIdents.Children)); } else { throw new Exception(); } element = new Optional(subElement); } else { List<Element> identifiers = new List<Element>(); foreach (var elementNode in partNode.Children.Where(x => x.NodeType == NodeType.Element)) { Element identifier; if (grammar.Patterns.Any(x => x.Name == elementNode.Children[0].Text)) { identifier = new PatternIdentifier { Name = elementNode.Children[0].Text }; } else { identifier = new DefIdentifier { Name = elementNode.Children[0].Text }; } element = identifier; if (elementNode.Children.Any(x => x.NodeType == NodeType.Plus)) { element = new OneOrMore (element); } else if (elementNode.Children.Any(x => x.NodeType == NodeType.Star)) { element = new ZeroOrMore(element); } identifiers.Add(identifier); } if (identifiers.Count > 1) { element = new OneOf(identifiers); } } if (element == null) { throw new Exception(); } if (partNode.Children.Count == 1 && partNode.Children[0].NodeType == NodeType.Optional) { if (partNode.Children[0].Children.Any(x => x.NodeType == NodeType.Plus)) { element = new OneOrMore(element); } else if (partNode.Children[0].Children.Any(x => x.NodeType == NodeType.Star)) { element = new ZeroOrMore(element); } } else { if (partNode.Children.Any(x => x.NodeType == NodeType.Plus)) { element = new OneOrMore(element); } else if (partNode.Children.Any(x => x.NodeType == NodeType.Star)) { element = new ZeroOrMore(element); } } def.Elements.Add(element); } } } var ignoresNode = root.Children.SingleOrDefault(x => x.NodeType == NodeType.Ignores); if (ignoresNode != null) { foreach (var ignoreNode in ignoresNode.Children) { grammar.Ignores.Add(new Ignore { Name = ignoreNode.Text }); } } var discardsNode = root.Children.SingleOrDefault(x => x.NodeType == NodeType.Discards); if (discardsNode != null) { foreach (var discardNode in discardsNode.Children) { grammar.Discards.Add(new Discard { Name = discardNode.Text }); } } SetParents(grammar); return grammar; }
public string GrammarToDefString(Grammar grammar) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendLine($"grammar {MakeSafe(grammar.Name)}"); if (grammar.CaseSensitive) { stringBuilder.AppendLine("caseSensitive"); } if (grammar.Defs != null && grammar.Defs.Count > 0) { stringBuilder.AppendLine("defs"); foreach (var def in grammar.Defs) { stringBuilder.Append($" {MakeSafe(def.Name)} :"); foreach (var element in def.Elements) { stringBuilder.Append(" "); ElementToString(stringBuilder, element); } stringBuilder.AppendLine(); } } if (grammar.Patterns != null && grammar.Patterns.Count > 0) { stringBuilder.AppendLine("patterns"); foreach (var pattern in grammar.Patterns) { stringBuilder.Append($" {MakeSafe(pattern.Name)}"); if (pattern.Texts.Length > 0) { stringBuilder.Append($" : {String.Join(" ", pattern.Texts.Select(MakeSafe))}"); } stringBuilder.AppendLine(); } } if (grammar.Ignores != null && grammar.Ignores.Count > 0) { stringBuilder.AppendLine("ignore"); foreach (var ignore in grammar.Ignores) { stringBuilder.AppendLine($" {MakeSafe(ignore.Name)}"); } } if (grammar.Discards != null && grammar.Discards.Count > 0) { stringBuilder.AppendLine("discard"); foreach (var discard in grammar.Discards) { stringBuilder.AppendLine($" {MakeSafe(discard.Name)}"); } } return stringBuilder.ToString(); }
public string BuildLexer(Grammar grammar) { string ret = $@"using System.Collections.Generic; using V2.Parsing.Core; namespace {grammar.Name} {{ public class Lexer : LexerBase<TokenType> {{ public Lexer() {{ EndOfFile = TokenType.EndOfFile; Patterns = new List<PatternBase<TokenType>> {{"; foreach (var pattern in grammar.Patterns) { string patternType = "Token"; if (pattern.Texts.Length == 1 && pattern.Texts[0].StartsWith("^")) { patternType = "Regex"; } else if (pattern.Texts.Length > 1) { patternType = "String"; } ret += $@" new {patternType}Pattern<TokenType>(TokenType.{pattern.Name.ToIdentifier()}, "; if (pattern.Texts.Length == 0) { ret += $@"""{pattern.Name}"""; } else { ret += String.Join(", ", pattern.Texts.Select(x => $@"""{x.Replace("`", "'")}""")); } ret += "),"; } ret += $@" }}; Ignore = new List<TokenType> {{"; foreach (var v in grammar.Ignores) { ret += $@" TokenType.{v.Name.ToIdentifier()},"; } ret += $@" }}; CaseSensitive = {(grammar.CaseSensitive ? "true" : "false")}; }} }} }}"; return ret; }
public string BuildParser(Grammar grammar) { string ret = $@"using System.Collections.Generic; using V2.Parsing.Core; namespace {grammar.Name} {{ public class Parser : ParserBase<TokenType, NodeType> {{ public Parser() : base(new Lexer()) {{ base.Discard = new List<string> {{"; foreach (var discard in grammar.Discards) { ret += $@" ""{discard.Name}"","; } ret += $@" }}; }} public override Node<NodeType> Root() {{ Node<NodeType> root = new Node<NodeType>(null, NodeType.Root); return {grammar.Defs.First().Name}(root); }} public Node<NodeType> Grammar(Node<NodeType> parent) {{ var child = Consume(parent, TokenType.Grammar, NodeType.Grammar); Consume(child, TokenType.Identifier, NodeType.Identifier); if (AreTokenTypes(TokenType.NewLine, TokenType.CaseSensitive)) {{ Consume(child, TokenType.NewLine, NodeType.NewLine); Consume(child, TokenType.CaseSensitive, NodeType.CaseSensitive); }} if (AreTokenTypes(TokenType.NewLine, TokenType.Defs)) {{ Consume(child, TokenType.NewLine, NodeType.NewLine); Defs(child); }} if (AreTokenTypes(TokenType.NewLine, TokenType.Patterns)) {{ Consume(child, TokenType.NewLine, NodeType.NewLine); Patterns(child); }} if (AreTokenTypes(TokenType.NewLine, TokenType.Ignore)) {{ Consume(child, TokenType.NewLine, NodeType.NewLine); Ignores(child); }} if (AreTokenTypes(TokenType.NewLine, TokenType.Discard)) {{ Consume(child, TokenType.NewLine, NodeType.NewLine); Discards(child); }} return child; }} public Node<NodeType> Defs(Node<NodeType> parent) {{ var child = Consume(parent, TokenType.Defs, NodeType.Defs); while (AreTokenTypes(TokenType.NewLine, TokenType.Identifier, TokenType.Colon)) {{ Def(child); }} return child; }} public Node<NodeType> Def(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Def); Consume(child, TokenType.NewLine, NodeType.NewLine); Consume(child, TokenType.Identifier, NodeType.Identifier); Consume(child, TokenType.Colon, NodeType.Colon); Part(child); while (IsTokenType(TokenType.OpenSquare, TokenType.Identifier)) {{ Part(child); }} return child; }} public Node<NodeType> Part(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Part); if (IsTokenType(TokenType.Identifier)) {{ OptionalElements(child); }} else if (IsTokenType(TokenType.OpenSquare)) {{ Optional(child); }} return child; }} public Node<NodeType> Optional(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Optional); Consume(child, TokenType.OpenSquare, NodeType.OpenSquare); Identifiers(child); Consume(child, TokenType.CloseSquare, NodeType.CloseSquare); if (IsTokenType(TokenType.Plus, TokenType.Star)) {{ if (IsTokenType(TokenType.Plus)) {{ Consume(child, TokenType.Plus, NodeType.Plus); }} else if (IsTokenType(TokenType.Star)) {{ Consume(child, TokenType.Star, NodeType.Star); }} }} return child; }} public Node<NodeType> Identifiers(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Identifiers); if (AreTokenTypes(TokenType.Identifier, TokenType.Identifier)) {{ RequiredIdents(child); }} else if (AreTokenTypes(TokenType.Identifier)) {{ OptionalIdents(child); }} return child; }} public Node<NodeType> RequiredIdents(Node<NodeType> parent) {{ var child = Add(parent, NodeType.RequiredIdents); Consume(child, TokenType.Identifier, NodeType.Identifier); do {{ Consume(child, TokenType.Identifier, NodeType.Identifier); }} while (AreTokenTypes(TokenType.Identifier)); return child; }} public Node<NodeType> OptionalIdents(Node<NodeType> parent) {{ var child = Add(parent, NodeType.OptionalIdents); Consume(child, TokenType.Identifier, NodeType.Identifier); while (AreTokenTypes(TokenType.Pipe)) {{ Consume(child, TokenType.Pipe, NodeType.Pipe); Consume(child, TokenType.Identifier, NodeType.Identifier); }} return child; }} public Node<NodeType> RequiredElements(Node<NodeType> parent) {{ var child = Add(parent, NodeType.RequiredElements); Element(child); do {{ Element(child); }} while (AreTokenTypes(TokenType.Identifier)); return child; }} public Node<NodeType> OptionalElements(Node<NodeType> parent) {{ var child = Add(parent, NodeType.OptionalElements); Element(child); while (AreTokenTypes(TokenType.Pipe, TokenType.Identifier)) {{ Consume(child, TokenType.Pipe, NodeType.Pipe); Element(child); }} return child; }} public Node<NodeType> Element(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Element); Consume(child, TokenType.Identifier, NodeType.Identifier); if (IsTokenType(TokenType.Plus, TokenType.Star)) {{ if (IsTokenType(TokenType.Plus)) {{ Consume(child, TokenType.Plus, NodeType.Plus); }} else if (IsTokenType(TokenType.Star)) {{ Consume(child, TokenType.Star, NodeType.Star); }} }} return child; }} public Node<NodeType> Patterns(Node<NodeType> parent) {{ var child = Consume(parent, TokenType.Patterns, NodeType.Patterns); while (AreTokenTypes(TokenType.NewLine, TokenType.Identifier)) {{ Pattern(child); }} return child; }} public Node<NodeType> Pattern(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Pattern); Consume(child, TokenType.NewLine, NodeType.NewLine); Consume(child, TokenType.Identifier, NodeType.Identifier); if (IsTokenType(TokenType.Colon)) {{ Consume(child, TokenType.Colon, NodeType.Colon); while (IsTokenType(TokenType.Identifier)) {{ Consume(child, TokenType.Identifier, NodeType.Identifier); }} }} return child; }} public Node<NodeType> Ignores(Node<NodeType> parent) {{ var child = Consume(parent, TokenType.Ignore, NodeType.Ignores); while (AreTokenTypes(TokenType.NewLine, TokenType.Identifier)) {{ Ignore(child); }} return child; }} public Node<NodeType> Ignore(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Ignore); Consume(child, TokenType.NewLine, NodeType.NewLine); Consume(child, TokenType.Identifier, NodeType.Identifier); return child; }} public Node<NodeType> Discards(Node<NodeType> parent) {{ var child = Consume(parent, TokenType.Discard, NodeType.Discards); while (AreTokenTypes(TokenType.NewLine, TokenType.Identifier)) {{ Discard(child); }} return child; }} public new Node<NodeType> Discard(Node<NodeType> parent) {{ var child = Add(parent, NodeType.Discard); Consume(child, TokenType.NewLine, NodeType.NewLine); Consume(child, TokenType.Identifier, NodeType.Identifier); return child; }} }} }}"; return ret; }
public object CreateParser(Grammar grammar) { string tokenType = BuildTokenType(grammar); string lexer = BuildLexer(grammar); string nodeType = BuildNodeType(grammar); string parser = BuildParser(grammar); var assembly = Build(tokenType, lexer, nodeType, parser); return Activator.CreateInstance(assembly.GetType($"{grammar.Name}.Parser")); }
private void RefreshGrammarTree(Grammar grammar) { try { grammarTree.Nodes.Clear(); TreeNode treeNode = new TreeNode(grammar.Name); grammarTree.Nodes.Add(treeNode); AddDefNodes(grammar, treeNode); grammarTree.ExpandAll(); } catch (Exception) { } }