public string GenerateGrammar(Grammar grammar)
{
string ret = "";
var defs = GetThings(grammar.Root, ThingType.Def);
var tokens = GetThings(grammar.Root, ThingType.Token);
var texts = GetThings(grammar.Root, ThingType.Text);
foreach (Thing def in defs)
{
ret += Environment.NewLine + def.Name + " :";
ret = DefChidren(def, "", ref ret);
}
ret += Environment.NewLine;
foreach (Thing def in tokens)
{
ret += Environment.NewLine + def.Name + " : " + def.Text;
}
ret += Environment.NewLine;
foreach (Thing def in texts)
{
ret += Environment.NewLine + def.Name + " : " + def.Text;
}
return ret;
}
public void ObjectRef()
{
var dot = new Token("dot", ".");
var text = new Text("Text", ".+");
var root = new Def("ObjectRef", text, new Optional(dot, text), new Optional(dot, text));
Grammar grammar = new Grammar
{
StringQuote = '\'',
Root = root
};
string actual = GenerateAndBuildParser(grammar, "abc");
Assert.That(actual, Is.EqualTo(@"
ObjectRef
Text - abc"));
actual = GenerateAndBuildParser(grammar, "abc.def");
Assert.That(actual, Is.EqualTo(@"
ObjectRef
Text - abc
Dot
Text - def"));
actual = GenerateAndBuildParser(grammar, "abc.def.ghi");
Assert.That(actual, Is.EqualTo(@"
ObjectRef
Text - abc
Dot
Text - def
Dot
Text - ghi"));
}
private string GenerateAndBuildParser(Grammar grammar, string text)
{
Generator generator = new Generator();
Builder builder = new Builder();
string lexerDef = generator.GenerateLexer(grammar);
string parserDef = generator.GenerateParser(grammar);
var assembly = builder.Build(lexerDef, parserDef);
object parser = Activator.CreateInstance(assembly.GetType("Xxx.Parser"));
object walker = Activator.CreateInstance(assembly.GetType("Xxx.Walker"));
var node = parser.GetType().GetMethod("Parse").Invoke(parser, new object[] { text });
return (string)walker.GetType().GetMethod("NodesToString").Invoke(walker, new[] { node });
}
public void Table()
{
var _as = new Token("as");
var _space = new Token("WhiteSpace", " ");
var text = new Text("Text", ".+");
var root = new Def("Table", text, new Optional(_as), new Optional(text));
Grammar grammar = new Grammar
{
StringQuote = '\'',
Root = root,
IgnoreTokens = new[] { _space }
};
string actual = GenerateAndBuildParser(grammar, "abc");
Assert.That(actual, Is.EqualTo(@"
Table
Text - abc"));
actual = GenerateAndBuildParser(grammar, "abc ABC");
Assert.That(actual, Is.EqualTo(@"
Table
Text - abc
Text - ABC"));
actual = GenerateAndBuildParser(grammar, "abc as ABC");
Assert.That(actual, Is.EqualTo(@"
Table
Text - abc
As
Text - ABC"));
}
public void StarOrObjectRef()
{
var dot = new Token("dot", ".");
var text = new Text("Text", ".+");
var star = new Token("star", "*");
var _space = new Token("whitespace", " ");
var objectRef = new Def("ObjectRef", text, new Optional(dot, text), new Optional(dot, text));
var root = new Def("StarOrObjectRef", new OneOf(star, objectRef));
Grammar grammar = new Grammar
{
StringQuote = '\'',
Root = root,
IgnoreTokens = new[] { _space }
};
string actual = GenerateAndBuildParser(grammar, "*");
Assert.That(actual, Is.EqualTo(@""));
}
public Grammar BuildGrammar(Node<GrammarGrammar.NodeType> parent)
{
Grammar grammar = new Grammar();
var nameNode = parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Text);
if (nameNode != null)
{
grammar.Name = nameNode.Text;
}
List<Token> punctuation = GetTokens(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Punctuation));
List<Token> keywords = GetTokens(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Keywords));
List<Token> texts = GetTokens(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Texts));
grammar.Punctuation = punctuation.Select(x => new Token(x.Name, x.Text)).ToList();
grammar.Keywords = keywords.Select(x => new Token(x.Name, x.Text)).ToList();
grammar.Texts = texts.Select(x => new Token(x.Name, x.Text)).ToList();
List<string> ignoreNames = GetNames(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Ignore));
List<string> discardNames = GetNames(parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Discard));
List<Def> defs = new List<Def>();
List<Node<GrammarGrammar.NodeType>> reversed = parent.Children.FirstOrDefault(x => x.NodeType == GrammarGrammar.NodeType.Defs).Children;
reversed.Reverse();
foreach (var def in reversed)
{
var textNode = def.Children.First(x => x.NodeType == GrammarGrammar.NodeType.Text);
var deff = new Def(textNode.Text);
defs.Add(deff);
foreach (var part in def.Children.Where(x => x.NodeType == GrammarGrammar.NodeType.Part))
{
Thing thing = null;
Thing thing2 = null;
var names = part.Children.Single(x => x.NodeType == GrammarGrammar.NodeType.Names);
if (names.Children.Last().NodeType == GrammarGrammar.NodeType.Plus
|| part.Children.Last().NodeType == GrammarGrammar.NodeType.Plus)
{
thing = thing2 = new OneOrMore();
}
else if (names.Children.Last().NodeType == GrammarGrammar.NodeType.Star
|| part.Children.Last().NodeType == GrammarGrammar.NodeType.Star)
{
thing = thing2 = new ZeroOrMore();
}
if (part.Children.First().NodeType == GrammarGrammar.NodeType.OpenSquare)
{
if (thing == null)
{
thing = thing2 = new Optional();
}
else
{
thing2 = new Optional();
thing.Children = new List<Thing>() { thing2 };
}
}
if (names.Children.Any(x => x.NodeType == GrammarGrammar.NodeType.Pipe))
{
if (thing == null)
{
thing = thing2 = new OneOf();
}
else
{
thing2 = new OneOf();
thing.Children = new List<Thing>() { thing2 };
}
}
List<Thing> things = new List<Thing>();
foreach (var name in names.Children.Where(x => x.NodeType == GrammarGrammar.NodeType.Text))
{
Thing childThing = null;
var token = grammar.Punctuation.SingleOrDefault(x => x.Name == name.Text);
if (token != null)
{
things.Add(token);
}
else
{
token = grammar.Keywords.SingleOrDefault(x => x.Name == name.Text);
if (token != null)
{
things.Add(token);
}
else
{
token = grammar.Texts.SingleOrDefault(x => x.Name == name.Text);
if (token != null)
{
things.Add(token);
}
else
{
var def2 = defs.SingleOrDefault(x => x.Name == name.Text);
if (def2 != null)
{
things.Add(def2);
}
else
{
throw new Exception();
}
}
}
}
}
if (thing == null)
{
deff.Children.AddRange(things);
}
else
{
thing2.Children = things;
deff.Children.Add(thing);
}
}
}
grammar.Root = defs.Last();
List<Token> ignoreTokens = new List<Token>();
List<string> discardThings = new List<string>();
ignoreTokens.AddRange(keywords.Where(x => ignoreNames.Contains(x.Name)));
ignoreTokens.AddRange(punctuation.Where(x => ignoreNames.Contains(x.Name)));
discardThings.AddRange(keywords.Where(x => discardNames.Contains(x.Name)).Select(x => x.Name));
discardThings.AddRange(punctuation.Where(x => discardNames.Contains(x.Name)).Select(x => x.Name));
discardThings.AddRange(defs.Where(x => discardNames.Contains(x.Name)).Select(x => x.Name));
grammar.IgnoreTokens = ignoreTokens.ToArray();
grammar.DiscardThings = discardThings.ToArray();
grammar.StringQuote = '\'';
return grammar;
}
public string GenerateParser(Grammar grammar)
{
StringBuilder stringBuilder = new StringBuilder();
var defs = GetThings(grammar.Root, ThingType.Def);
var tokens = GetThings(grammar.Root, ThingType.Token);
var texts = GetThings(grammar.Root, ThingType.Text);
//tokens.AddRange(grammar.DiscardThings.Where(x => !tokens.Contains(x)));
stringBuilder.Append(
$@"using System;
using System.Collections.Generic;
using Parsing.Core;
namespace Xxx
{{
public class {grammar.Name}Parser : ParserBase<TokenType, NodeType>
{{
private readonly List<string> _discardThings;
public {grammar.Name}Parser() : base(new {grammar.Name}Lexer())
{{
_discardThings = new List<string>
{{");
foreach (var text in grammar.DiscardThings)
{
stringBuilder.Append($@"
""{text.ToIdentifier()}"",");
}
stringBuilder.Append($@"
}};
}}
public override List<string> DiscardThings {{ get {{ return _discardThings; }} }}
public override Node<NodeType> Root()
{{
return {grammar.Root.Name}(null);
}}");
foreach (Thing def in defs)
{
string parentOrChild = grammar.DiscardThings.All(x => x != def.Name)
? "child"
: "parent";
stringBuilder.Append($@"
public Node<NodeType> {def.Name}(Node<NodeType> parent)
{{");
if (grammar.DiscardThings.All(x => x != def.Name))
{
stringBuilder.Append($@"
var {parentOrChild} = Add(parent, NodeType.{def.Name});");
}
GenerateParserDef(def, parentOrChild, stringBuilder, 0);
stringBuilder.Append($@"
return {parentOrChild};
}}");
}
stringBuilder.Append(@"
}
public enum NodeType
{");
foreach (var def in defs.Where(x => !IsNullOrWhiteSpace(x.Name)).Select(x => x.Name).Distinct())
{
stringBuilder.Append(@"
" + def + ",");
}
foreach (var text in texts.Where(x => !IsNullOrWhiteSpace(x.Name)).Select(x => x.Name.ToIdentifier()).Distinct())
{
stringBuilder.Append(@"
" + text + ",");
}
foreach (var token in tokens.Where(x => !IsNullOrWhiteSpace(x.Name)).Select(x => x.Name.ToIdentifier()).Distinct())
{
stringBuilder.Append(@"
" + token + ",");
}
stringBuilder.Append(@"
}
public class Walker
{
public string NodesToString(object node)
{
Node<NodeType> parent = node as Node<NodeType>;
string ret = """";
NodesToString(parent, ref ret, 0);
return ret;
}
private void NodesToString(Node<NodeType> parent, ref string ret, int indent)
{
ret += Environment.NewLine + new string(' ', indent) + parent.NodeType +(String.IsNullOrWhiteSpace(parent.Text) ? """" : "" - "" + parent.Text);
foreach (Node<NodeType> child in parent.Children)
{
NodesToString(child, ref ret, indent + 1);
}
}
}
}");
return stringBuilder.ToString();
}
public string GenerateLexer(Grammar grammar)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.Append($@"using System.Collections.Generic;
using Parsing.Core;
namespace Xxx
{{
public class {grammar.Name}Lexer : LexerBase<TokenType>
{{
private readonly Dictionary<char, TokenType> _punctuation;
private readonly Dictionary<string, TokenType> _keywords;
private readonly Dictionary<string, TokenType> _texts;
private readonly List<TokenType> _ignoreTokenTypes;
private readonly char _stringQuote;
public {grammar.Name}Lexer()
{{
_punctuation = new Dictionary<char, TokenType>
{{");
foreach (Token token in grammar.Punctuation)
{
stringBuilder.Append($@"
{{ '{token.Text}', TokenType.{token.Name.ToIdentifier()} }},");
}
stringBuilder.Append($@"
}};
_keywords = new Dictionary<string, TokenType>
{{");
foreach (Token token in grammar.Keywords)
{
stringBuilder.Append($@"
{{ ""{token.Text}"", TokenType.{token.Name.ToIdentifier()} }},");
}
stringBuilder.Append($@"
}};
_texts = new Dictionary<string, TokenType>
{{");
foreach (Token text in grammar.Texts)
{
stringBuilder.Append($@"
{{ ""^{text.Text}$"", TokenType.{text.Name} }},");
}
stringBuilder.Append($@"
}};
_ignoreTokenTypes = new List<TokenType>
{{");
foreach (Token text in grammar.IgnoreTokens)
{
stringBuilder.Append($@"
TokenType.{text.Name.ToIdentifier()},");
}
stringBuilder.Append($@"
}};");
stringBuilder.Append($@"
_stringQuote = '{(grammar.StringQuote == '\'' ? "\\" : "") + grammar.StringQuote}';
}}");
stringBuilder.Append(@"
public override TokenType EndOfFileTokenType { get { return TokenType.EndOfFile; } }
public override TokenType StringTokenType { get { return TokenType.String; } }
public override Dictionary<char, TokenType> Punctuation { get { return _punctuation; } }
public override Dictionary<string, TokenType> KeyWords { get { return _keywords; } }
public override Dictionary<string, TokenType> Texts { get { return _texts; } }
public override List<TokenType> IgnoreTokenTypes { get { return _ignoreTokenTypes; } }
public override char StringQuote { get { return _stringQuote; } }
}
");
stringBuilder.Append(@"
public enum TokenType
{
EndOfFile,
String,");
foreach (var text in grammar.Punctuation)
{
stringBuilder.Append(@"
" + text.Name.ToIdentifier() + ",");
}
foreach (var token in grammar.Keywords)
{
stringBuilder.Append(@"
" + token.Name.ToIdentifier() + ",");
}
foreach (var token in grammar.Texts)
{
stringBuilder.Append(@"
" + token.Name.ToIdentifier() + ",");
}
stringBuilder.Append(@"
}
}");
return stringBuilder.ToString();
}
private void RefreshGrammar(Grammar grammar)
{
genGrammar.Text = grammar.Name;
var defs = GetThings(grammar.Root, ThingType.Def);
foreach (var def in defs)
{
RefreshGrammar(def);
}
}