/// <summary>
/// Initializes a new instance of the <see cref="DataModelTypeBuilder"/> class.
/// </summary>
/// <param name="decl">The declaration declaring this type.</param>
protected DataModelTypeBuilder(GrammarSymbol decl)
{
this.DeclSymbol = decl;
this.G4DeclaredName = decl.GetLogicalText();
this.SummaryText = decl.Annotations.GetAnnotationValue("summary");
this.RemarksText = decl.Annotations.GetAnnotationValue("remarks");
this.CSharpName
= decl.Annotations.GetAnnotationValue("className")
?? LinguisticTransformer.ToCSharpName(this.G4DeclaredName);
this.InterfaceName = decl.Annotations.GetAnnotationValue("interface");
this.Pattern = decl.Annotations.GetAnnotationValue("pattern");
this.RootObject = decl.Annotations.GetAnnotationValue("rootObject") != null;
this.G4DeclaredValues = new List <string>();
string serializedValuesText = decl.Annotations.GetAnnotationValue("serializedValues");
if (!string.IsNullOrEmpty(serializedValuesText))
{
this.SerializedValues = new List <string>(serializedValuesText.Split(','));
}
else
{
this.SerializedValues = new List <string>();
}
}
private static GrammarSymbol GetDeclaringIdentifierSymbol(GrammarSymbol currentSymbol, string productionName)
{
GrammarSymbol declSymbol;
switch (currentSymbol.Kind)
{
case SymbolKind.ZeroOrOneQuantifier:
case SymbolKind.ZeroOrMoreQuantifier:
case SymbolKind.OneOrMoreQuantifier:
declSymbol = currentSymbol[0];
break;
case SymbolKind.Identifier:
declSymbol = currentSymbol;
break;
default:
declSymbol = GrammarSymbol.Empty;
break;
}
if (declSymbol.Kind != SymbolKind.Identifier)
{
throw new G4ParseFailureException(currentSymbol.GetLocation(), Strings.UnrecognizedDataModel, productionName);
}
return(declSymbol);
}
/// <summary>Parses a G4 quantifier.</summary>
/// <exception cref="G4ParseFailureException">Thrown when the input is not valid.</exception>
/// <returns>A GrammarSymbol containing the quantifier.</returns>
public GrammarSymbol ParseQuantifier()
{
GrammarSymbol nonTerminal = this.ParseNonTerminal();
Token current = _iterator.Current;
SymbolKind quantifierKind;
switch (current.Kind)
{
case TokenKind.Star:
quantifierKind = SymbolKind.ZeroOrMoreQuantifier;
break;
case TokenKind.Plus:
quantifierKind = SymbolKind.OneOrMoreQuantifier;
break;
case TokenKind.Question:
quantifierKind = SymbolKind.ZeroOrOneQuantifier;
break;
default:
quantifierKind = SymbolKind.Default;
break;
}
if (quantifierKind != SymbolKind.Default)
{
_iterator.Move();
return(new GrammarSymbol(nonTerminal.FirstToken, current, quantifierKind, ImmutableArray.Create(nonTerminal)));
}
else
{
return(nonTerminal);
}
}
/// <summary>Parses a G4 group.</summary>
/// <exception cref="G4ParseFailureException">Thrown when the input is not valid.</exception>
/// <returns>A GrammarSymbol containing the group.</returns>
public GrammarSymbol ParseGroup()
{
ImmutableArray <GrammarSymbol> .Builder quantifiers = ImmutableArray.CreateBuilder <GrammarSymbol>();
for (;;)
{
Token current = _iterator.Current;
switch (current.Kind)
{
// Quantifier must start with a nonTerminal
case TokenKind.Identifier:
case TokenKind.String:
// CharacterRange
case TokenKind.Lparen:
quantifiers.Add(this.ParseQuantifier());
break;
default:
GrammarSymbol result = quantifiers.CreateEnclosingSymbol(SymbolKind.Group);
if (result == null)
{
throw new G4ParseFailureException(current.GetLocation(), Strings.EmptyGroup);
}
return(result);
}
}
}
public void CompileProduction(GrammarSymbol production)
{
Debug.Assert(production.Kind == SymbolKind.Production);
GrammarSymbol decl = production[0];
Debug.Assert(decl.Kind == SymbolKind.ProductionDecl);
if (decl.FirstToken != production.FirstToken)
{
// Honestly I don't know why "fragment" productions are even
// in the grammar if they're not supposed to be produced; but this is what
// the original MSR code did and we need to parse their grammar files.
Debug.Assert(production.FirstToken.GetText() == "fragment");
return;
}
string productionName = decl.GetLogicalText();
if (s_basicBuiltinTypes.Any(type => type.G4DeclaredName == productionName))
{
// Builtins don't matter in the grammar.
return;
}
GrammarSymbol productionIs = production[1];
switch (productionIs.Kind)
{
case SymbolKind.String:
this.CompileEnumValueType(decl, productionIs.FirstToken);
break;
case SymbolKind.Identifier:
case SymbolKind.ZeroOrMoreQuantifier:
case SymbolKind.OneOrMoreQuantifier:
case SymbolKind.ZeroOrOneQuantifier:
this.CompileStandardType(decl, ImmutableArray.Create(productionIs));
break;
case SymbolKind.Group:
this.CompileStandardType(decl, productionIs.Children);
break;
case SymbolKind.Alternation:
if (productionIs.Children.All(child => child.Kind == SymbolKind.String))
{
this.CompileEnumValueType(decl, productionIs.Children);
break;
}
else if (productionIs.Children.All(child => child.Kind == SymbolKind.Identifier))
{
this.CompileBaseType(decl, productionIs.Children);
break;
}
goto default;
default:
throw new G4ParseFailureException(production.GetLocation(), Strings.UnrecognizedDataModel, productionName);
}
}
private void CompileStringValueType(GrammarSymbol decl)
{
var result = new DataModelLeafTypeBuilder(decl);
var member = new DataModelMember("STRING", "StringValue", "stringValue", null, "stringValue", null, null, null, null, null, 0, false);
result.Members.Add(member);
result.ToStringEntries.Add(new ToStringEntry(member));
this.AddCompiledType(result);
}
private void CompileEnumValueType(GrammarSymbol decl, Token token)
{
var result = new DataModelLeafTypeBuilder(decl);
var declaredValues = new List <string>();
declaredValues.Add(token.GetText());
result.G4DeclaredValues = declaredValues;
this.AddCompiledType(result);
}
private static bool MatchesDelimitedCollectionPattern(GrammarSymbol currentSymbol, GrammarSymbol memberDeclSymbol, GrammarSymbol lookAheadSymbol)
{
// x ('delimiter' x)*
// or
// x? ('delimiter' x)*
return((currentSymbol.Kind == SymbolKind.ZeroOrOneQuantifier || currentSymbol.Kind == SymbolKind.Identifier) &&
lookAheadSymbol.Kind == SymbolKind.ZeroOrMoreQuantifier &&
lookAheadSymbol[0].Kind == SymbolKind.Group &&
lookAheadSymbol[0][0].Kind == SymbolKind.String &&
lookAheadSymbol[0][1].Kind == SymbolKind.Identifier &&
memberDeclSymbol.FirstToken.TextEqual(lookAheadSymbol[0][1].FirstToken));
}
private void CompileEnumValueType(GrammarSymbol decl, ImmutableArray <GrammarSymbol> enumMembers)
{
var result = new DataModelLeafTypeBuilder(decl);
var declaredValues = new List <string>();
foreach (GrammarSymbol symbol in enumMembers)
{
declaredValues.Add(symbol.FirstToken.GetText());
}
result.G4DeclaredValues = declaredValues;
this.AddCompiledType(result);
}
/// <summary>
/// Initializes a <see cref="DataModelMetadata"/> instance from the given grammar.
/// </summary>
/// <exception cref="ArgumentException">
/// Thrown when the supplied <see cref="GrammarSymbol"/> is not a valid grammar.
/// </exception>
/// <param name="grammar">
/// The grammar from which the <see cref="DataModelMetadata"/> shall be generated.
/// </param>
/// <returns>
/// A <see cref="DataModelMetadata"/> containing data from the grammar <paramref name="grammar"/>.
/// </returns>
public static DataModelMetadata FromGrammar(GrammarSymbol grammar)
{
if (grammar.Kind != SymbolKind.Grammar)
{
throw new ArgumentException("FromGrammar requires a grammar.");
}
GrammarSymbol decl = grammar.Children[0];
GrammarSymbol grammarId = decl.Children[0];
string grammarName = grammarId.GetLogicalText();
System.Collections.Immutable.ImmutableArray <Annotation> annotations = grammarId.Annotations;
string nameSpace = annotations.GetAnnotationValue("namespace") ?? grammarName;
return(new DataModelMetadata(grammarName, nameSpace, annotations.HasAnnotation("generateLocations"), !annotations.HasAnnotation("noGenerateEquals")));
}
private static List <string> GetStringValuesAfter(ImmutableArray <GrammarSymbol> groupMemberList, int idx)
{
var result = new List <string>();
for (; idx < groupMemberList.Length; ++idx)
{
GrammarSymbol current = groupMemberList[idx];
if (current.Kind != SymbolKind.String)
{
break;
}
result.Add(current.GetLogicalText());
}
return(result);
}
/// <summary>Parses a G4 grammar production.</summary>
/// <exception cref="G4ParseFailureException">Thrown when the input is not valid.</exception>
/// <returns>A GrammarSymbol containing the production.</returns>
public GrammarSymbol ParseProduction()
{
Token firstToken = _iterator.Current; // May be != identifierToken if 'fragment'
Token identifierToken = _iterator.Require(TokenKind.Identifier);
if (identifierToken.GetText().Equals("fragment"))
{
identifierToken = _iterator.Require(TokenKind.Identifier);
}
ImmutableArray <Annotation> productionAnnotations = _iterator.ConsumeAnnotations();
var declaration = new GrammarSymbol(identifierToken, SymbolKind.ProductionDecl, productionAnnotations);
_iterator.Require(TokenKind.Colon);
GrammarSymbol alternation = this.ParseAlternation();
_iterator.DiscardAnnotations();
Token lastToken = _iterator.Require(TokenKind.Semicolon);
return(new GrammarSymbol(firstToken, lastToken, SymbolKind.Production, ImmutableArray.Create(declaration, alternation)));
}
/// <summary>Parses a G4 non-terminal.</summary>
/// <exception cref="G4ParseFailureException">Thrown when the input is not valid.</exception>
/// <returns>A GrammarSymbol containing the non-terminal.</returns>
public GrammarSymbol ParseNonTerminal()
{
ImmutableArray <Annotation> annotations = _iterator.ConsumeAnnotations();
Token current = _iterator.Consume();
if (current.Kind == TokenKind.Identifier)
{
return(new GrammarSymbol(current, SymbolKind.Identifier, annotations));
}
if (current.Kind == TokenKind.Lparen)
{
GrammarSymbol result = this.ParseAlternation();
_iterator.Require(TokenKind.Rparen);
return(result);
}
if (current.Kind != TokenKind.String)
{
throw G4ParseFailureException.UnexpectedToken(current);
}
if (_iterator.Current.Kind == TokenKind.Dots)
{
CheckForBadCharacterRangeCharacterLength(current);
_iterator.Move(); // dots
Token secondString = _iterator.Require(TokenKind.String);
CheckForBadCharacterRangeCharacterLength(secondString);
CheckForCorrectlyOrderedCharacterRange(current, secondString);
return(new GrammarSymbol(current, secondString, SymbolKind.CharacterRange));
}
else
{
return(new GrammarSymbol(current, SymbolKind.String));
}
}
public DataModelLeafTypeBuilder(GrammarSymbol decl)
: base(decl)
{
this.Members = ImmutableArray.CreateBuilder <DataModelMember>();
this.ToStringEntries = new List <ToStringEntry>();
}
//private static IDisposable sw;
/// <summary>Main entry-point for this application.</summary>
/// <param name="args">Array of command-line argument strings.</param>
/// <returns>Exit-code for the process - 0 for success, else an error code.</returns>
public static int Main(string[] args)
{
using (var config = new DataModelGeneratorConfiguration())
{
if (!config.ParseArgs(args))
{
return(1);
}
string inputText;
using (var sr = new StreamReader(config.InputStream))
{
inputText = sr.ReadToEnd();
}
GrammarSymbol grammar;
try
{
var factory = new TokenTextIndex(inputText);
System.Collections.Immutable.ImmutableArray <Token> tokens = Lexer.Lex(factory);
var parser = new Parser(tokens);
grammar = parser.ParseGrammar();
}
catch (G4ParseFailureException ex)
{
Console.Error.WriteLine(ex.Message);
return(1);
}
var builder = new DataModelBuilder();
for (int idx = 1; idx < grammar.Children.Length; ++idx)
{
try
{
GrammarSymbol prod = grammar.Children[idx];
builder.CompileProduction(prod);
}
catch (G4ParseFailureException ex)
{
Console.WriteLine(Strings.FailedToCompileProduction + ex.Message);
}
}
string generatedCSharp;
string generatedJsonSchema;
try
{
DataModel model = builder.Link(config.InputFilePath, DataModelMetadata.FromGrammar(grammar));
model = MemberHoister.HoistTypes(model);
var cr = new CodeWriter();
CodeGenerator.WriteDataModel(cr, model);
generatedCSharp = cr.ToString();
cr = new CodeWriter(2, ' ');
JsonSchemaGenerator.WriteSchema(cr, model);
generatedJsonSchema = cr.ToString();
}
catch (G4ParseFailureException ex)
{
Console.Error.WriteLine(ex.Message);
return(1);
}
if (config.ToConsole)
{
Console.WriteLine(generatedCSharp);
}
using (var sw = new StreamWriter(config.OutputStream))
{
sw.Write(generatedCSharp);
}
string jsonSchemaPath = Path.GetFileNameWithoutExtension(config.OutputFilePath);
jsonSchemaPath = Path.Combine(Path.GetDirectoryName(config.OutputFilePath), jsonSchemaPath + ".schema.json");
File.WriteAllText(jsonSchemaPath, generatedJsonSchema);
return(0);
}
}
private void CompileBaseType(GrammarSymbol decl, ImmutableArray <GrammarSymbol> children)
{
_compiledBases.Add(decl.GetLogicalText(),
new DataModelBaseTypeBuilder(decl, children.Select(child => child.GetLogicalText())));
}
private void CompileStandardType(GrammarSymbol decl, ImmutableArray <GrammarSymbol> groupList)
{
var result = new DataModelLeafTypeBuilder(decl);
int idx = 0;
while (idx < groupList.Length)
{
List <string> staticStrings = GetStringValuesAfter(groupList, idx);
foreach (string staticString in staticStrings)
{
result.ToStringEntries.Add(new ToStringEntry(staticString, null));
}
idx += staticStrings.Count;
if (idx >= groupList.Length)
{
break;
}
GrammarSymbol currentSymbol = groupList[idx];
GrammarSymbol memberDeclSymbol = GetDeclaringIdentifierSymbol(currentSymbol, result.G4DeclaredName);
GrammarSymbol lookAheadSymbol = GetSymbolOrEmpty(groupList, idx + 1);
// Check for patterns:
if (MatchesDelimitedCollectionPattern(currentSymbol, memberDeclSymbol, lookAheadSymbol))
{
// x ('delimiter' x)*
string delimeter = lookAheadSymbol[0][0].GetLogicalText();
result.AddMember(memberDeclSymbol, 1, currentSymbol.Kind == SymbolKind.Identifier, delimeter);
idx += 2;
continue;
}
int rank;
bool required;
switch (currentSymbol.Kind)
{
case SymbolKind.Identifier:
// x
rank = 0;
required = true;
break;
case SymbolKind.ZeroOrOneQuantifier:
// x?
rank = 0;
required = false;
break;
case SymbolKind.ZeroOrMoreQuantifier:
// x*
rank = 1;
required = false;
break;
case SymbolKind.OneOrMoreQuantifier:
// x+
rank = 1;
required = true;
break;
default:
throw new G4ParseFailureException(currentSymbol.GetLocation(), Strings.UnrecognizedDataModel, result.G4DeclaredName);
}
++idx;
result.AddMember(memberDeclSymbol, rank, required);
}
this.AddCompiledType(result);
}
public DataModelBaseTypeBuilder(GrammarSymbol decl, IEnumerable <string> derivedDecls)
: base(decl)
{
this.DerivedDecls = derivedDecls.ToImmutableArray();
}
internal void AddMember(GrammarSymbol declSymbol, int rank, bool required, string delimeter = null)
{
string declaredName = declSymbol.GetLogicalText();
string annotationName = declSymbol.Annotations.GetAnnotationValue("name");
string pattern = declSymbol.Annotations.GetAnnotationValue("pattern");
string minimum = declSymbol.Annotations.GetAnnotationValue("minimum");
string minItems = declSymbol.Annotations.GetAnnotationValue("minItems");
string uniqueItems = declSymbol.Annotations.GetAnnotationValue("uniqueItems");
string defaultValue = declSymbol.Annotations.GetAnnotationValue("default");
string cSharpName = annotationName ?? LinguisticTransformer.ToCSharpName(declaredName);
string serializedName = declSymbol.Annotations.GetAnnotationValue("serializedName");
if (serializedName == null)
{
serializedName = LinguisticTransformer.ToJsonName(cSharpName);
}
if (serializedName == null)
{
serializedName = LinguisticTransformer.ToJsonName(declaredName);
}
string argumentName = declSymbol.Annotations.GetAnnotationValue("argumentName");
if (argumentName == null)
{
if (annotationName == null)
{
argumentName = LinguisticTransformer.ToArgumentName(declaredName);
}
else
{
argumentName = LinguisticTransformer.ToArgumentName(annotationName);
}
}
foreach (DataModelMember existingMember in this.Members)
{
if (existingMember.CSharpName == cSharpName)
{
throw new G4ParseFailureException(declSymbol.GetLocation(), Strings.DuplicateCSharpMemberName, cSharpName);
}
if (existingMember.SerializedName == serializedName)
{
throw new G4ParseFailureException(declSymbol.GetLocation(), Strings.DuplicateJsonMemberName, serializedName);
}
if (existingMember.ArgumentName == argumentName)
{
throw new G4ParseFailureException(declSymbol.GetLocation(), Strings.DuplicateArgumentName, argumentName);
}
}
DataModelMember newMember = new DataModelMember(
declaredName,
cSharpName,
serializedName,
declSymbol.Annotations.GetAnnotationValue("summary"),
argumentName,
pattern,
minimum,
minItems,
uniqueItems,
defaultValue,
rank,
required
);
this.Members.Add(newMember);
this.ToStringEntries.Add(new ToStringEntry(delimeter, newMember));
}
