private static string GetTypeHint(TypeDecl decl, ElementDecl element)
{
string GetParamNamespace(TypeDecl declaration, IDeclKey key) =>
!PyExtensions.IsPackageEquals(declaration, key) ? PyExtensions.GetAlias(key) + "." : "";
string GetFinalHint(string typeHint) =>
element.Vector == YesNo.Y ? $"List[{typeHint}]" : typeHint;
if (element.Value != null)
{
string hint = GetValue(element.Value);
return(GetFinalHint(hint));
}
else if (element.Data != null)
{
string paramNamespace = GetParamNamespace(decl, element.Data);
string hint = $"{paramNamespace}{element.Data.Name}";
return(GetFinalHint(hint));
}
else if (element.Key != null)
{
return(GetFinalHint("str"));
}
else if (element.Enum != null)
{
string paramNamespace = GetParamNamespace(decl, element.Enum);
string hint = $"{paramNamespace}{element.Enum.Name}";
return(GetFinalHint(hint));
}
else
{
throw new ArgumentException("Can't deduct type");
}
}
public static void WriteElements(TypeDecl decl, CodeWriter writer)
{
foreach (var declare in decl.Declare.Handlers)
{
var @return = "void";
var @params = string.Join(", ", declare.Params.Select(param => $"{GetType(param)} {param.Name.Underscore()}"));
var function = $"{@return} {declare.Name.Underscore()}({@params})";
var comment = CommentHelper.FormatComment(declare.Comment);
writer.AppendLines(comment);
var implement = decl.Implement?.Handlers.FirstOrDefault(i => i.Name == declare.Name);
// Abstract
if (implement == null)
{
writer.AppendLine($"virtual {function} = 0;");
}
// Override
else if (implement.Override == YesNo.Y)
{
writer.AppendLine($"virtual {function} override;");
}
// No modifiers.
else
{
writer.AppendLine($"{function};");
}
writer.AppendNewLineWithoutIndent();
}
}
private static string GetTypeHint(TypeDecl decl, ParamDecl parameter)
{
if (parameter.Value != null)
{
string result = GetValue(parameter.Value);
return(parameter.Vector == YesNo.Y ? $"List[{result}]" : result);
}
else if (parameter.Data != null)
{
string paramNamespace = !PyExtensions.IsPackageEquals(decl, parameter.Data)
? PyExtensions.GetAlias(parameter.Data) + "."
: "";
string result = $"{paramNamespace}{parameter.Data.Name}";
return(parameter.Vector == YesNo.Y ? $"List[{result}]" : result);
}
else if (parameter.Key != null)
{
return(parameter.Vector == YesNo.Y ? "List[str]" : "str");
}
else if (parameter.Enum != null)
{
string paramNamespace = !PyExtensions.IsPackageEquals(decl, parameter.Enum)
? PyExtensions.GetAlias(parameter.Enum) + "."
: "";
string result = $"{paramNamespace}{parameter.Enum.Name}";
return(parameter.Vector == YesNo.Y ? $"List[{result}]" : result);
}
else
{
throw new ArgumentException("Can't deduct type");
}
}
public static IEnumerable <string> ForKeyHeader(TypeDecl decl, Dictionary <string, string> declSet)
{
var settings = GeneratorSettingsProvider.Get(decl.Module.ModuleName);
var includes = new List <string>
{
$"#include <{settings.Namespace}/declare.hpp>",
"#include <dot/system/ptr.hpp>",
"#include <dc/types/record/record.hpp>",
$"#include <{declSet[decl.Name]}/{decl.Name.Underscore()}_data.hpp>"
};
// Include from field types
includes.AddRange(decl.Elements.Where(t => t.Data != null)
.Select(t => t.Data.Name).Distinct()
.Select(t => $"#include <{declSet[t]}/{t.Underscore()}_data.hpp>"));
includes.AddRange(decl.Elements.Where(t => t.Key != null)
.Select(t => t.Key.Name).Distinct()
.Select(t => $"#include <{declSet[t]}/{t.Underscore()}_key.hpp>"));
var knownModules = GeneratorSettingsProvider.KnownModules();
includes.AddRange(decl.Elements.Where(t => t.Enum != null)
// Skip external enum
.Where(t => knownModules.Contains(t.Enum.Module.ModuleName))
.Select(t => t.Enum.Name).Distinct()
.Select(t => $"#include <{declSet[t]}/{t.Underscore()}.hpp>"));
return(includes);
}
public static string BuildDataSource(TypeDecl decl, Dictionary <string, string> declSet)
{
var writer = new CodeWriter();
var settings = GeneratorSettingsProvider.Get(decl.Module.ModuleName);
writer.AppendLines(settings.Copyright);
writer.AppendNewLineWithoutIndent();
writer.AppendLine($"#include <{settings.Namespace}/precompiled.hpp>");
writer.AppendLine($"#include <{settings.Namespace}/implement.hpp>");
writer.AppendLine($"#include <{declSet[decl.Name]}/{decl.Name.Underscore()}_data.hpp>");
writer.AppendLine($"#include <dc/platform/context/context_base.hpp>");
writer.AppendNewLineWithoutIndent();
writer.AppendLine($"namespace {settings.Namespace}");
writer.AppendLine("{");
writer.PushIndent();
BuildClassImplementation(decl, writer);
writer.PopIndent();
writer.AppendLine("}");
return(writer.ToString());
}
public static string BuildDataHeader(TypeDecl decl, Dictionary <string, string> declSet)
{
var writer = new CodeWriter();
var settings = GeneratorSettingsProvider.Get(decl.Module.ModuleName);
writer.AppendLines(settings.Copyright);
writer.AppendNewLineWithoutIndent();
writer.AppendLine("#pragma once");
writer.AppendNewLineWithoutIndent();
// includes
var includes = IncludesProvider.ForDataHeader(decl, declSet);
foreach (string include in includes)
{
writer.AppendLine(include);
}
writer.AppendNewLineWithoutIndent();
writer.AppendLine($"namespace {settings.Namespace}");
writer.AppendLine("{");
writer.PushIndent();
BuildClassDeclaration(decl, writer);
writer.PopIndent();
writer.AppendLine("}");
return(writer.ToString());
}
private static void BuildClassImplementation(TypeDecl decl, CodeWriter writer)
{
var settings = GeneratorSettingsProvider.Get(decl.Module.ModuleName);
var type = decl.Name.Underscore();
writer.AppendLine($"dot::type_t {type}_key_impl::type() {{ return typeof(); }}");
writer.AppendLine($"dot::type_t {type}_key_impl::typeof()");
writer.AppendLine("{");
writer.PushIndent();
writer.AppendLine("static dot::type_t type_ =");
writer.PushIndent();
writer.AppendLine($"dot::make_type_builder<self>(\"{settings.Namespace}\", \"{type}\")");
var keyElements = decl.Elements.Where(e => decl.Keys.Contains(e.Name)).ToList();
foreach (var element in keyElements)
{
var name = element.Name.Underscore();
writer.AppendLine($"->with_field(\"{name}\", &self::{name})");
}
writer.AppendLine($"->template with_base<key<{type}_key_impl, {type}_data_impl>>()");
writer.AppendLine($"->with_constructor(&make_{type}_key, {{ }})");
writer.AppendLine("->build();");
writer.PopIndent();
writer.AppendLine("return type_;");
writer.PopIndent();
writer.AppendLine("}");
}
private static FileInfo ConvertType(TypeDecl decl, List <IDecl> declarations)
{
// Decompose package to folder and file name.
int dotIndex = decl.Category.LastIndexOf('.');
string fileName = decl.Category.Substring(dotIndex + 1) + ".py";
string folderName = decl.Category.Substring(0, dotIndex).Replace('.', '/');
var dataFile = new FileInfo
{
Content = PythonRecordBuilder.Build(decl, declarations).AppendCopyright(decl),
FileName = fileName,
FolderName = folderName
};
return(dataFile);
}
private static List <FileInfo> ConvertType(TypeDecl decl, Dictionary <string, string> includePath)
{
string pathInProject = includePath[decl.Name];
List <FileInfo> result = new List <FileInfo>();
var dataHeader = new FileInfo
{
Content = CppDataBuilder.BuildDataHeader(decl, includePath),
FileName = $"{decl.Name.Underscore()}_data.hpp",
FolderName = pathInProject
};
result.Add(dataHeader);
var dataSource = new FileInfo
{
Content = CppDataBuilder.BuildDataSource(decl, includePath),
FileName = $"{decl.Name.Underscore()}_data.cpp",
FolderName = pathInProject
};
result.Add(dataSource);
if (decl.Keys.Any())
{
var keyHeader = new FileInfo
{
Content = CppKeyBuilder.BuildKeyHeader(decl, includePath),
FileName = $"{decl.Name.Underscore()}_key.hpp",
FolderName = pathInProject
};
result.Add(keyHeader);
var keySource = new FileInfo
{
Content = CppKeyBuilder.BuildKeySource(decl, includePath),
FileName = $"{decl.Name.Underscore()}_key.cpp",
FolderName = pathInProject
};
result.Add(keySource);
}
return(result);
}
private static void WriteMethods(TypeDecl decl, CodeWriter writer)
{
bool HasImplement(HandlerDeclareItem declare)
{
return(decl.Implement?.Handlers.FirstOrDefault(t => t.Name == declare.Name) == null);
}
var declarations = decl.Declare.Handlers;
foreach (var declare in declarations)
{
bool isAbstract = !HasImplement(declare);
if (isAbstract)
{
writer.AppendLine("@abstractmethod");
}
var parameters = "";
foreach (var parameter in declare.Params)
{
parameters += ($", {parameter.Name.Underscore()}: {GetTypeHint(decl, parameter)}");
}
writer.AppendLine($"def {declare.Name.Underscore()}(self{parameters}):");
writer.PushIndent();
writer.AppendLines(CommentHelper.PyComment(declare.Comment));
writer.AppendLine(isAbstract ? "pass" : "raise NotImplemented");
if (declarations.IndexOf(declare) != declarations.Count - 1)
{
writer.AppendNewLineWithoutIndent();
}
writer.PopIndent();
}
}
private static void BuildClassImplementation(TypeDecl decl, CodeWriter writer)
{
var settings = GeneratorSettingsProvider.Get(decl.Module.ModuleName);
var type = decl.Name.Underscore();
bool isRecord = decl.Keys.Any();
bool isDerived = decl.Inherit != null;
writer.AppendLine($"dot::type_t {type}_data_impl::type() {{ return typeof(); }}");
writer.AppendLine($"dot::type_t {type}_data_impl::typeof()");
writer.AppendLine("{");
writer.PushIndent();
writer.AppendLine("static dot::type_t type_ =");
writer.PushIndent();
writer.AppendLine($"dot::make_type_builder<self>(\"{settings.Namespace}\", \"{type}\")");
foreach (var element in decl.Elements)
{
var name = element.Name.Underscore();
writer.AppendLine($"->with_field(\"{name}\", &self::{name})");
}
var baseType = isRecord ? $"record<{type}_key_impl, {type}_data_impl>" :
isDerived ? $"{decl.Inherit.Name.Underscore()}_data" :
"data";
writer.AppendLine($"->template with_base<{baseType}>()");
writer.AppendLine($"->with_constructor(&make_{type}_data, {{ }})");
writer.AppendLine("->build();");
writer.PopIndent();
writer.AppendLine("return type_;");
writer.PopIndent();
writer.AppendLine("}");
}
/// <summary>
/// Converts type inherited from Data to TypeDecl
/// </summary>
public static TypeDecl TypeToDecl(System.Type type, CommentNavigator navigator, ProjectNavigator projNavigator)
{
if (!type.IsSubclassOf(typeof(Data)))
{
throw new ArgumentException($"Cannot create type declaration from type: {type.FullName}.");
}
TypeDecl decl = new TypeDecl();
decl.Module = new ModuleKey {
ModuleName = type.Namespace
};
decl.Category = projNavigator?.GetTypeLocation(type);
decl.Name = type.Name;
decl.Label = GetLabelFromAttribute(type) ?? type.Name;
decl.Comment = GetCommentFromAttribute(type) ?? navigator?.GetXmlComment(type);
decl.Kind = GetKind(type);
decl.IsRecord = type.IsSubclassOf(typeof(Record));
decl.Inherit = IsRoot(type.BaseType)
? null
: CreateTypeDeclKey(type.BaseType.Namespace, type.BaseType.Name);
decl.Index = GetIndexesFromAttributes(type);
// Skip special (property getters, setters, etc) and inherited methods
List <MethodInfo> handlers = type.GetMethods(PublicInstanceDeclaredFlags)
.Where(IsProperHandler)
.ToList();
var declares = new List <HandlerDeclareItem>();
var implements = new List <HandlerImplementItem>();
foreach (MethodInfo method in handlers)
{
// Abstract methods have only declaration
if (method.IsAbstract)
{
declares.Add(ToDeclare(method, navigator));
}
// Overriden methods are marked with override
else if (method.GetBaseDefinition() != method)
{
// TODO: Temp adding declare to avoid signature search in bases.
declares.Add(ToDeclare(method, navigator));
implements.Add(ToImplement(method));
}
// Case for methods without modifiers
else
{
declares.Add(ToDeclare(method, navigator));
implements.Add(ToImplement(method));
}
}
// Add method information to declaration
if (declares.Any())
{
decl.Declare = new HandlerDeclareBlock {
Handlers = declares
}
}
;
if (implements.Any())
{
decl.Implement = new HandlerImplementBlock {
Handlers = implements
}
}
;
List <PropertyInfo> dataProperties = type.GetProperties(PublicInstanceDeclaredFlags)
.Where(p => IsAllowedType(p.PropertyType))
.Where(IsPublicGetSet).ToList();
decl.Elements = dataProperties.Select(p => ToElement(p, navigator)).ToList();
decl.Keys = GetKeyProperties(type)
.Where(p => IsAllowedType(p.PropertyType))
.Where(IsPublicGetSet)
.Select(t => t.Name).ToList();
return(decl);
}
/// <summary>
/// Generate python classes from declaration.
/// </summary>
public static string Build(TypeDecl decl, List <IDecl> declarations)
{
var writer = new CodeWriter();
string name = decl.Name;
// Determine if we are inside datacentric package
// based on module name. This affects the imports
// and namespace use.
bool insideDc = PyExtensions.GetPackage(decl) == "datacentric";
// If not generating for DataCentric package, use dc. namespace
// in front of datacentric types, otherwise use no prefix
string dcNamespacePrefix = insideDc ? "" : "dc.";
PythonImportsBuilder.WriteImports(decl, declarations, writer);
writer.AppendNewLineWithoutIndent();
writer.AppendNewLineWithoutIndent();
// Get base classes for current declaration
List <string> bases = new List <string>();
if (decl.Keys.Any())
{
bases.Add(dcNamespacePrefix + "Record");
}
else if (decl.Inherit != null)
{
// Full package name and short namespace of the parent class,
// or null if there is no parent
bool parentClassInDifferentModule = !PyExtensions.IsPackageEquals(decl, decl.Inherit);
string parentPackage = PyExtensions.GetPackage(decl.Inherit);
string parentClassNamespacePrefix =
parentClassInDifferentModule ? PyExtensions.GetAlias(parentPackage) + "." : "";
bases.Add(parentClassNamespacePrefix + decl.Inherit.Name);
}
else
{
bases.Add(dcNamespacePrefix + "Data");
}
if (decl.Kind == TypeKind.Abstract)
{
bases.Add("ABC");
}
// Python 3.8:
// if (decl.Kind == TypeKind.Final)
// writer.AppendLine("@final");
writer.AppendLine("@attr.s(slots=True, auto_attribs=True)");
writer.AppendLine($"class {name}({string.Join(", ", bases)}):");
writer.PushIndent();
writer.AppendLines(CommentHelper.PyComment(decl.Comment));
writer.AppendNewLineWithoutIndent();
if (!decl.Elements.Any())
{
writer.AppendLine("pass");
}
foreach (var element in decl.Elements)
{
// TODO: Should be replaced with callable with specific format instead of skipping
string skipRepresentation = element.Vector == YesNo.Y ? ", repr=False" : "";
writer.AppendLine($"{element.Name.Underscore()}: {GetTypeHint(decl, element)} = attr.ib(default=None, kw_only=True{skipRepresentation}{GetMetaData(element)})");
writer.AppendLines(CommentHelper.PyComment(element.Comment));
if (element != decl.Elements.Last())
{
writer.AppendNewLineWithoutIndent();
}
}
// Add to_key and create_key() methods
if (decl.Keys.Any())
{
var keyElements = decl.Elements.Where(e => decl.Keys.Contains(e.Name)).ToList();
writer.AppendNewLineWithoutIndent();
writer.AppendLine("def to_key(self) -> str:");
writer.PushIndent();
writer.AppendLine(CommentHelper.PyComment($"Get {decl.Name} key."));
writer.AppendLines($"return '{decl.Name}='{GetToKeyArgs(decl.Name, keyElements, true)}");
writer.PopIndent();
writer.AppendNewLineWithoutIndent();
var namedParams = keyElements.Select(e => $"{e.Name.Underscore()}: {GetTypeHint(decl, e)}").ToList();
var joinedNamedParams = string.Join(", ", namedParams);
string start = "def create_key(";
// Check if tokens should be separated by new line
if (4 + start.Length + joinedNamedParams.Length > 120)
{
var indent = new string(' ', start.Length);
joinedNamedParams = string.Join("," + Environment.NewLine + indent, namedParams);
}
writer.AppendLine("@classmethod");
writer.AppendLines($"def create_key(cls, *, {joinedNamedParams}) -> str:");
writer.PushIndent();
writer.AppendLine(CommentHelper.PyComment($"Create {decl.Name} key."));
writer.AppendLines($"return '{decl.Name}='{GetToKeyArgs(decl.Name, keyElements, false)}");
writer.PopIndent();
}
if (decl.Declare != null)
{
writer.AppendNewLineWithoutIndent();
WriteMethods(decl, writer);
}
// Class end
writer.PopIndent();
return(writer.ToString());
}
private static void BuildClassDeclaration(TypeDecl decl, CodeWriter writer)
{
var settings = GeneratorSettingsProvider.Get(decl.Module.ModuleName);
var type = decl.Name.Underscore();
bool isRecord = decl.Keys.Any();
bool isDerived = decl.Inherit != null;
// Self-forward
writer.AppendLine($"class {type}_data_impl; using {type}_data = dot::ptr<{type}_data_impl>;");
if (isRecord)
{
writer.AppendLine($"class {type}_key_impl; using {type}_key = dot::ptr<{type}_key_impl>;");
}
// Get unique keys and data from elements
var dataForwards = decl.Elements.Where(e => e.Data != null)
.Where(e => e.Data.Module.ModuleName == decl.Module.ModuleName)
.Select(e => $"{e.Data.Name.Underscore()}_data").ToList();
var keysForwards = decl.Elements.Where(e => e.Key != null)
.Where(e => e.Key.Module.ModuleName == decl.Module.ModuleName)
.Select(e => $"{e.Key.Name.Underscore()}_key").ToList();
var forwards = keysForwards.Union(dataForwards).Distinct();
// Appends forwards
foreach (var f in forwards)
{
writer.AppendLine($"class {f}_impl; using {f} = dot::ptr<{f}_impl>;");
}
writer.AppendNewLineWithoutIndent();
writer.AppendLine($"inline {type}_data make_{type}_data();");
writer.AppendNewLineWithoutIndent();
writer.AppendLines(CommentHelper.FormatComment(decl.Comment));
var baseType = isRecord ? $"record_impl<{type}_key_impl, {type}_data_impl>" :
isDerived ? $"{decl.Inherit.Name.Underscore()}_data_impl" :
"data_impl";
writer.AppendLine($"class {settings.DeclSpec} {type}_data_impl : public {baseType}");
writer.AppendLine("{");
writer.PushIndent();
writer.AppendLine($"typedef {type}_data_impl self;");
writer.AppendLine($"friend {type}_data make_{type}_data();");
writer.PopIndent();
writer.AppendNewLineWithoutIndent();
writer.AppendLine("public: // FIELDS");
writer.AppendNewLineWithoutIndent();
var elements = decl.Elements;
if (elements.Any())
{
writer.PushIndent();
CppElementBuilder.WriteElements(decl.Elements, writer);
writer.PopIndent();
}
if (decl.Declare != null)
{
writer.AppendLine("public: // METHODS");
writer.AppendNewLineWithoutIndent();
writer.PushIndent();
CppMethodBuilder.WriteElements(decl, writer);
writer.PopIndent();
}
writer.AppendLine("public:");
writer.PushIndent();
writer.AppendLine("virtual dot::type_t type();");
writer.AppendLine("static dot::type_t typeof();");
writer.PopIndent();
writer.AppendLine("};");
writer.AppendNewLineWithoutIndent();
writer.AppendLine("/// Create an empty instance.");
writer.AppendLine($"inline {type}_data make_{type}_data() {{ return new {type}_data_impl; }}");
}
/// <summary>
/// Add import statements for given declaration.
/// </summary>
public static void WriteImports(TypeDecl decl, List <IDecl> declarations, CodeWriter writer)
{
// Always import attr module
writer.AppendLine("import attr");
// Instant is generated as Union[dt.datetime, dc.Instant] thus dt import is required
if (decl.Elements.Any(e => e.Value != null && (e.Value.Type == ValueParamType.Instant ||
e.Value.Type == ValueParamType.NullableInstant)))
{
writer.AppendLine("import datetime as dt");
}
// If type is abstract - ABC import is needed
if (decl.Kind == TypeKind.Abstract)
{
writer.AppendLine("from abc import ABC");
}
// Check if ObjectId is used
bool hasObjectId = decl.Elements.Any(e => e.Value != null &&
(e.Value.Type == ValueParamType.TemporalId ||
e.Value.Type == ValueParamType.NullableTemporalId));
if (hasObjectId)
{
writer.AppendLine("from bson import ObjectId");
}
// Check imports from typing
var typingImports = new List <string>();
// Python 3.8
// if (decl.Keys.Any() || decl.Kind == TypeKind.Final)
// typingImports.Add("final");
if (decl.Elements.Any(e => e.Vector == YesNo.Y))
{
typingImports.Add("List");
}
if (typingImports.Any())
{
var items = string.Join(", ", typingImports);
writer.AppendLine($"from typing import {items}");
}
bool insideDc = PyExtensions.GetPackage(decl) == "datacentric";
List <string> packagesToImport = new List <string>();
List <string> individualImports = new List <string>();
// Import parent class package as its namespace, or if inside the same package,
// import individual class instead
if (decl.Inherit != null)
{
if (PyExtensions.IsPackageEquals(decl, decl.Inherit))
{
IDecl parentDecl = declarations.FindByKey(decl.Inherit);
individualImports.Add($"from {parentDecl.Category} import {decl.Inherit.Name}");
}
else
{
packagesToImport.Add(PyExtensions.GetPackage(decl.Inherit));
}
}
// Import datacentric package as dc, or if inside datacentric,
// import individual classes instead
else if (decl.IsRecord)
{
if (insideDc)
{
individualImports.Add("from datacentric.storage.record import Record");
}
else
{
packagesToImport.Add("datacentric");
}
}
// First child class of Data
else
{
if (insideDc)
{
individualImports.Add("from datacentric.storage.data import Data");
}
else
{
packagesToImport.Add("datacentric");
}
}
foreach (var data in decl.Elements.Where(d => d.Data != null).Select(d => d.Data))
{
if (PyExtensions.IsPackageEquals(decl, data))
{
IDecl dataDecl = declarations.FindByKey(data);
individualImports.Add($"from {dataDecl.Category} import {data.Name}");
}
else
{
packagesToImport.Add(PyExtensions.GetPackage(data));
}
}
foreach (var enumElement in decl.Elements.Where(d => d.Enum != null).Select(d => d.Enum))
{
if (PyExtensions.IsPackageEquals(decl, enumElement))
{
IDecl enumDecl = declarations.FindByKey(enumElement);
individualImports.Add($"from {enumDecl.Category} import {enumElement.Name}");
}
else
{
packagesToImport.Add(PyExtensions.GetPackage(enumElement));
}
}
foreach (var package in packagesToImport.Distinct())
{
writer.AppendLine($"import {package} as {PyExtensions.GetAlias(package)}");
}
foreach (var import in individualImports.Distinct())
{
writer.AppendLine(import);
}
}
