/// <summary>
/// Gets the content of aggregate keys
/// </summary>
/// <param name="script"></param>
/// <param name="keyBuffName"></param>
/// <returns></returns>
private string GetAggregateKeyExtraction(DonutScript script, string keyBuffName)
{
var rootIgn = script.GetRootIntegration();
var ignKeys = rootIgn.AggregateKeys;
if (ignKeys == null || !ignKeys.Any())
{
throw new Exception("Integration has no aggregate keys!");
}
var keys = ignKeys.Where(x => x.Operation != null);
// if (keys == null || !keys.Any())
// {
// throw new Exception("Integration has no evaluable aggregate keys!");
// }
var buffer = new StringBuilder();
int iKey = 0;
buffer.AppendLine($"var {keyBuffName} = new Dictionary<string, object>();");
foreach (var key in keys)
{
var keyName = $"aggKey{iKey}";
var evalFnName = $"{keyName}_fn";
var evalFn = key.Operation.GetCallCode(evalFnName).ToString();
//var keyVar = $"var {keyName} = {evalFnName}(document[\"{key.Arguments}\"]);";
var keyLine = $"{keyBuffName}[\"{key.Name}\"] = {evalFnName}(document[\"{key.Arguments}\"]);";
buffer.AppendLine(evalFn);
buffer.AppendLine(keyLine);
iKey++;
}
buffer.AppendLine($"var groupKey = Context.AddMetaGroup({keyBuffName});");
return(buffer.ToString());
}
/// <summary>
/// Generates a donutfile context
/// </summary>
/// <param name="namespace"></param>
/// <param name="script"></param>
/// <returns></returns>
public string GenerateContext(string @namespace, DonutScript script)
{
string ctxTemplate;
_expVisitor.Clear();
_expVisitor.SetScript(script);
if (script.Type == null)
{
throw new ArgumentException("Script type is null!");
}
var baseName = script.Type.GetContextName();
using (StreamReader reader = new StreamReader(GetTemplate("DonutContext.txt")))
{
ctxTemplate = reader.ReadToEnd();
if (string.IsNullOrEmpty(ctxTemplate))
{
throw new Exception("Template empty!");
}
ctxTemplate = ctxTemplate.Replace("$Namespace", @namespace);
ctxTemplate = ctxTemplate.Replace("$ClassName", baseName);
var cacheSetMembers = GetCacheSetMembers(script);
ctxTemplate = ctxTemplate.Replace("$CacheMembers", cacheSetMembers);
var dataSetMembers = GetDataSetMembers(script);
ctxTemplate = ctxTemplate.Replace("$DataSetMembers", dataSetMembers);
var mappers = GetContextTypeMappers(script);
ctxTemplate = ctxTemplate.Replace("$Mappers", mappers);
//Items: $Namespace, $ClassName, $CacheMembers, $DataSetMembers, $Mappers
}
return(ctxTemplate);
}
private string GetContextTypeMappers(DonutScript dscript)
{
//Template:
//RedisCacher.RegisterCacheMap<MapTypeName, TypeToMapNamme>
var sb = new StringBuilder();
return(sb.ToString());
}
private static void GetIntegrationRecordVars(DonutScript script, StringBuilder fBuilder)
{
foreach (var integration in script.GetDatasetMembers())
{
var iName = integration.GetPropertyName();
var record = $"var rec{iName} = this.Context.{iName}.Records;";
fBuilder.AppendLine(record);
}
}
private string GetDataSetMembers(DonutScript dscript)
{
var dtSources = dscript.GetDatasetMembers();
var content = new StringBuilder();
foreach (var source in dtSources)
{
var sourceProperty = $"[SourceFromIntegration(\"{source.Name}\")]\n" +
"public DataSet<BsonDocument> " + source.GetPropertyName() + " { get; set; }";
content.AppendLine(sourceProperty);
}
return(content.ToString());
}
/// <summary>
/// Generates the code that processes each incoming document, in order to gather information for features.
/// </summary>
/// <param name="script"></param>
/// <returns></returns>
private string ProcessFeaturePrepContent(string donutfileContent, DonutScript script)
{
string aggregateKeyContent = GetAggregateKeyExtraction(script, "aggKeyBuff");
var placeholder = "$ExtractionBody";
var donutCV = new DonutFeatureGeneratingExpressionVisitor(script);
var donutFeatureGen = new DonutFeatureCodeGenerator(script, donutCV);
var donutFeatures = script.Features.Where(x => x.IsDonutNativeFeature());
donutFeatureGen.AddAll(donutFeatures);
var donutFeatureCode = donutFeatureGen.GetScriptContent(donutfileContent);
donutFeatureCode.PrepareScript = aggregateKeyContent + donutFeatureCode.PrepareScript;
donutfileContent = donutfileContent.Replace(placeholder, donutFeatureCode.PrepareScript);
return(donutfileContent);
}
private string GetCacheSetMembers(DonutScript dscript)
{
var featureAssignments = dscript.Features;
var content = new StringBuilder();
foreach (var fassign in featureAssignments)
{
var name = fassign.Member.Name;
var sName = name.Replace(' ', '_');
var typeName = "string";
//Resolve the type name if needed
var sourceProperty = $"public CacheSet<{typeName}> " + sName + " { get; set; }";
content.AppendLine(sourceProperty);
}
return(content.ToString());
}
public string GenerateProjectDefinition(string assemblyName, DonutScript script)
{
using (var template = new StreamReader(GetTemplate("Project.txt")))
{
var content = template.ReadToEnd();
var referenceBuilder = new StringBuilder();
foreach (var reference in References)
{
var referenceTemplate = $"<PackageReference Incude=\"{reference.Display}\" />\n";
referenceBuilder.Append(referenceTemplate);
//<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="2.0.1" />
}
content = content.Replace("$References", referenceBuilder.ToString());
return content;
}
}
private string GetFeatureYieldsContent(DonutScript script)
{
var fBuilder = new StringBuilder();
var donutFnResolver = new DonutFunctions();
foreach (var feature in script.Features)
{
IExpression accessor = feature.Value;
string fName = feature.Member.ToString();
string featureContent = "";
var featureFType = accessor.GetType();
if (featureFType == typeof(NameExpression))
{
var member = (accessor as NameExpression).Member?.ToString();
//In some cases we might just use the field
if (string.IsNullOrEmpty(member))
{
member = accessor.ToString();
}
featureContent = $"yield return pair(\"{fName}\", doc[\"{member}\"]);";
}
else if (featureFType == typeof(CallExpression))
{
if (donutFnResolver.IsAggregate(accessor as CallExpression))
{
//We're dealing with an aggregate call
//var aggregateContent = GenerateFeatureFunctionCall(accessor as CallExpression, feature);
}
else
{
//We're dealing with a function call
//featureContent = GenerateFeatureFunctionCall(accessor as CallExpression, feature).Content;
}
}
else
{
throw new NotImplementedException();
}
if (!string.IsNullOrEmpty(featureContent))
{
fBuilder.AppendLine(featureContent);
}
}
return(fBuilder.ToString());
}
public DonutFeatureCodeGenerator(DonutScript script, DonutFeatureGeneratingExpressionVisitor expVisitor) : base(script)
{
_donutFnResolver = new DonutFunctions();
_rootIntegration = script.Integrations.FirstOrDefault();
if (_rootIntegration == null)
{
throw new InvalidIntegrationException("Script has no integrations");
}
if (_rootIntegration.Fields == null || _rootIntegration.Fields.Count == 0)
{
throw new InvalidIntegrationException("Integration has no fields");
}
_rootDataMember = script.GetDatasetMember(_rootIntegration.Name);
_outputCollection = _rootIntegration.FeaturesCollection;
if (string.IsNullOrEmpty(_outputCollection))
{
throw new InvalidOperationException("Root integration must have a features collection set.");
}
_expVisitor = expVisitor ?? throw new ArgumentNullException(nameof(expVisitor));
_functions = new List <IDonutFunction>();
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public DonutScript ParseDonutScript(IIntegration ign)
{
_sourceIntegrations.Clear();
DonutScript newScript = new DonutScript();
Reader.DiscardToken(TokenType.Define);
var newSymbolName = Reader.DiscardToken(TokenType.Symbol);
newScript.Type = new ScriptTypeInfo(newSymbolName.Value);
_sourceIntegrations = ReadFrom();
var orderBy = ReadOrderBy();
var expressions = ReadExpressions();
newScript.StartingOrderBy = orderBy;
foreach (var expression in expressions)
{
var expressionType = expression.GetType();
if (expressionType == typeof(AssignmentExpression))
{
newScript.Features.Add(expression as AssignmentExpression);
}
else if (expressionType == typeof(TargetExpression))
{
var targetExpr = expression as TargetExpression;
var targetFields = targetExpr.Attributes.Select(a => ign.Fields.FirstOrDefault(f => f.Name == a));
newScript.Targets = new List <ModelTarget>();
foreach (var targetField in targetFields)
{
((List <ModelTarget>)newScript.Targets).Add(new ModelTarget(targetField));
}
}
else
{
throw new NotImplementedException();
}
}
newScript.AddIntegrations(_sourceIntegrations.Select(i => new Donut.Data.DataIntegration(i)).ToArray());
return(newScript);
}
public static IDonutBuilder Create <TData>(DonutScript script,
IIntegration integration, IServiceProvider serviceProvider)
where TData : class, IIntegratedDocument
{
if (string.IsNullOrEmpty(script.AssemblyPath))
{
return(null);
}
if (!File.Exists(script.AssemblyPath))
{
throw new Exception("Donut assembly not found!");
}
var asm = Assembly.LoadFrom(script.AssemblyPath);
var donutTypes = asm.GetTypes();
var donutType = donutTypes.FirstOrDefault(x => x.IsInstanceOfType(typeof(IDonutfile)));
var donutContextType = donutTypes.FirstOrDefault(x => x.IsInstanceOfType(typeof(IDonutContext)));
var featureEmitterType = donutTypes.FirstOrDefault(x => x.IsInstanceOfType(typeof(IDonutFeatureEmitter)));
var cacher = serviceProvider.GetService(typeof(IRedisCacher)) as IRedisCacher;
var builder = Create <TData>(donutType, donutContextType, integration, cacher, serviceProvider);
builder.SetEmitterType(featureEmitterType);
return(builder);
}
/// <summary>
/// Generates a donutfile
/// </summary>
/// <param name="namespace"></param>
/// <param name="script"></param>
/// <returns></returns>
public string GenerateDonut(string @namespace, DonutScript script)
{
string donutTemplate;
var baseName = script.Type.GetClassName();
var conutextName = script.Type.GetContextName();
_expVisitor.Clear();
_expVisitor.SetScript(script);
var encoding = System.Text.Encoding.UTF8;
using (StreamReader reader = new StreamReader(GetTemplate("Donutfile.txt"), encoding))
{
donutTemplate = reader.ReadToEnd();
if (string.IsNullOrEmpty(donutTemplate))
{
throw new Exception("Template empty!");
}
var donutComment = $"//Generated on {DateTime.UtcNow} UTC\n";
donutComment += $"//Root collection on {script.GetRootIntegration().Collection}\n";
donutTemplate = $"{donutComment}\n" + donutTemplate;
donutTemplate = donutTemplate.Replace("$Namespace", @namespace);
donutTemplate = donutTemplate.Replace("$ClassName", baseName);
donutTemplate = donutTemplate.Replace("$ContextTypeName", conutextName);
var prepareScript = GeneratePrepareExtractionContent(script);
donutTemplate = donutTemplate.Replace("$PrepareExtraction", prepareScript);
donutTemplate = ProcessFeaturePrepContent(donutTemplate, script); //donutTemplate.Replace("$ExtractionBody", GetFeaturePrepContent(donutTemplate, script));
donutTemplate = donutTemplate.Replace("$CompleteExtraction", GetFeatureExtractionCompletion(script));
var propertiesContent = GenerateDonutPropertiesContent(!string.IsNullOrEmpty(prepareScript));
donutTemplate = donutTemplate.Replace("$DonutProperties", propertiesContent);
donutTemplate = donutTemplate.Replace("$OnFinished", GenerateOnDounutFinishedContent(script));
donutTemplate += "\n\n" +
"/* Donut script: \n" + script.ToString() +
"\n*/";
//Items: $ClassName, $ContextTypeName, $ExtractionBody, $OnFinished
}
return(donutTemplate);
}
/// <summary>
///
/// </summary>
/// <param name="script"></param>
/// <returns></returns>
private string GeneratePrepareExtractionContent(DonutScript script)
{
var fBuilder = new StringBuilder();
var rootIntegration = script.Integrations.FirstOrDefault();
if (rootIntegration == null)
{
throw new InvalidIntegrationException("Script has no integrations");
}
if (rootIntegration.Fields == null || rootIntegration.Fields.Count == 0)
{
throw new InvalidIntegrationException("Integration has no fields");
}
//Get our record variables
GetIntegrationRecordVars(script, fBuilder);
var aggregates = new AggregateFeatureCodeGenerator(script, _expVisitor);
var aggFeatures = script.Features.Where(x => x.IsDonutAggregateFeature());
aggregates.AddAll(aggFeatures);
var aggregatePipeline = aggregates.GetScriptContent();
fBuilder.Append(aggregatePipeline);
return(fBuilder.ToString());
}
public string GenerateFeatureGenerator(string @namespace, DonutScript script)
{
string fgenTemplate;
var donutName = script.Type.GetClassName();
var conutextName = script.Type.GetContextName();
_expVisitor.Clear();
_expVisitor.SetScript(script);
using (StreamReader reader = new StreamReader(GetTemplate("FeatureGenerator.txt")))
{
fgenTemplate = reader.ReadToEnd();
if (string.IsNullOrEmpty(fgenTemplate))
{
throw new Exception("Template empty!");
}
fgenTemplate = fgenTemplate.Replace("$Namespace", @namespace);
fgenTemplate = fgenTemplate.Replace("$DonutType", donutName);
fgenTemplate = fgenTemplate.Replace("$DonutContextType", conutextName);
fgenTemplate = fgenTemplate.Replace("$ContextTypeName", conutextName);
fgenTemplate = fgenTemplate.Replace("$FeatureYields", GetFeatureYieldsContent(script));
//Items: $Namespace, $DonutType, $FeatureYields
}
return(fgenTemplate);
}
private string GetFeatureExtractionCompletion(DonutScript script)
{
var code = "";
return(code);
}
//private DataIntegration _integration;
public DonutScriptCodeGenerator(DonutScript script)
{
_expVisitor = new AggregateFeatureGeneratingExpressionVisitor(script);
//_integration = integration;
}
public FeatureCodeGenerator(DonutScript script)
{
this.Script = script;
}
private string GenerateOnDounutFinishedContent(DonutScript script)
{
return(String.Empty);
}
public DonutCompiler(DonutScript dscript)
{
_script = dscript;
_codeGen = dscript.GetCodeGenerator() as DonutScriptCodeGenerator;
_assembliesDir = Path.Combine(Environment.CurrentDirectory, "donutAssemblies");
}
public void SetScript(DonutScript script)
{
_script = script;
}