public void TakeNonPersistentExprs(ITransformationSchema schema)
{
this.nonPersistentExprs.Clear();
foreach (AssignmentObject assignmentObject in schema.AssignmentObjects)
{
IExpression expr = assignmentObject.Expression;
if (!assignmentObject.IsPersistentAssignment)
{
string renderResult = this._opRendererResolver(expr.OperatorSymbol).Render(expr);
if (expr.IsScalar)
{
this.nonPersistentExprs.Add(assignmentObject.Expression, $"DECLARE @{assignmentObject.Name} {this.GetType(expr)} = {renderResult};");
}
else
{
StringBuilder result = new StringBuilder();
if (expr.OperatorSymbol == "fill_time_series")
{
result.AppendLine((this._opRendererResolver(expr.OperatorSymbol) as FillTimeSeriesOperatorRenderer).RenderWith(expr));
}
result.AppendLine($"SELECT * INTO #{assignmentObject.Name} FROM (\n{renderResult}) AS t");
if (expr.OperatorSymbol == "exists_in" && expr.OperatorDefinition.Keyword == "false")
{
result.Append($" WHERE {expr.Structure.Measures[0].ComponentName} = 0");
}
this.nonPersistentExprs.Add(assignmentObject.Expression, result.ToString());
}
}
}
}
/// <summary>
/// Performs removing a dead code for a transformation schema.
/// </summary>
/// <param name="schema">The transformation schema to modify.</param>
public void Modify(ITransformationSchema schema)
{
this.schema = schema;
List <string> usedObjects = new List <string>(); // object list used in the schema
for (byte stage = 1; stage <= 3; stage++)
{
if (stage == 2) // stage of removing unused non persistent objects (refs) from the schema
{
this.schema.AssignmentObjects.RemoveAll(ao => !ao.IsPersistentAssignment && !ao.Name.In(usedObjects.ToArray()));
}
else
{
foreach (AssignmentObject assignmentObject in this.schema.AssignmentObjects)
{
if (stage == 1 && assignmentObject.IsPersistentAssignment) // stage of searching used objects by the current object - if the persistent assignment object
{
usedObjects.Add(assignmentObject.Name); // persistent assignment object addition
usedObjects.AddRange(this.SearchRefs(assignmentObject.Expression)); // addition of used object by the current persistent assignment object
}
else if (stage == 3) // stage of removing unused non persistent objects (refs) from assignment objects users
{
if (assignmentObject.IsPersistentAssignment)
{
(assignmentObject.Users as List <AssignmentObject>).Clear(); // persistent assignment object doesn't need specified users - it's not a reference
}
else
{
(assignmentObject.Users as List <AssignmentObject>).RemoveAll(user => !user.Name.In(usedObjects.ToArray())); // removement of unused non persistent objects (refs)
}
}
}
}
}
}
/// <summary>
/// Performs replacing dataset operators to "join" operators.
/// </summary>
/// <param name="schema">The schema to modify.</param>
public void Modify(ITransformationSchema schema)
{
foreach (AssignmentObject assignmentObject in schema.AssignmentObjects)
{
assignmentObject.Expression = this.TransformDatasets(assignmentObject.Expression);
}
}
public void Constructor_JoinExpr_JoinExpr()
{
IExpression expr = TestExprFactory.GetExpression("join", ExpressionFactoryNameTarget.OperatorSymbol);
IExpression refExpr = ModelResolvers.ExprResolver();
ITransformationSchema schema = ModelResolvers.SchemaResolver();
refExpr.AddOperand("ds", expr);
expr.ContainingSchema = schema;
expr.ReferenceExpression = refExpr;
IJoinExpression joinExpr2 = ModelResolvers.JoinExprResolver(expr);
joinExpr2.BasicStructure = ModelResolvers.DsResolver();
JoinExpression joinExpr = new JoinExpression(joinExpr2);
Assert.Equal(expr.ParentExpression, joinExpr.ParentExpression);
Assert.Equal(expr.ContainingSchema, joinExpr.ContainingSchema);
Assert.Equal(expr.ExpressionText, joinExpr.ExpressionText);
Assert.Equal(expr.LineNumber, joinExpr.LineNumber);
Assert.Equal(expr.OperandsCollection, joinExpr.OperandsCollection);
Assert.Equal(expr.OperatorDefinition, joinExpr.OperatorDefinition);
Assert.Equal(expr.ParamSignature, joinExpr.ParamSignature);
Assert.Equal(expr.ReferenceExpression, joinExpr.ReferenceExpression);
Assert.Equal(expr.ResultName, joinExpr.ResultName);
Assert.Equal(expr.Structure, joinExpr.Structure);
Assert.Equal(joinExpr2.BasicStructure, joinExpr.BasicStructure);
}
/// <summary>
/// Performs putting "using" branches to "inner" and "left" "join" operator expressions that do not contain them.
/// </summary>
/// <param name="schema">The schema to modify.</param>
public void Modify(ITransformationSchema schema)
{
foreach (AssignmentObject assignmentObject in schema.AssignmentObjects)
{
this.ModifyJoins(assignmentObject.Expression);
}
}
public void MapNames(ITransformationSchema schema)
{
foreach (AssignmentObject assignmentObject in schema.AssignmentObjects)
{
this.MapName(assignmentObject.Expression, true);
}
}
public void SetContainingSchema(ITransformationSchema schema)
{
this.ContainingSchema = schema;
foreach (IExpression operand in this.OperandsCollection)
{
operand.SetContainingSchema(schema);
}
}
public string Render(ITransformationSchema schema)
{
if (this._conf.ExprType == ExpressionsType.Standard)
{
return(this.Render(this.RenderTransformationSchema(schema.GetExpressions())));
}
return(this.Render(this.RenderTransformationSchema(schema.Rulesets)));
}
/// <summary>
/// Initialises a new instance of the <see cref="AssignmentObject"/> class.
/// </summary>
/// <param name="schema">The transformation schema the assignment object is assigned to.</param>
/// <param name="expression">The expression to assign to assignment object.</param>
/// <param name="isPersistentAssignment">The value indicating whether the assignment is persistent.</param>
/// <param name="refsNames">The collection of assignment objects names that this instance's expression contains.</param>
public AssignmentObject(ITransformationSchema schema, IExpression expression, bool isPersistentAssignment, ICollection <string> refsNames)
{
this.ContainingSchema = schema;
this.Name = expression.ResultName;
this.Expression = expression;
this.IsPersistentAssignment = isPersistentAssignment;
this.Users = new List <AssignmentObject>();
foreach (string userName in refsNames)
{
this.ContainingSchema.AssignmentObjects[userName].Users.Add(this);
}
}
public string Render(ITransformationSchema schema)
{
StringBuilder scriptSB = new StringBuilder();
StringBuilder assignmentsSB = new StringBuilder();
try
{
this._mapper.MapNames(schema);
this._refs.TakeNonPersistentExprs(schema);
foreach (AssignmentObject assignmentObject in schema.AssignmentObjects)
{
if (this._refs.ContainsExpression(assignmentObject.Expression))
{
assignmentsSB.AppendLine(this._refs.RenderNonPersistentExpr(assignmentObject.Expression)); // non-persistent assignment expressions rendering
}
else
{
assignmentsSB.AppendLine(this.RenderPersistentExpr(assignmentObject.Expression)); // persistent assignment expressions rendering
}
}
if (this._conf.UseComments)
{
scriptSB.AppendLine($"-- Script generated: {DateTime.Now}");
}
scriptSB.AppendLine("BEGIN TRANSACTION\n");
for (int i = 1; i <= this._tmpTables.Count; i++)
{
scriptSB.AppendLine($"IF OBJECT_ID (N'tempdb..{this._tmpTables.Name}{i}', N'U') IS NOT NULL");
scriptSB.AppendLine($"DROP TABLE {this._tmpTables.Name}{i}\n");
}
scriptSB.AppendLine(this._refs.RenderDroppingOfTables());
scriptSB.Append(assignmentsSB.ToString());
scriptSB.AppendLine("COMMIT TRANSACTION");
scriptSB.AppendLine("GO");
}
catch (VtlTargetError ex)
{
this._logger?.LogError(ex, ex.Message);
}
catch (Exception ex)
{
this._logger?.LogCritical(ex, ex.Message);
}
return(scriptSB.ToString());
}
/// <summary>
/// Translates a given transformation schema to target's language code whose target name is given.
/// </summary>
/// <param name="schema">The transformation schema.</param>
/// <param name="targetName">The name of a target.</param>
/// <param name="schemaProcessed">Specifies if the schema has been processed by middle-end operations (ex. type inference).</param>
/// <returns></returns>
public string Translate(ITransformationSchema schema, string targetName, bool schemaProcessed = false)
{
ITargetRenderer targetRenderer = this._provider.GetTargetRenderer(targetName);
if (!schemaProcessed)
{
this._provider.GetMiddleEnd().Process(schema);
}
if (targetRenderer == null)
{
throw new NullReferenceException($"Target renderer named \"{targetName}\" has been not found.");
}
return(this._provider.GetTargetRenderer(targetName).Render(schema));
}
public void SetContainingSchema_Schema_SetsSchema()
{
ITransformationSchema schema = ModelResolvers.SchemaResolver();
Expression expr = new Expression();
expr.AddOperand("ds_1", ModelResolvers.ExprResolver());
Assert.Null(expr.ContainingSchema);
Assert.Null(expr.Operands["ds_1"].ContainingSchema);
expr.SetContainingSchema(schema);
Assert.Equal(expr.ContainingSchema, schema);
Assert.Equal(expr.Operands["ds_1"].ContainingSchema, schema);
}
public TranslationResponse Tanslate(TranslationParameters parameters)
{
this._dataModelAggregator.EnvironmentMapper.Mapping = parameters.DataMappers;
this._dataModelAggregator.DefaultNamespace = parameters.DefaultNamespace;
this._dataModelAggregator.DataModelsCollection.Clear();
foreach (var dataSource in parameters.DataSources)
{
switch (dataSource.Type.ToLower())
{
case "json":
this._dataModelAggregator.AddJsonModel(dataSource.Localization);
break;
case "sdmx":
this._dataModelAggregator.AddSdmxModel(dataSource.Localization, dataSource.Namespace);
break;
case "tsql":
this._dataModelAggregator.AddSqlServerModel(dataSource.Localization);
break;
default:
return(new TranslationResponse(new ArgumentException("Unexpected model type.")));
}
}
ITransformationSchema schema = this._treeGenerator.BuildTransformationSchema(parameters.Experession);
this._schemaModifiersApplier.Process(schema);
bool areErrors = this._errorCollectorProvider.ErrorCollectors.Sum(counter => counter.Errors.Count) > 0;
if (areErrors)
{
return(new TranslationResponse(this._errorCollectorProvider.ErrorCollectors.SelectMany(ec => ec.Errors).ToList()));
}
ITargetRenderer target = this._targetRenderers.FirstOrDefault(tr => tr.Name == parameters.Target);
if (target == null)
{
return(new TranslationResponse(new ArgumentException("Unexpected target type.")));
}
return(new TranslationResponse(target.Render(schema)));
}
static void Main(string[] args)
{
Translator translator = new Translator((configure) =>
{
configure.AddPlantUmlTarget((config) =>
{
config.AddDataStructureObject();
config.UseArrowFirstToLast();
config.ShowNumberLine();
});
configure.AddTsqlTarget((config) =>
{
config.AddComments();
});
configure.AddLogging((config) =>
{
config.AddConsole();
config.AddDebug();
});
});
translator.EnvironmentMapper.Mapping = new Dictionary <string, string>()
{
{ "Json", string.Empty },
{ "Regular", string.Empty },
{ "Namespace", "[DbSchema].[DbTable]." },
};
translator.DataModels.AddJsonModel($"{Directory.GetCurrentDirectory()}\\DataModel.json");
translator.DataModels.AddDictionaryModel(RegularModel.ModelConfiguration, "Regular");
translator.DataModels.DefaultNamespace = "Json";
ITransformationSchema schema = translator.CreateSchema(Example.Source);
if (translator.Errors.Count == 0)
{
string plantUmlResult = translator.Translate(schema, "PlantUML");
PlantUmlUrlConverter converter = new PlantUmlUrlConverter(plantUmlResult);
Process.Start("cmd.exe", $"/C start {converter.SVGUrl}");
string tsqlResult = translator.Translate(schema, "TSQL");
Debug.WriteLine(tsqlResult);
}
}
public string Translate(TranslateOptions parameters)
{
dataModelAggregator.DefaultNamespace = parameters.DefaultNamespace;
dataModelAggregator.DataModelsCollection.Clear();
dataModelAggregator.AddJsonModel(parameters.Model);
if (parameters.NamespaceMapping != null && parameters.NamespaceMapping != string.Empty)
{
// example mapping: "Json;Regular;Namespace=[DbSchema].[DbTable]."
foreach (string map in parameters.NamespaceMapping.Split(';'))
{
var sp = map.Split('=');
switch (sp.Length)
{
case 1:
this.mapper.Mapping.Add(sp[0], string.Empty);
break;
case 2:
this.mapper.Mapping.Add(sp[0], sp[1]);
break;
default:
throw new ArgumentException("Invalid format of namespace mapping argument");
}
}
}
else
{
this.mapper.Mapping.Add(parameters.DefaultNamespace, string.Empty);
}
ITransformationSchema schema = treeGenerator.BuildTransformationSchema(parameters.Input.OpenText().ReadToEnd());
schemaModifiersApplier.Process(schema);
ITargetRenderer target = targetRenderers.FirstOrDefault(tr => tr.Name.ToLower() == parameters.Target.ToLower());
if (target == null)
{
throw new ArgumentException("Unexpected target type.");
}
return(target.Render(schema));
}
/// <summary>
/// Performs processing of the schema.
/// </summary>
/// <param name="schema">The schema object to be processed.</param>
public void Process(ITransformationSchema schema)
{
foreach (ISchemaModifier mod in this._modifiers)
{
try
{
mod.Modify(schema);
}
catch (VtlOperatorError ex)
{
this._logger?.LogError(ex, ex.Message);
}
catch (Exception ex)
{
this._logger?.LogCritical(ex, ex.Message);
}
}
}
protected string TranslateVtl(string source, out List <Exception> errors)
{
ITransformationSchema schema = this.frontEnd.BuildTransformationSchema(source);
this.middleEnd.Process(schema);
ITargetRenderer tsqlRenderer = this.provider.GetTargetRenderer("TSQL");
string sql = tsqlRenderer.Render(schema);
errors = new List <Exception>();
foreach (ErrorCollector errCollector in this.errCollectorProvider.ErrorCollectors)
{
errors.AddRange(errCollector.Errors);
}
return(sql);
}
/// <summary>
/// Performs a type inference for the schema.
/// </summary>
/// <param name="schema">The schema to modify.</param>
public void Modify(ITransformationSchema schema)
{
foreach (IRuleset ruleset in schema.Rulesets)
{
foreach (IExpression expr in ruleset.RulesCollection)
{
this.Infer(expr);
}
ruleset.InferStructure();
}
foreach (AssignmentObject assignmentObject in schema.AssignmentObjects)
{
this.Infer(assignmentObject.Expression);
if (assignmentObject.IsPersistentAssignment)
{
this.ValidatePersistentAssignment(assignmentObject.Expression);
}
}
}
/// <summary>
/// Translates a given VTL 2.0 source code to target's language code whose target name is given.
/// </summary>
/// <param name="vtlSource">The VTL 2.0 source code.</param>
/// <param name="targetName">The name of a target.</param>
/// <returns></returns>
public string Translate(string vtlSource, string targetName)
{
ITransformationSchema schema = this.CreateSchema(vtlSource);
return(this.Translate(schema, targetName));
}
public static void Main(string[] args)
{
string source = Example.Source;
string connectionString = @"Server=...;Trusted_Connection=True;";
IServiceCollection services = new ServiceCollection();
services.AddVtlProcessing((configure) =>
{
configure.DataModels.DefaultNamespace = "Json";
configure.DataModels.AddSqlServerModel(connectionString);
configure.DataModels.AddJsonModel($"{Directory.GetCurrentDirectory()}\\DataModel.json"); // namespace name is in a json file
configure.DataModels.AddDictionaryModel((config) =>
{
config
.AddDataSet(
"R1",
(ComponentType.Identifier, BasicDataType.Integer, "Id1"),
(ComponentType.Identifier, BasicDataType.Integer, "Id2"),
(ComponentType.Measure, BasicDataType.Integer, "Me1"),
(ComponentType.Measure, BasicDataType.Integer, "Me2"),
(ComponentType.NonViralAttribute, BasicDataType.String, "At1"),
(ComponentType.ViralAttribute, BasicDataType.Integer, "At2")
)
.AddDataSet(
"R2",
(ComponentType.Identifier, BasicDataType.Integer, "Id1"),
(ComponentType.Measure, BasicDataType.String, "Me1"),
(ComponentType.Measure, BasicDataType.Integer, "Me2")
)
.AddDataSet(
"R_num",
(ComponentType.Identifier, BasicDataType.Integer, "Id1"),
(ComponentType.Measure, BasicDataType.Number, "Me2")
);
}
, "def_ns");
configure.EnvironmentMapper.Mapping = new Dictionary <string, string>()
{
{ "Json", string.Empty },
{ "Regular", string.Empty },
{ "Pivot", "[VtlProcessingTests].[Pivoting]." },
};
});
services.AddPlantUmlTarget((configure) =>
{
configure.AddDataStructureObject();
configure.UseArrowFirstToLast();
configure.ShowNumberLine();
configure.UseRuleExpressionsModel();
});
services.AddTsqlTarget((configure) =>
{
configure.AddComments();
});
services.AddLogging((configure) =>
{
configure.AddConsole();
configure.AddDebug();
configure.AddProvider(new ErrorCollectorProvider());
});
ServiceProvider provider = services.BuildServiceProvider();
ErrorCollectorProvider errColector = provider.GetService <ILoggerProvider>() as ErrorCollectorProvider;
ITransformationSchema schema = provider.GetFrontEnd().BuildTransformationSchema(source); // front-end
provider.GetMiddleEnd().Process(schema); // middle-end
bool areErrors = errColector.ErrorCollectors.Sum(counter => counter.Errors.Count) > 0;
// back-end:
ITargetRenderer plantUmlRenderer = provider.GetTargetRenderer("PlantUML");
string plantUmlResult = plantUmlRenderer.Render(schema);
PlantUmlUrlConverter converter = new PlantUmlUrlConverter(plantUmlResult);
if (!areErrors)
{
ITargetRenderer tsqlRenderer = provider.GetTargetRenderer("TSQL");
Process.Start("cmd.exe", $"/C start {converter.SVGUrl}");
FilesManager.ResultToFile(plantUmlResult, "result.plantuml");
FilesManager.ResultToFile(tsqlRenderer.Render(schema), "result.sql");
}
else
{
Debug.WriteLine($"\n\n{converter.SVGUrl}\n\n");
}
}
private void InitSchema(ref ITransformationSchema schema)
{
IDataStructure structureX = ModelResolvers.DsResolver();
structureX.DatasetName = "X";
IDataStructure structureY = ModelResolvers.DsResolver();
structureY.DatasetName = "Y";
IDataStructure constStructure = ModelResolvers.DsResolver();
constStructure.DatasetName = "const";
IExpression expr1_1 = TestExprFactory.GetExpression("get", ExpressionFactoryNameTarget.OperatorSymbol);
expr1_1.ParamSignature = "ds";
expr1_1.ExpressionText = "X";
expr1_1.Structure = structureX;
IExpression expr1_2 = TestExprFactory.GetExpression("get", ExpressionFactoryNameTarget.OperatorSymbol);
expr1_2.ParamSignature = "ds";
expr1_2.ExpressionText = "Y";
expr1_2.Structure = structureY;
IExpression expr2_1 = expr1_1; // X
IExpression expr2_2 = TestExprFactory.GetExpression("ref", ExpressionFactoryNameTarget.OperatorSymbol); // A
expr2_2.ParamSignature = "ds";
expr2_2.ExpressionText = "A";
expr2_2.Structure = structureY; // struktura Y
IExpression expr3_1 = expr2_2; // A
IExpression expr3_2 = TestExprFactory.GetExpression("const", ExpressionFactoryNameTarget.OperatorSymbol); // 2
expr3_2.ParamSignature = "ds";
expr3_2.ExpressionText = "2";
expr3_2.Structure = constStructure; // struktura const
IExpression expr4_1 = expr1_2; // Y
IExpression expr4_2 = TestExprFactory.GetExpression("ref", ExpressionFactoryNameTarget.OperatorSymbol); // B
expr4_2.ParamSignature = "ds";
expr4_2.ExpressionText = "B";
expr4_2.Structure = structureX; // struktura X
IExpression expr1 = TestExprFactory.GetExpression("+", ExpressionFactoryNameTarget.OperatorSymbol); // A := X + Y
expr1.ResultName = "A";
expr1.ExpressionText = "X + Y";
expr1.ParamSignature = "<root>";
expr1.OperandsCollection = new List <IExpression>()
{
expr1_1, expr1_2
};
expr1.Structure = structureY; // struktura Y
IExpression expr2 = TestExprFactory.GetExpression("-", ExpressionFactoryNameTarget.OperatorSymbol); // B := X - 2
expr2.ResultName = "B";
expr2.ExpressionText = "X - A";
expr2.ParamSignature = "<root>";
expr2.OperandsCollection = new List <IExpression>()
{
expr2_1, expr2_2
};
expr2.Structure = structureX; // struktura X
IExpression expr3 = TestExprFactory.GetExpression("*", ExpressionFactoryNameTarget.OperatorSymbol); // C <- A * 2
expr3.ResultName = "C";
expr3.ExpressionText = "A - 2";
expr3.ParamSignature = "<root>";
expr3.OperandsCollection = new List <IExpression>()
{
expr3_1, expr3_2
};
expr3.Structure = structureY; // strktura Y
IExpression expr4 = TestExprFactory.GetExpression("+", ExpressionFactoryNameTarget.OperatorSymbol); // D := Y + B
expr4.ResultName = "D";
expr4.ExpressionText = "Y + B";
expr4.ParamSignature = "<root>";
expr4.OperandsCollection = new List <IExpression>()
{
expr4_1, expr4_2
};
expr4.Structure = structureY; // struktura Y
schema = ModelResolvers.SchemaResolver();
schema.AssignmentObjects.Add(new AssignmentObject(schema, expr1, false, new string[0])); // A := X + Y
schema.AssignmentObjects.Add(new AssignmentObject(schema, expr2, false, new string[] { "A" })); // B := X - 2
schema.AssignmentObjects.Add(new AssignmentObject(schema, expr3, true, new string[] { "A" })); // C <- A * 2
schema.AssignmentObjects.Add(new AssignmentObject(schema, expr4, false, new string[] { "B" })); // D := Y + B
}
