/// <summary>
/// Transform the data from SQL to a restful API via Data Bus
/// </summary>
/// <param name="bindingExecution"></param>
/// <param name="binding"></param>
/// <param name="entity"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <long> TransformDataAsync(
BindingExecution bindingExecution,
Binding binding,
Entity entity,
CancellationToken cancellationToken)
{
bindingExecution.CheckWhetherArgumentIsNull(nameof(bindingExecution));
binding.CheckWhetherArgumentIsNull(nameof(binding));
entity.CheckWhetherArgumentIsNull(nameof(entity));
try
{
SwitchLogLevel(await this.GetPluginLogLevelSystemAttributeValue() ?? LogLevel.Warning);
this.LogDebug($"Entering HierarchicalDataTransformer.TransformDataAsync(BindingId = {binding.Id})", bindingExecution);
HierarchicalConfiguration config = this.GetConfiguration(binding, bindingExecution, entity);
this.LogDebug($"Plugin Configuration: {config}", bindingExecution);
JobData jobData = await this.GetJobData(binding, bindingExecution, entity);
this.LogDebug($"JobData: {Serialize(jobData)}", bindingExecution);
return(await this.RunDatabusAsync(config, jobData, bindingExecution, cancellationToken));
}
catch (Exception e)
{
this.LogError($"HierarchicalDataTransformer.TransformDataAsync threw an exception: {e}", e, bindingExecution);
throw;
}
}
protected override async Task <long> LoadStagingEntityAsync(
BindingExecution bindingExecution,
Binding binding,
Entity entity,
CancellationToken cancellationToken)
{
bindingExecution.CheckWhetherArgumentIsNull(nameof(bindingExecution));
binding.CheckWhetherArgumentIsNull(nameof(bindingExecution));
entity.CheckWhetherArgumentIsNull(nameof(entity));
using (IProcessingContextWrapper processingContextWrapper =
this.processingContextWrapperFactory.CreateProcessingContextWrapper())
{
string pathToRExe = processingContextWrapper
.GetSystemAttribute(AttributeName.PathToRExecutable)?.AttributeValueText;
string pathToRModelFolder = processingContextWrapper
.GetSystemAttribute(AttributeName.PathToRModelFolder)?.AttributeValueText;
string sourceConnectionDatabase = binding.SourceConnection.Database;
var rScriptSourceEntityInfos = new List <RScriptSourceEntityInfo>();
foreach (int sourceEntityId in binding.SourcedByEntities.Select(e => e.SourceEntityId))
{
Entity sourceEntity = await this.metadataServiceClient.GetEntityAsync(sourceEntityId);
rScriptSourceEntityInfos.Add(new RScriptSourceEntityInfo
{
DatabaseName = sourceEntity.DatabaseName,
EntityName = sourceEntity.EntityName,
SchemaName = sourceEntity.SchemaName,
});
// HACK: sourceConnectionDatabase is always set to master which doesn't work when R is trying to update the table
sourceConnectionDatabase = sourceEntity.DatabaseName;
}
var rScriptParameters = new RScriptParameters
{
BindingScript = binding.Script,
Script = binding.UserDefinedSql,
CompletedSuccessfullyText = CompletedSuccessfullyText,
DestinationDatabaseName = entity.Connection.Database,
DestinationServer = entity.Connection.Server,
DestinationSystemTypeCode = entity.Connection.DataSystemTypeCode,
PathToRModelFolder = pathToRModelFolder,
SourceConnectionDatabase = sourceConnectionDatabase,
SourceDataSystemTypeCode = binding.SourceConnection.DataSystemTypeCode,
SourceServer = binding.SourceConnection.Server,
SourceEntities = rScriptSourceEntityInfos
};
var augmentedRScript = new ScriptParser().GetAugmentedRScript(rScriptParameters);
return(await this.StartScriptR(pathToRExe, bindingExecution, augmentedRScript));
}
}