private List <IncrementalColumn> GetIncrementalConfigurations(Binding binding, BindingExecution bindingExecution, Field[] sourceEntityFields)
{
this.LogDebug("Checking Incremental Configurations", bindingExecution);
var incrementalColumns = new List <IncrementalColumn>();
if (bindingExecution.LoadType != BindingLoadType.Incremental)
{
return(incrementalColumns);
}
if (binding.IncrementalConfigurations.Count == 0)
{
return(incrementalColumns);
}
foreach (IncrementalConfiguration incrementalConfiguration in binding.IncrementalConfigurations)
{
this.LogDebug($"Processing incremental configuration: {Serialize(incrementalConfiguration)} \n\tfor binding: {Serialize(bindingExecution)}");
IncrementalValue incrementalValue;
if (bindingExecution.BatchExecution.IncrementalStartDateTime.HasValue)
{
incrementalValue = new IncrementalValue
{
DestinationBindingId = bindingExecution.BindingId,
LastMaxIncrementalDate = bindingExecution.BatchExecution.IncrementalStartDateTime
};
}
else
{
using (IProcessingContextWrapper processingContextWrapper = this.processingContextWrapperFactory.CreateProcessingContextWrapper())
{
incrementalValue = processingContextWrapper.GetIncrementalValue(incrementalConfiguration);
}
}
this.LogDebug($"Found incrementalValue: {Serialize(incrementalValue)}");
if (incrementalValue?.LastMaxIncrementalDate == null)
{
continue;
}
incrementalColumns.Add(new IncrementalColumn
{
Name = incrementalConfiguration.IncrementalColumnName,
Operator = IncrementalOperator.GreaterThanOrEqualTo,
Type = sourceEntityFields.First(f => f.FieldName == incrementalConfiguration.IncrementalColumnName).DataType,
Value = Serialize(incrementalValue.LastMaxIncrementalDate.Value).Replace("\"", string.Empty)
});
}
this.LogDebug($"processed the following incrementalColumns: {Serialize(incrementalColumns)}");
return(incrementalColumns);
}
protected override async Task <long> LoadStagingEntityAsync(
BindingExecution bindingExecution,
Binding binding,
Entity entity,
CancellationToken cancellationToken)
{
bindingExecution.CheckWhetherArgumentIsNull(nameof(bindingExecution));
binding.CheckWhetherArgumentIsNull(nameof(bindingExecution));
entity.CheckWhetherArgumentIsNull(nameof(entity));
using (IProcessingContextWrapper processingContextWrapper =
this.processingContextWrapperFactory.CreateProcessingContextWrapper())
{
string pathToRExe = processingContextWrapper
.GetSystemAttribute(AttributeName.PathToRExecutable)?.AttributeValueText;
string pathToRModelFolder = processingContextWrapper
.GetSystemAttribute(AttributeName.PathToRModelFolder)?.AttributeValueText;
string sourceConnectionDatabase = binding.SourceConnection.Database;
var rScriptSourceEntityInfos = new List <RScriptSourceEntityInfo>();
foreach (int sourceEntityId in binding.SourcedByEntities.Select(e => e.SourceEntityId))
{
Entity sourceEntity = await this.metadataServiceClient.GetEntityAsync(sourceEntityId);
rScriptSourceEntityInfos.Add(new RScriptSourceEntityInfo
{
DatabaseName = sourceEntity.DatabaseName,
EntityName = sourceEntity.EntityName,
SchemaName = sourceEntity.SchemaName,
});
// HACK: sourceConnectionDatabase is always set to master which doesn't work when R is trying to update the table
sourceConnectionDatabase = sourceEntity.DatabaseName;
}
var rScriptParameters = new RScriptParameters
{
BindingScript = binding.Script,
Script = binding.UserDefinedSql,
CompletedSuccessfullyText = CompletedSuccessfullyText,
DestinationDatabaseName = entity.Connection.Database,
DestinationServer = entity.Connection.Server,
DestinationSystemTypeCode = entity.Connection.DataSystemTypeCode,
PathToRModelFolder = pathToRModelFolder,
SourceConnectionDatabase = sourceConnectionDatabase,
SourceDataSystemTypeCode = binding.SourceConnection.DataSystemTypeCode,
SourceServer = binding.SourceConnection.Server,
SourceEntities = rScriptSourceEntityInfos
};
var augmentedRScript = new ScriptParser().GetAugmentedRScript(rScriptParameters);
return(await this.StartScriptR(pathToRExe, bindingExecution, augmentedRScript));
}
}