/// <summary>
/// Transform the data from SQL to a restful API via Data Bus
/// </summary>
/// <param name="bindingExecution"></param>
/// <param name="binding"></param>
/// <param name="entity"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <long> TransformDataAsync(
BindingExecution bindingExecution,
Binding binding,
Entity entity,
CancellationToken cancellationToken)
{
bindingExecution.CheckWhetherArgumentIsNull(nameof(bindingExecution));
binding.CheckWhetherArgumentIsNull(nameof(binding));
entity.CheckWhetherArgumentIsNull(nameof(entity));
try
{
SwitchLogLevel(await this.GetPluginLogLevelSystemAttributeValue() ?? LogLevel.Warning);
this.LogDebug($"Entering HierarchicalDataTransformer.TransformDataAsync(BindingId = {binding.Id})", bindingExecution);
HierarchicalConfiguration config = this.GetConfiguration(binding, bindingExecution, entity);
this.LogDebug($"Plugin Configuration: {config}", bindingExecution);
JobData jobData = await this.GetJobData(binding, bindingExecution, entity);
this.LogDebug($"JobData: {Serialize(jobData)}", bindingExecution);
return(await this.RunDatabusAsync(config, jobData, bindingExecution, cancellationToken));
}
catch (Exception e)
{
this.LogError($"HierarchicalDataTransformer.TransformDataAsync threw an exception: {e}", e, bindingExecution);
throw;
}
}
/// <summary>
/// Gets configuration from config file (json) unless specified in attributes on the binding or entity
/// </summary>
/// <param name="binding"></param>
/// <param name="bindingExecution"></param>
/// <param name="entity"></param>
/// <returns></returns>
private HierarchicalConfiguration GetConfiguration(Binding binding, BindingExecution bindingExecution, Entity entity)
{
string directoryName = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
if (directoryName == null)
{
throw new InvalidOperationException("Could not find plugin configuration file base path.");
}
string fullPath = Path.Combine(directoryName, Resources.DefaultConfigFileName);
string json = File.ReadAllText(fullPath);
dynamic deserialized = JsonConvert.DeserializeObject(json);
dynamic databusConfiguration = deserialized.DatabusConfiguration;
var queryConfig = new QueryConfig
{
ConnectionString = this.BuildConnectionString(binding),
Url = this.BuildUrl(entity) ?? databusConfiguration.Url,
MaximumEntitiesToLoad = this.GetAttributeInt(binding.AttributeValues, AttributeNames.MaxEntitiesToLoad)
?? databusConfiguration.MaximumEntitiesToLoad,
EntitiesPerBatch = this.GetAttributeInt(binding.AttributeValues, AttributeNames.EntitiesPerBatch)
?? databusConfiguration.EntitiesPerBatch,
EntitiesPerUploadFile = this.GetAttributeInt(binding.AttributeValues, AttributeNames.EntitiesPerUploadFile)
?? databusConfiguration.EntitiesPerUploadFile,
LocalSaveFolder = this.BuildLocalSaveFolder(
binding.AttributeValues.GetAttributeTextValue(AttributeNames.LocalSaveFolder),
binding,
bindingExecution)
?? this.BuildLocalSaveFolder(Convert.ToString(databusConfiguration.LocalSaveFolder), binding, bindingExecution),
WriteTemporaryFilesToDisk = this.GetAttributeBool(binding.AttributeValues, AttributeNames.WriteTempFilesToDisk)
?? databusConfiguration.WriteTemporaryFilesToDisk,
WriteDetailedTemporaryFilesToDisk = this.GetAttributeBool(binding.AttributeValues, AttributeNames.DetailedTempFiles)
?? databusConfiguration.WriteDetailedTemporaryFilesToDisk,
CompressFiles = this.GetAttributeBool(binding.AttributeValues, AttributeNames.CompressFiles)
?? databusConfiguration.CompressFiles,
UploadToUrl = this.GetAttributeBool(binding.AttributeValues, AttributeNames.UploadToUrl)
?? databusConfiguration.UploadToUrl,
UrlMethod = this.GetHtmlMethod(entity.AttributeValues?.GetAttributeTextValue(AttributeNames.HttpMethod))
?? this.GetHtmlMethod(Convert.ToString(databusConfiguration.UrlMethod))
?? HttpMethod.Post
};
this.CreateLocalSaveFolderIfNotExists(queryConfig);
var hierarchicalConfig = new HierarchicalConfiguration
{
ClientSpecificConfiguration = this.GetClientSpecificConfiguration(entity, deserialized.ClientSpecificConfigurations),
DatabusConfiguration = queryConfig
};
return(hierarchicalConfig);
}
/// <summary>
/// Execute DataBus with the given configuration and job data
/// </summary>
/// <param name="config"></param>
/// <param name="jobData"></param>
/// <param name="bindingExecution"></param>
/// <param name="cancellationToken"></param>
private async Task <long> RunDatabusAsync(
HierarchicalConfiguration config,
JobData jobData,
BindingExecution bindingExecution,
CancellationToken cancellationToken)
{
var container = new UnityContainer();
// TODO - need to eventually remove anything UPMC-specific from this plugin.
UpmcSpecificConfiguration upmcSpecificConfiguration = (UpmcSpecificConfiguration)config.ClientSpecificConfiguration;
if (upmcSpecificConfiguration != null)
{
container.RegisterInstance <IHttpRequestInterceptor>(
new UpmcHmacAuthorizationRequestInterceptor(upmcSpecificConfiguration.AppId, upmcSpecificConfiguration.AppSecret, upmcSpecificConfiguration.TenantSecret));
}
var rowCounter = new RowCounterBatchEventsLogger(this.loggingRepository, bindingExecution);
ILogger databusLogger = CreateLogger <DatabusRunner>();
container.RegisterInstance(databusLogger);
container.RegisterInstance <IBatchEventsLogger>(rowCounter);
var jobEventsLogger = new JobEventsLogger(this.loggingRepository, bindingExecution);
container.RegisterInstance <IJobEventsLogger>(jobEventsLogger);
container.RegisterInstance <IQuerySqlLogger>(new QuerySqlLogger(this.loggingRepository, bindingExecution));
container.RegisterInstance <IHttpResponseLogger>(new MyHttpResponseLogger(this.loggingRepository, bindingExecution));
var job = new Job {
Config = config.DatabusConfiguration, Data = jobData
};
this.LogDebug($"Executing DatabusRunner.RunRestApiPipeline with:\n\tcontainer: {Serialize(container)}\n\tjob: {Serialize(job)}");
try
{
await this.runner.RunRestApiPipelineAsync(container, job, cancellationToken);
}
catch (Exception e)
{
this.LogError($"Databus threw an error: {e}", e);
throw;
}
SetupSerilogLogger(); // re-setup logger as Databus is closing it
this.LogDebug($"Databus execution complete. Processed { jobEventsLogger.NumberOfEntities } records.");
return(jobEventsLogger.NumberOfEntities);
}