/// <summary> /// Transform the data from SQL to a restful API via Data Bus /// </summary> /// <param name="bindingExecution"></param> /// <param name="binding"></param> /// <param name="entity"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <long> TransformDataAsync( BindingExecution bindingExecution, Binding binding, Entity entity, CancellationToken cancellationToken) { bindingExecution.CheckWhetherArgumentIsNull(nameof(bindingExecution)); binding.CheckWhetherArgumentIsNull(nameof(binding)); entity.CheckWhetherArgumentIsNull(nameof(entity)); try { SwitchLogLevel(await this.GetPluginLogLevelSystemAttributeValue() ?? LogLevel.Warning); this.LogDebug($"Entering HierarchicalDataTransformer.TransformDataAsync(BindingId = {binding.Id})", bindingExecution); HierarchicalConfiguration config = this.GetConfiguration(binding, bindingExecution, entity); this.LogDebug($"Plugin Configuration: {config}", bindingExecution); JobData jobData = await this.GetJobData(binding, bindingExecution, entity); this.LogDebug($"JobData: {Serialize(jobData)}", bindingExecution); return(await this.RunDatabusAsync(config, jobData, bindingExecution, cancellationToken)); } catch (Exception e) { this.LogError($"HierarchicalDataTransformer.TransformDataAsync threw an exception: {e}", e, bindingExecution); throw; } }
/// <summary> /// Gets configuration from config file (json) unless specified in attributes on the binding or entity /// </summary> /// <param name="binding"></param> /// <param name="bindingExecution"></param> /// <param name="entity"></param> /// <returns></returns> private HierarchicalConfiguration GetConfiguration(Binding binding, BindingExecution bindingExecution, Entity entity) { string directoryName = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); if (directoryName == null) { throw new InvalidOperationException("Could not find plugin configuration file base path."); } string fullPath = Path.Combine(directoryName, Resources.DefaultConfigFileName); string json = File.ReadAllText(fullPath); dynamic deserialized = JsonConvert.DeserializeObject(json); dynamic databusConfiguration = deserialized.DatabusConfiguration; var queryConfig = new QueryConfig { ConnectionString = this.BuildConnectionString(binding), Url = this.BuildUrl(entity) ?? databusConfiguration.Url, MaximumEntitiesToLoad = this.GetAttributeInt(binding.AttributeValues, AttributeNames.MaxEntitiesToLoad) ?? databusConfiguration.MaximumEntitiesToLoad, EntitiesPerBatch = this.GetAttributeInt(binding.AttributeValues, AttributeNames.EntitiesPerBatch) ?? databusConfiguration.EntitiesPerBatch, EntitiesPerUploadFile = this.GetAttributeInt(binding.AttributeValues, AttributeNames.EntitiesPerUploadFile) ?? databusConfiguration.EntitiesPerUploadFile, LocalSaveFolder = this.BuildLocalSaveFolder( binding.AttributeValues.GetAttributeTextValue(AttributeNames.LocalSaveFolder), binding, bindingExecution) ?? this.BuildLocalSaveFolder(Convert.ToString(databusConfiguration.LocalSaveFolder), binding, bindingExecution), WriteTemporaryFilesToDisk = this.GetAttributeBool(binding.AttributeValues, AttributeNames.WriteTempFilesToDisk) ?? databusConfiguration.WriteTemporaryFilesToDisk, WriteDetailedTemporaryFilesToDisk = this.GetAttributeBool(binding.AttributeValues, AttributeNames.DetailedTempFiles) ?? databusConfiguration.WriteDetailedTemporaryFilesToDisk, CompressFiles = this.GetAttributeBool(binding.AttributeValues, AttributeNames.CompressFiles) ?? databusConfiguration.CompressFiles, UploadToUrl = this.GetAttributeBool(binding.AttributeValues, AttributeNames.UploadToUrl) ?? databusConfiguration.UploadToUrl, UrlMethod = this.GetHtmlMethod(entity.AttributeValues?.GetAttributeTextValue(AttributeNames.HttpMethod)) ?? this.GetHtmlMethod(Convert.ToString(databusConfiguration.UrlMethod)) ?? HttpMethod.Post }; this.CreateLocalSaveFolderIfNotExists(queryConfig); var hierarchicalConfig = new HierarchicalConfiguration { ClientSpecificConfiguration = this.GetClientSpecificConfiguration(entity, deserialized.ClientSpecificConfigurations), DatabusConfiguration = queryConfig }; return(hierarchicalConfig); }
/// <summary> /// Execute DataBus with the given configuration and job data /// </summary> /// <param name="config"></param> /// <param name="jobData"></param> /// <param name="bindingExecution"></param> /// <param name="cancellationToken"></param> private async Task <long> RunDatabusAsync( HierarchicalConfiguration config, JobData jobData, BindingExecution bindingExecution, CancellationToken cancellationToken) { var container = new UnityContainer(); // TODO - need to eventually remove anything UPMC-specific from this plugin. UpmcSpecificConfiguration upmcSpecificConfiguration = (UpmcSpecificConfiguration)config.ClientSpecificConfiguration; if (upmcSpecificConfiguration != null) { container.RegisterInstance <IHttpRequestInterceptor>( new UpmcHmacAuthorizationRequestInterceptor(upmcSpecificConfiguration.AppId, upmcSpecificConfiguration.AppSecret, upmcSpecificConfiguration.TenantSecret)); } var rowCounter = new RowCounterBatchEventsLogger(this.loggingRepository, bindingExecution); ILogger databusLogger = CreateLogger <DatabusRunner>(); container.RegisterInstance(databusLogger); container.RegisterInstance <IBatchEventsLogger>(rowCounter); var jobEventsLogger = new JobEventsLogger(this.loggingRepository, bindingExecution); container.RegisterInstance <IJobEventsLogger>(jobEventsLogger); container.RegisterInstance <IQuerySqlLogger>(new QuerySqlLogger(this.loggingRepository, bindingExecution)); container.RegisterInstance <IHttpResponseLogger>(new MyHttpResponseLogger(this.loggingRepository, bindingExecution)); var job = new Job { Config = config.DatabusConfiguration, Data = jobData }; this.LogDebug($"Executing DatabusRunner.RunRestApiPipeline with:\n\tcontainer: {Serialize(container)}\n\tjob: {Serialize(job)}"); try { await this.runner.RunRestApiPipelineAsync(container, job, cancellationToken); } catch (Exception e) { this.LogError($"Databus threw an error: {e}", e); throw; } SetupSerilogLogger(); // re-setup logger as Databus is closing it this.LogDebug($"Databus execution complete. Processed { jobEventsLogger.NumberOfEntities } records."); return(jobEventsLogger.NumberOfEntities); }