public static async Task <string> CloseDataQC([ActivityTrigger] DataQCDataOpsCloseParameters parms, ILogger log) { log.LogInformation($"CloseDataQC: Starting"); DataOpParameters pipe = parms.Parameters; DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>(); DataQC qc = new DataQC(pipe.StorageAccount); await qc.CloseDataQC(qcParms.DataConnector, parms.Failures); log.LogInformation($"CloseDataQC: Complete"); return("Data QC Closed"); }
public static async Task <string> RunOrchestrator( [OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log) { string response = "OK"; List <DataOpParameters> pipelines = context.GetInput <List <DataOpParameters> >(); log.LogInformation($"RunOrchestrator: Number of pipelines: {pipelines.Count}."); foreach (var pipe in pipelines) { context.SetCustomStatus($"Starting pipe number {pipe.Id} with name {pipe.Name}"); if (pipe.Name == "CreateIndex") { response = await context.CallActivityAsync <string>("ManageDataOps_CreateIndex", pipe); } else if (pipe.Name == "DataQC") { List <QcResult> qcList = await context.CallActivityAsync <List <QcResult> >("ManageDataOps_InitDataQC", pipe); var tasks = new Task <List <int> > [qcList.Count]; for (int i = 0; i < qcList.Count; i++) { int qcId = qcList[i].Id; JObject pipeParm = JObject.Parse(pipe.JsonParameters); pipeParm["RuleId"] = qcId; pipe.JsonParameters = pipeParm.ToString(); tasks[i] = context.CallActivityAsync <List <int> >("ManageDataOps_DataQC", pipe); } await Task.WhenAll(tasks); List <RuleFailures> failures = new List <RuleFailures>(); for (int i = 0; i < qcList.Count; i++) { failures.Add(new RuleFailures { RuleId = qcList[i].Id, Failures = tasks[i].Result }); } DataQCDataOpsCloseParameters parms = new DataQCDataOpsCloseParameters() { Parameters = pipe, Failures = failures }; log.LogInformation($"RunOrchestrator: Ready to close data QC"); string stat = await context.CallActivityAsync <string>("ManageDataOps_CloseDataQC", parms); } else if (pipe.Name == "DataTransfer") { log.LogInformation($"Starting data transfer"); List <string> files = await context.CallActivityAsync <List <string> >("ManageDataOps_InitDataTransfer", pipe); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); if (parms.SourceType == "DataBase") { foreach (string file in files) { JObject pipeParm = JObject.Parse(pipe.JsonParameters); pipeParm["Table"] = file; pipe.JsonParameters = pipeParm.ToString(); string stat = await context.CallActivityAsync <string>("ManageDataOps_DeleteDataTransfer", pipe); } } foreach (string file in files) { JObject pipeParm = JObject.Parse(pipe.JsonParameters); pipeParm["Table"] = file; pipe.JsonParameters = pipeParm.ToString(); string stat = await context.CallActivityAsync <string>("ManageDataOps_DataTransfer", pipe); } } else if (pipe.Name == "Predictions") { log.LogInformation($"Starting Predictions"); List <PredictionCorrection> predictionList = await context.CallActivityAsync <List <PredictionCorrection> >("ManageDataOps_InitPredictions", pipe); var tasks = new Task <string> [predictionList.Count]; JObject pipeParm = JObject.Parse(pipe.JsonParameters); for (int i = 0; i < predictionList.Count; i++) { int id = predictionList[i].Id; pipeParm["PredictionId"] = id; pipe.JsonParameters = pipeParm.ToString(); string stat = await context.CallActivityAsync <string>("ManageDataOps_Prediction", pipe); } } else { log.LogInformation($"Artifact {pipe.Name} does not exist"); } context.SetCustomStatus($"Completed pipe number {pipe.Id} with name {pipe.Name}"); } log.LogInformation($"RunOrchestrator: All pipelines processed"); return(response); }