public static async Task <string> CloseDataQC([ActivityTrigger] DataQCDataOpsCloseParameters parms, ILogger log)
{
log.LogInformation($"CloseDataQC: Starting");
DataOpParameters pipe = parms.Parameters;
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
DataQC qc = new DataQC(pipe.StorageAccount);
await qc.CloseDataQC(qcParms.DataConnector, parms.Failures);
log.LogInformation($"CloseDataQC: Complete");
return("Data QC Closed");
}
public static async Task <string> RunOrchestrator(
[OrchestrationTrigger] IDurableOrchestrationContext context,
ILogger log)
{
string response = "OK";
List <DataOpParameters> pipelines = context.GetInput <List <DataOpParameters> >();
log.LogInformation($"RunOrchestrator: Number of pipelines: {pipelines.Count}.");
foreach (var pipe in pipelines)
{
context.SetCustomStatus($"Starting pipe number {pipe.Id} with name {pipe.Name}");
if (pipe.Name == "CreateIndex")
{
response = await context.CallActivityAsync <string>("ManageDataOps_CreateIndex", pipe);
}
else if (pipe.Name == "DataQC")
{
List <QcResult> qcList = await context.CallActivityAsync <List <QcResult> >("ManageDataOps_InitDataQC", pipe);
var tasks = new Task <List <int> > [qcList.Count];
for (int i = 0; i < qcList.Count; i++)
{
int qcId = qcList[i].Id;
JObject pipeParm = JObject.Parse(pipe.JsonParameters);
pipeParm["RuleId"] = qcId;
pipe.JsonParameters = pipeParm.ToString();
tasks[i] = context.CallActivityAsync <List <int> >("ManageDataOps_DataQC", pipe);
}
await Task.WhenAll(tasks);
List <RuleFailures> failures = new List <RuleFailures>();
for (int i = 0; i < qcList.Count; i++)
{
failures.Add(new RuleFailures {
RuleId = qcList[i].Id, Failures = tasks[i].Result
});
}
DataQCDataOpsCloseParameters parms = new DataQCDataOpsCloseParameters()
{
Parameters = pipe,
Failures = failures
};
log.LogInformation($"RunOrchestrator: Ready to close data QC");
string stat = await context.CallActivityAsync <string>("ManageDataOps_CloseDataQC", parms);
}
else if (pipe.Name == "DataTransfer")
{
log.LogInformation($"Starting data transfer");
List <string> files = await context.CallActivityAsync <List <string> >("ManageDataOps_InitDataTransfer", pipe);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
if (parms.SourceType == "DataBase")
{
foreach (string file in files)
{
JObject pipeParm = JObject.Parse(pipe.JsonParameters);
pipeParm["Table"] = file;
pipe.JsonParameters = pipeParm.ToString();
string stat = await context.CallActivityAsync <string>("ManageDataOps_DeleteDataTransfer", pipe);
}
}
foreach (string file in files)
{
JObject pipeParm = JObject.Parse(pipe.JsonParameters);
pipeParm["Table"] = file;
pipe.JsonParameters = pipeParm.ToString();
string stat = await context.CallActivityAsync <string>("ManageDataOps_DataTransfer", pipe);
}
}
else if (pipe.Name == "Predictions")
{
log.LogInformation($"Starting Predictions");
List <PredictionCorrection> predictionList = await context.CallActivityAsync <List <PredictionCorrection> >("ManageDataOps_InitPredictions", pipe);
var tasks = new Task <string> [predictionList.Count];
JObject pipeParm = JObject.Parse(pipe.JsonParameters);
for (int i = 0; i < predictionList.Count; i++)
{
int id = predictionList[i].Id;
pipeParm["PredictionId"] = id;
pipe.JsonParameters = pipeParm.ToString();
string stat = await context.CallActivityAsync <string>("ManageDataOps_Prediction", pipe);
}
}
else
{
log.LogInformation($"Artifact {pipe.Name} does not exist");
}
context.SetCustomStatus($"Completed pipe number {pipe.Id} with name {pipe.Name}");
}
log.LogInformation($"RunOrchestrator: All pipelines processed");
return(response);
}
