public static async Task <string> DeleteDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"DeleteDataTransfer: Starting deleting");
DataTransfer dt = new DataTransfer(pipe.StorageAccount);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
await dt.DeleteTable(parms.TargetName, parms.Table);
log.LogInformation($"DeleteDataTransfer: Complete deleting {parms.Table}");
return($"DeleteDataTransfer Complete");
}
public static async Task <string> DataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"DataTransfer: Starting");
DataTransfer dt = new DataTransfer(pipe.StorageAccount);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
await dt.CopyFiles(parms);
log.LogInformation($"DataTransfer: Complete copying {parms.Table}");
return($"OK");
}
public static async Task <List <string> > InitDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"InitDataTransfer: Starting");
DataTransfer dt = new DataTransfer(pipe.StorageAccount);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
List <string> files = await dt.GetFiles(parms.SourceName);
log.LogInformation($"InitDataTransfer: Complete");
return(files);
}
public static async Task <string> CloseDataQC([ActivityTrigger] DataQCDataOpsCloseParameters parms, ILogger log)
{
log.LogInformation($"CloseDataQC: Starting");
DataOpParameters pipe = parms.Parameters;
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
DataQC qc = new DataQC(pipe.StorageAccount);
await qc.CloseDataQC(qcParms.DataConnector, parms.Failures);
log.LogInformation($"CloseDataQC: Complete");
return("Data QC Closed");
}
public static async Task <List <QcResult> > InitDataQC([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"InitDataQC: Starting");
DataQC qc = new DataQC(pipe.StorageAccount);
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
List <QcResult> qcList = await qc.GetQCRules(qcParms);
log.LogInformation($"InitDataQC: Number of QC rules are {qcList.Count}");
await qc.ClearQCFlags(qcParms.DataConnector);
log.LogInformation($"InitDataQC: Complete");
return(qcList);
}
public static async Task <List <PredictionCorrection> > InitPredictions([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"InitPredictions: Starting");
List <PredictionCorrection> predictionList = new List <PredictionCorrection>();
Predictions predictions = new Predictions(pipe.StorageAccount);
PredictionParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <PredictionParameters>();
predictionList = await predictions.GetPredictions(parms.DataConnector);
log.LogInformation($"Number of predictions are {predictionList.Count}");
log.LogInformation($"InitPredictions: Complete");
return(predictionList);
}
public static async Task <string> DataPrediction([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
try
{
log.LogInformation($"Prediction: Starting prediction");
Predictions predictions = new Predictions(pipe.StorageAccount);
PredictionParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <PredictionParameters>();
log.LogInformation($"Prediction: Processing prediction id {parms.PredictionId}");
await predictions.ExecutePrediction(parms);
log.LogInformation($"Prediction: Complete");
}
catch (Exception ex)
{
log.LogInformation($"Prediction:Serious exception {ex}");
}
return($"Prediction Rule {pipe.Id}");
}
public static async Task <List <int> > DataQC([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
List <int> result = new List <int>();
try
{
log.LogInformation($"DataQC: Starting");
DataQC qc = new DataQC(pipe.StorageAccount);
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
result = await qc.ExecuteQcRule(qcParms);
log.LogInformation($"DataQC: completed rule id {qcParms.RuleId} with result = {result}");
}
catch (Exception ex)
{
log.LogInformation($"InitDataQC:Serious exception {ex}");
}
return(result);
}
public static async Task <string> CreateIndex([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"CreateIndex: Starting");
CreateIndexParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <CreateIndexParameters>();
try
{
Sources sr = new Sources(pipe.StorageAccount);
ConnectParameters target = await sr.GetSourceParameters(parms.TargetName);
ConnectParameters source = await sr.GetSourceParameters(parms.SourceName);
Indexer index = new Indexer(pipe.StorageAccount);
int parentNodes = await index.Initialize(target, source, parms.Taxonomy, parms.Filter);
List <ParentIndexNodes> nodes = await index.IndexParent(parentNodes, parms.Filter);
for (int j = 0; j < nodes.Count; j++)
{
ParentIndexNodes node = nodes[j];
for (int i = 0; i < node.NodeCount; i++)
{
await index.IndexChildren(j, i, node.ParentNodeId);
}
}
index.CloseIndex();
}
catch (Exception ex)
{
log.LogInformation($"CreateIndex: Serious exception {ex}");
}
log.LogInformation($"CreateIndex: Complete");
return($"Hello {pipe.Name}!");
}
