public static async Task <string> DeleteDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"DeleteDataTransfer: Starting deleting"); DataTransfer dt = new DataTransfer(pipe.StorageAccount); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); await dt.DeleteTable(parms.TargetName, parms.Table); log.LogInformation($"DeleteDataTransfer: Complete deleting {parms.Table}"); return($"DeleteDataTransfer Complete"); }
public static async Task <string> DataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"DataTransfer: Starting"); DataTransfer dt = new DataTransfer(pipe.StorageAccount); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); await dt.CopyFiles(parms); log.LogInformation($"DataTransfer: Complete copying {parms.Table}"); return($"OK"); }
public static async Task <List <string> > InitDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"InitDataTransfer: Starting"); DataTransfer dt = new DataTransfer(pipe.StorageAccount); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); List <string> files = await dt.GetFiles(parms.SourceName); log.LogInformation($"InitDataTransfer: Complete"); return(files); }
public static async Task <string> CloseDataQC([ActivityTrigger] DataQCDataOpsCloseParameters parms, ILogger log) { log.LogInformation($"CloseDataQC: Starting"); DataOpParameters pipe = parms.Parameters; DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>(); DataQC qc = new DataQC(pipe.StorageAccount); await qc.CloseDataQC(qcParms.DataConnector, parms.Failures); log.LogInformation($"CloseDataQC: Complete"); return("Data QC Closed"); }
public static async Task <List <QcResult> > InitDataQC([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"InitDataQC: Starting"); DataQC qc = new DataQC(pipe.StorageAccount); DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>(); List <QcResult> qcList = await qc.GetQCRules(qcParms); log.LogInformation($"InitDataQC: Number of QC rules are {qcList.Count}"); await qc.ClearQCFlags(qcParms.DataConnector); log.LogInformation($"InitDataQC: Complete"); return(qcList); }
public static async Task <List <PredictionCorrection> > InitPredictions([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"InitPredictions: Starting"); List <PredictionCorrection> predictionList = new List <PredictionCorrection>(); Predictions predictions = new Predictions(pipe.StorageAccount); PredictionParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <PredictionParameters>(); predictionList = await predictions.GetPredictions(parms.DataConnector); log.LogInformation($"Number of predictions are {predictionList.Count}"); log.LogInformation($"InitPredictions: Complete"); return(predictionList); }
public static async Task <string> DataPrediction([ActivityTrigger] DataOpParameters pipe, ILogger log) { try { log.LogInformation($"Prediction: Starting prediction"); Predictions predictions = new Predictions(pipe.StorageAccount); PredictionParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <PredictionParameters>(); log.LogInformation($"Prediction: Processing prediction id {parms.PredictionId}"); await predictions.ExecutePrediction(parms); log.LogInformation($"Prediction: Complete"); } catch (Exception ex) { log.LogInformation($"Prediction:Serious exception {ex}"); } return($"Prediction Rule {pipe.Id}"); }
public static async Task <List <int> > DataQC([ActivityTrigger] DataOpParameters pipe, ILogger log) { List <int> result = new List <int>(); try { log.LogInformation($"DataQC: Starting"); DataQC qc = new DataQC(pipe.StorageAccount); DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>(); result = await qc.ExecuteQcRule(qcParms); log.LogInformation($"DataQC: completed rule id {qcParms.RuleId} with result = {result}"); } catch (Exception ex) { log.LogInformation($"InitDataQC:Serious exception {ex}"); } return(result); }
public static async Task <string> CreateIndex([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"CreateIndex: Starting"); CreateIndexParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <CreateIndexParameters>(); try { Sources sr = new Sources(pipe.StorageAccount); ConnectParameters target = await sr.GetSourceParameters(parms.TargetName); ConnectParameters source = await sr.GetSourceParameters(parms.SourceName); Indexer index = new Indexer(pipe.StorageAccount); int parentNodes = await index.Initialize(target, source, parms.Taxonomy, parms.Filter); List <ParentIndexNodes> nodes = await index.IndexParent(parentNodes, parms.Filter); for (int j = 0; j < nodes.Count; j++) { ParentIndexNodes node = nodes[j]; for (int i = 0; i < node.NodeCount; i++) { await index.IndexChildren(j, i, node.ParentNodeId); } } index.CloseIndex(); } catch (Exception ex) { log.LogInformation($"CreateIndex: Serious exception {ex}"); } log.LogInformation($"CreateIndex: Complete"); return($"Hello {pipe.Name}!"); }