public static async Task <IActionResult> DataQcResults(
[HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("GetDataQcResults: Starting.");
string jsonResult = "OK";
try
{
string storageAccount = Common.Helpers.Common.GetStorageKey(req);
string source = Common.Helpers.Common.GetQueryString(req, "name");
DataQC qc = new DataQC(storageAccount);
DataQCParameters qcParms = new DataQCParameters();
qcParms.DataConnector = source;
List <QcResult> qcResults = new List <QcResult>();
qcResults = await qc.GetQCRules(qcParms);
jsonResult = JsonConvert.SerializeObject(qcResults, Formatting.Indented);
}
catch (Exception ex)
{
log.LogError($"GetDataQcResults: Error getting QCResults: {ex}");
return(new BadRequestObjectResult($"Error getting QCResults: {ex}"));
}
log.LogInformation("GetDataQcResults: Complete.");
return(new OkObjectResult(jsonResult));
}
public async Task <List <int> > ExecuteQcRule(DataQCParameters qcParms)
{
try
{
ConnectParameters connector = await GetConnector(qcParms.DataConnector);
RuleModel rule = await GetRuleAndFunctionInfo(qcParms.DataConnector, qcParms.RuleId);
string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
_accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson);
DbUtilities dbConn = new DbUtilities();
dbConn.OpenConnection(connector);
manageQCFlags = new ManageIndexTable(_accessDefs, connector.ConnectionString, rule.DataType);
List <int> failedObjects = await QualityCheckDataType(dbConn, rule, connector);
dbConn.CloseConnection();
return(failedObjects);
}
catch (Exception ex)
{
Exception error = new Exception($"DataQc: Could process rule {qcParms.RuleId}, {ex}");
throw error;
}
}
public async Task <ActionResult <string> > ExecuteRule(DataQCParameters qcParams)
{
string result = "[]";
try
{
if (qcParams == null)
{
return(BadRequest());
}
string tmpConnString = Request.Headers["AzureStorageConnection"];
DataQC qc = new DataQC(tmpConnString);
List <int> failedObjects = new List <int>();
failedObjects = await qc.ExecuteQcRule(qcParams);
RuleFailures ruleFailures = new RuleFailures()
{
RuleId = qcParams.RuleId, Failures = failedObjects
};
result = JsonConvert.SerializeObject(ruleFailures);
}
catch (Exception ex)
{
return(BadRequest(ex.ToString()));
}
return(result);
}
public static async Task <string> CloseDataQC([ActivityTrigger] DataQCDataOpsCloseParameters parms, ILogger log)
{
log.LogInformation($"CloseDataQC: Starting");
DataOpParameters pipe = parms.Parameters;
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
DataQC qc = new DataQC(pipe.StorageAccount);
await qc.CloseDataQC(qcParms.DataConnector, parms.Failures);
log.LogInformation($"CloseDataQC: Complete");
return("Data QC Closed");
}
public static async Task <List <QcResult> > InitDataQC([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"InitDataQC: Starting");
DataQC qc = new DataQC(pipe.StorageAccount);
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
List <QcResult> qcList = await qc.GetQCRules(qcParms);
log.LogInformation($"InitDataQC: Number of QC rules are {qcList.Count}");
await qc.ClearQCFlags(qcParms.DataConnector);
log.LogInformation($"InitDataQC: Complete");
return(qcList);
}
public async Task <List <QcResult> > GetQCRules(DataQCParameters qcParms)
{
List <QcResult> qcResult = new List <QcResult>();
ConnectParameters connector = await GetConnector(qcParms.DataConnector);
RuleManagement rules = new RuleManagement(_azureConnectionString);
IndexAccess idxAccess = new IndexAccess();
string jsonString = await rules.GetActiveQCRules(connector.SourceName);
qcResult = JsonConvert.DeserializeObject <List <QcResult> >(jsonString);
foreach (QcResult qcItem in qcResult)
{
string query = $" where QC_STRING like '%{qcItem.RuleKey};%'";
qcItem.Failures = idxAccess.IndexCountByQuery(query, connector.ConnectionString);
}
return(qcResult);
}
public async Task <ActionResult <List <QcResult> > > Get(string source)
{
List <QcResult> results = new List <QcResult>();
try
{
string tmpConnString = Request.Headers["AzureStorageConnection"];
DataQC qc = new DataQC(tmpConnString);
DataQCParameters qcParms = new DataQCParameters();
qcParms.DataConnector = source;
//qcResults = await qc.GetQCRules(qcParms);
results = await qc.GetResults(source);
}
catch (Exception ex)
{
return(BadRequest($"{ex}"));
}
return(results);
}
public static async Task <List <int> > DataQC([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
List <int> result = new List <int>();
try
{
log.LogInformation($"DataQC: Starting");
DataQC qc = new DataQC(pipe.StorageAccount);
DataQCParameters qcParms = JObject.Parse(pipe.JsonParameters).ToObject <DataQCParameters>();
result = await qc.ExecuteQcRule(qcParms);
log.LogInformation($"DataQC: completed rule id {qcParms.RuleId} with result = {result}");
}
catch (Exception ex)
{
log.LogInformation($"InitDataQC:Serious exception {ex}");
}
return(result);
}
public Task ProcessQCRule(DataQCParameters qcParams)
{
throw new NotImplementedException();
}
public async Task <ActionResult <string> > CloseRulesExecution(string source, DataQCParameters qcParams)
{
try
{
if (qcParams == null)
{
return(BadRequest());
}
string tmpConnString = Request.Headers["AzureStorageConnection"];
DataQC qc = new DataQC(tmpConnString);
await qc.CloseDataQC(source, qcParams.Failures);
}
catch (Exception ex)
{
return(BadRequest(ex.ToString()));
}
return(Ok($"OK"));
}