private PredictionResult ProcessPrediction(QcRuleSetup qcSetup, string predictionURL, RuleModel rule)
{
PredictionResult result = new PredictionResult();
try
{
var jsonString = JsonConvert.SerializeObject(qcSetup);
var content = new StringContent(jsonString, Encoding.UTF8, "application/json");
HttpResponseMessage response = Client.PostAsync(predictionURL, content).Result;
if (!response.IsSuccessStatusCode)
{
result.Status = "Server error";
return(result);
}
using (HttpContent respContent = response.Content)
{
string tr = respContent.ReadAsStringAsync().Result;
result = JsonConvert.DeserializeObject <PredictionResult>(tr);
}
}
catch (Exception Ex)
{
//logger.LogWarning("ProcessDataObject: Problems with URL");
}
return(result);
}
private string ProcessQcRule(QcRuleSetup qcSetup, RuleModel rule)
{
string returnResult = "Passed";
var jsonString = JsonConvert.SerializeObject(qcSetup);
var content = new StringContent(jsonString, Encoding.UTF8, "application/json");
try
{
using (HttpClient client = new HttpClient())
{
using (HttpResponseMessage response = client.PostAsync(rule.RuleFunction, content).Result)
{
using (HttpContent respContent = response.Content)
{
var tr = respContent.ReadAsStringAsync().Result;
var azureResponse = JsonConvert.DeserializeObject(tr);
returnResult = azureResponse.ToString();
}
}
}
}
catch (Exception ex)
{
Exception error = new Exception("DataQcWithProgressBar: Problems with URL: ", ex);
throw error;
}
return(returnResult);
}
public static PredictionResult PredictDepthUsingIDW(QcRuleSetup qcSetup, DbUtilities dbConn)
{
double? depth = null;
PredictionResult result = new PredictionResult
{
Status = "Failed"
};
DataTable nb = RuleMethodUtilities.GetNeighbors(dbConn, qcSetup);
if (nb != null)
{
depth = RuleMethodUtilities.CalculateDepthUsingIdw(nb, qcSetup);
}
if (depth != null)
{
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject dataObject = JObject.Parse(qcSetup.DataObject);
dataObject[rule.DataAttribute] = depth;
string remark = dataObject["REMARK"] + $";{rule.DataAttribute} has been predicted by QCEngine;";
dataObject["REMARK"] = remark;
result.DataObject = dataObject.ToString();
result.DataType = rule.DataType;
result.SaveType = "Update";
result.IndexId = qcSetup.IndexId;
result.Status = "Passed";
}
return(result);
}
public static string Completeness(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject dataObject = JObject.Parse(qcSetup.DataObject);
JToken value = dataObject.GetValue(rule.DataAttribute);
if (value == null)
{
//log.Warning($"Attribute is Null");
}
else
{
string strValue = value.ToString();
if (string.IsNullOrWhiteSpace(strValue))
{
returnStatus = "Failed";
}
else
{
double number;
bool canConvert = double.TryParse(strValue, out number);
if (canConvert)
{
if (number == -99999)
{
returnStatus = "Failed";
}
}
}
}
return(returnStatus);
}
public static PredictionResult PredictLogDepthAttributes(QcRuleSetup qcSetup, DbUtilities dbConn)
{
PredictionResult result = new PredictionResult
{
Status = "Failed"
};
JObject dataObject = JObject.Parse(qcSetup.DataObject);
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
string jsonLog = RuleMethodUtilities.GetLogCurveDepths(dbConn, qcSetup.DataObject);
if (!string.IsNullOrEmpty(jsonLog))
{
JObject logObject = JObject.Parse(jsonLog);
string attribute = rule.DataAttribute;
dataObject[attribute] = logObject[attribute];
string remark = dataObject["REMARK"] + $";{attribute} was calculated from curve array;";
dataObject["REMARK"] = remark;
result.DataObject = dataObject.ToString();
result.DataType = rule.DataType;
result.SaveType = "Update";
result.IndexId = qcSetup.IndexId;
result.Status = "Passed";
}
return(result);
}
public static string Consistency(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject dataObject = JObject.Parse(qcSetup.DataObject);
JToken value = dataObject.GetValue(rule.DataAttribute);
string strValue = value.ToString();
if (Common.CompletenessCheck(strValue) == "Passed")
{
string dataType = rule.DataType;
IndexModel index = Task.Run(() => indexData.GetIndex(qcSetup.IndexId, qcSetup.DataConnector)).GetAwaiter().GetResult();
if (index != null)
{
string query = " where " + index.DataKey;
DataAccessDef dataAccessDef = accessDefs.First(x => x.DataType == dataType);
string sql = dataAccessDef.Select + query;
IADODataAccess consistencyConn = new ADODataAccess();
DataTable ct = consistencyConn.GetDataTable(sql, qcSetup.ConsistencyConnectorString);
Dictionary <string, string> columnTypes = ct.GetColumnTypes();
if (ct.Rows.Count > 0)
{
string strRefValue = ct.Rows[0][rule.DataAttribute].ToString();
string valueType = columnTypes[rule.DataAttribute];
if (Common.CompletenessCheck(strRefValue) == "Passed")
{
returnStatus = RuleMethodUtilities.ConsistencyCheck(strValue, strRefValue, valueType);
}
}
}
}
return(returnStatus);
}
public static string CurveSpikes(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
string error = "";
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject dataObject = JObject.Parse(qcSetup.DataObject);
double nullValue = Common.GetLogNullValue(dataObject.ToString());
CurveSpikeParameters spikeParams = new CurveSpikeParameters()
{
WindowSize = 5,
SeveritySize = 4
};
if (!string.IsNullOrEmpty(rule.RuleParameters))
{
try
{
spikeParams = JsonConvert.DeserializeObject <CurveSpikeParameters>(rule.RuleParameters);
if (spikeParams.WindowSize == 0)
{
spikeParams.WindowSize = 5;
}
if (spikeParams.SeveritySize == 0)
{
spikeParams.SeveritySize = 4;
}
}
catch (Exception ex)
{
error = $"Bad Json, {ex}";
}
}
spikeParams.NullValue = nullValue;
List <double> measuredValues = new List <double>();
try
{
JToken value = dataObject.GetValue("MEASURED_VALUE");
string strValue = value.ToString();
if (!string.IsNullOrEmpty(strValue))
{
measuredValues = strValue.Split(',').Select(double.Parse).ToList();
}
}
catch (Exception ex)
{
error = $"Problems with the log arrays, {ex}";
}
bool spike = RuleMethodUtilities.CurveHasSpikes(measuredValues, spikeParams);
if (spike)
{
returnStatus = "Failed";
}
return(returnStatus);
}
private async Task <List <int> > QualityCheckDataType(DbUtilities dbConn, RuleModel rule, ConnectParameters connector)
{
List <int> failedObjects = new List <int>();
QcRuleSetup qcSetup = new QcRuleSetup();
qcSetup.Database = connector.Catalog;
qcSetup.DatabasePassword = connector.Password;
qcSetup.DatabaseServer = connector.DatabaseServer;
qcSetup.DatabaseUser = connector.User;
qcSetup.DataConnector = connector.ConnectionString;
string ruleFilter = rule.RuleFilter;
string jsonRules = JsonConvert.SerializeObject(rule);
qcSetup.RuleObject = jsonRules;
bool externalQcMethod = rule.RuleFunction.StartsWith("http");
DataTable indexTable = manageQCFlags.GetIndexTable();
if (rule.RuleFunction == "Uniqueness")
{
CalculateKey(rule, indexTable);
}
if (rule.RuleFunction == "Consistency")
{
qcSetup.ConsistencyConnectorString = await GetConsistencySource(rule.RuleParameters);
}
foreach (DataRow idxRow in indexTable.Rows)
{
string jsonData = idxRow["JSONDATAOBJECT"].ToString();
if (!string.IsNullOrEmpty(jsonData))
{
qcSetup.IndexId = Convert.ToInt32(idxRow["INDEXID"]);
qcSetup.IndexNode = idxRow["TextIndexNode"].ToString();
qcSetup.DataObject = jsonData;
string result = "Passed";
if (!Filter(jsonData, ruleFilter))
{
if (externalQcMethod)
{
result = ProcessQcRule(qcSetup, rule);
}
else
{
Type type = typeof(QCMethods);
MethodInfo info = type.GetMethod(rule.RuleFunction);
result = (string)info.Invoke(null, new object[] { qcSetup, indexTable, _accessDefs, _indexData });
}
if (result == "Failed")
{
failedObjects.Add(qcSetup.IndexId);
}
}
}
}
return(failedObjects);
}
public static PredictionResult DeleteDataObject(QcRuleSetup qcSetup, DbUtilities dbConn)
{
PredictionResult result = new PredictionResult();
result.SaveType = "Delete";
result.Status = "Passed";
result.IndexId = qcSetup.IndexId;
return(result);
}
public static DataTable GetNeighbors(DbUtilities dbConn, QcRuleSetup qcSetup)
{
int indexId = qcSetup.IndexId;
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
string failRule = rule.FailRule;
string path = $"$.{rule.DataAttribute}";
string strProcedure = $"EXEC spGetNeighborsNoFailuresDepth {indexId}, '%{failRule}%', '{path}'";
DataTable nb = dbConn.GetDataTable(strProcedure, "");
return(nb);
}
public static string Entirety(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
string indexNode = qcSetup.IndexNode;
string dataName = rule.RuleParameters;
IEnumerable <IndexModel> indexModels = Task.Run(() => indexData.GetChildrenWithName(qcSetup.DataConnector, indexNode, dataName)).GetAwaiter().GetResult();
if (indexModels.Count() == 0)
{
returnStatus = "Failed";
}
return(returnStatus);
}
public static string StringLength(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Failed";
string error = "";
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject dataObject = JObject.Parse(qcSetup.DataObject);
JToken value = dataObject.GetValue(rule.DataAttribute);
StringLengthParameters stringParams = new StringLengthParameters()
{
Min = 20,
Max = 20
};
if (!string.IsNullOrEmpty(rule.RuleParameters))
{
try
{
stringParams = JsonConvert.DeserializeObject <StringLengthParameters>(rule.RuleParameters);
if (stringParams.Min == 0)
{
stringParams.Min = 20;
}
if (stringParams.Max == 0)
{
stringParams.Max = stringParams.Min;
}
}
catch (Exception ex)
{
error = $"Bad parameter Json, {ex}";
}
}
string strValue = value.ToString();
strValue = strValue.Trim();
int stringLength = strValue.Length;
if (stringLength >= stringParams.Min & stringLength <= stringParams.Max)
{
returnStatus = "Passed";
}
else
{
returnStatus = "Failed";
}
return(returnStatus);
}
private async Task MakePredictions(RuleModel rule, ConnectParameters connector)
{
QcRuleSetup qcSetup = new QcRuleSetup();
qcSetup.Database = connector.Catalog;
qcSetup.DatabasePassword = connector.Password;
qcSetup.DatabaseServer = connector.DatabaseServer;
qcSetup.DatabaseUser = connector.User;
string jsonRules = JsonConvert.SerializeObject(rule);
qcSetup.RuleObject = jsonRules;
string predictionURL = rule.RuleFunction;
bool externalQcMethod = rule.RuleFunction.StartsWith("http");
indexTable = manageIndexTable.GetIndexTable();
foreach (DataRow idxRow in indexTable.Rows)
{
string jsonData = idxRow["JSONDATAOBJECT"].ToString();
if (!string.IsNullOrEmpty(jsonData))
{
qcSetup.IndexId = Convert.ToInt32(idxRow["INDEXID"]);
qcSetup.IndexNode = idxRow["TextIndexNode"].ToString();
qcSetup.DataObject = jsonData;
PredictionResult result;
if (externalQcMethod)
{
result = ProcessPrediction(qcSetup, predictionURL, rule);
if (result.Status == "Server error")
{
break;
}
}
else
{
Type type = typeof(PredictionMethods);
MethodInfo info = type.GetMethod(rule.RuleFunction);
result = (PredictionResult)info.Invoke(null, new object[] { qcSetup, _dbConn });
}
await ProcessResult(result, rule);
}
}
}
public static string ValidityRange(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
bool canConvert;
double minRange;
double maxRange;
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject parameterObject = JObject.Parse(rule.RuleParameters);
JToken jMinRangeValue = parameterObject.GetValue("MinRange");
canConvert = double.TryParse(jMinRangeValue.ToString(), out minRange);
if (!canConvert)
{
minRange = -99999.0;
}
JToken jMaxRangeValue = parameterObject.GetValue("MaxRange");
canConvert = double.TryParse(jMaxRangeValue.ToString(), out maxRange);
if (!canConvert)
{
maxRange = 99999.0;
}
JObject dataObject = JObject.Parse(qcSetup.DataObject);
JToken value = dataObject.GetValue(rule.DataAttribute);
double? number = value.GetNumberFromJToken();
if (number != null)
{
if (number >= minRange & number <= maxRange)
{
returnStatus = "Passed";
}
else
{
returnStatus = "Failed";
}
}
return(returnStatus);
}
public static PredictionResult PredictMissingDataObjects(QcRuleSetup qcSetup, DbUtilities dbConn)
{
PredictionResult result = new PredictionResult
{
Status = "Failed"
};
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
string rulePar = rule.RuleParameters;
JObject ruleParObject = JObject.Parse(rulePar);
string dataType = ruleParObject["DataType"].ToString();
if (string.IsNullOrEmpty(rulePar))
{
throw new NullReferenceException("Rule parameter is null.");
}
string emptyJson = RuleMethodUtilities.GetJsonForMissingDataObject(rulePar, dbConn);
if (emptyJson == "Error")
{
throw new NullReferenceException("Could not create an empty json data object, maybe you are missing Datatype in parameters");
}
string json = RuleMethodUtilities.PopulateJsonForMissingDataObject(rulePar, emptyJson, qcSetup.DataObject);
if (json == "Error")
{
throw new NullReferenceException("Could not create an json data object, problems with keys in parameters");
}
json = RuleMethodUtilities.AddDefaultsForMissingDataObjects(rulePar, json);
if (json == "Error")
{
throw new NullReferenceException("Could not create an json data object, problems with defaults in parameters");
}
result.DataObject = json;
result.DataType = dataType;
result.SaveType = "Insert";
result.IndexId = qcSetup.IndexId;
result.Status = "Passed";
return(result);
}
public static string Uniqueness(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
string query = $"INDEXID = '{qcSetup.IndexId}'";
DataRow[] idxRows = dt.Select(query);
if (idxRows.Length == 1)
{
string key = idxRows[0]["UNIQKEY"].ToString();
if (!string.IsNullOrEmpty(key))
{
query = $"UNIQKEY = '{key}'";
DataRow[] dtRows = dt.Select(query);
if (dtRows.Length > 1)
{
returnStatus = "Failed";
}
}
}
return(returnStatus);
}
public static string IsNumber(QcRuleSetup qcSetup, DataTable dt, List <DataAccessDef> accessDefs, IIndexDBAccess indexData)
{
string returnStatus = "Passed";
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
JObject dataObject = JObject.Parse(qcSetup.DataObject);
JToken value = dataObject.GetValue(rule.DataAttribute);
if (value == null)
{
//log.Warning($"Attribute is Null");
}
else
{
string strValue = value.ToString();
bool isValid = false;
isValid = isNumberTest.IsMatch(strValue);
if (!isValid)
{
returnStatus = "Failed";
}
}
return(returnStatus);
}
public static PredictionResult PredictDominantLithology(QcRuleSetup qcSetup, DbUtilities dbConn)
{
PredictionResult result = new PredictionResult
{
Status = "Failed"
};
JObject dataObject = JObject.Parse(qcSetup.DataObject);
string uwi = dataObject["UWI"].ToString();
string curveName = "GR";
JToken value = dataObject.GetValue("PICK_DEPTH");
double? pickDepth = value.GetNumberFromJToken();
if (pickDepth == null || pickDepth == -99999.0)
{
result.Status = "Failed";
}
else
{
string select = "SELECT JSONDATAOBJECT from pdo_qc_index";
string query = " where IndexNode = '/'";
DataTable idx = dbConn.GetDataTable(select, query);
if (idx.Rows.Count == 1)
{
string jsondata = idx.Rows[0]["JSONDATAOBJECT"].ToString();
ConnectParameters source = JsonConvert.DeserializeObject <ConnectParameters>(jsondata);
List <DataAccessDef> accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(source.DataAccessDefinition);
DataAccessDef logType = accessDefs.First(x => x.DataType == "Log");
select = logType.Select;
query = $" where CURVE_ID = '{curveName}' and UWI = '{uwi}'";
DataTable lc = dbConn.GetDataTable(select, query);
if (lc.Rows.Count == 1)
{
double logNullValue = Common.GetDataRowNumber(lc.Rows[0], "NULL_REPRESENTATION");
DataAccessDef logCurvedType = accessDefs.First(x => x.DataType == "LogCurve");
select = logCurvedType.Select;
query = $" where CURVE_ID = '{curveName}' and UWI = '{uwi}'";
DataTable lg = dbConn.GetDataTable(select, query);
DataTable sortedCurve = RuleMethodUtilities.GetSortedLogCurve(lg, uwi);
if (sortedCurve.Rows.Count > 0)
{
int rowNumber = RuleMethodUtilities.GetRowNumberForPickDepth(sortedCurve, (double)pickDepth);
double?smoothLogValue = RuleMethodUtilities.GetSmoothLogValue(sortedCurve, logNullValue, rowNumber);
string rock;
if (smoothLogValue >= 0 & smoothLogValue < 10)
{
rock = "Salt";
}
if (smoothLogValue >= 10 & smoothLogValue < 20)
{
rock = "Limestone";
}
else if (smoothLogValue >= 20 & smoothLogValue < 55)
{
rock = "Sandstone";
}
else if (smoothLogValue >= 55 & smoothLogValue < 150)
{
rock = "Shale";
}
else
{
rock = "Unknown";
}
dataObject["DOMINANT_LITHOLOGY"] = rock;
string remark = dataObject["REMARK"] + $";Pick depth has been predicted by QCEngine";
dataObject["REMARK"] = remark;
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
result.DataObject = dataObject.ToString();
result.DataType = rule.DataType;
result.SaveType = "Update";
result.IndexId = qcSetup.IndexId;
result.Status = "Passed";
}
}
}
}
return(result);
}
public static PredictionResult PredictFormationOrder(QcRuleSetup qcSetup, DbUtilities dbConn)
{
List <StratUnits> inv = new List <StratUnits>();
PredictionResult result = new PredictionResult
{
Status = "Failed"
};
string formation;
string tempTable = "#MinMaxAllFormationPick";
DataTable idx = new DataTable();
try
{
string select = "select * from #MinMaxAllFormationPick";
string query = "";
idx = dbConn.GetDataTable(select, query);
}
catch (Exception ex)
{
if (ex.InnerException.Message.Contains("Invalid object name"))
{
string select = $"EXEC spGetMinMaxAllFormationPick";
string query = "";
idx = dbConn.GetDataTable(select, query);
string SQLCreateTempTable = Common.GetCreateSQLFromDataTable(tempTable, idx);
dbConn.SQLExecute(SQLCreateTempTable);
dbConn.BulkCopy(idx, tempTable);
}
else
{
throw;
}
}
JObject dataObject = JObject.Parse(qcSetup.DataObject);
formation = dataObject["STRAT_UNIT_ID"].ToString();
string tmpFormation = Common.FixAposInStrings(formation);
string condition = $"STRAT_UNIT_ID = '{tmpFormation}'";
var rows = idx.Select(condition);
if (rows.Length == 1)
{
RuleModel rule = JsonConvert.DeserializeObject <RuleModel>(qcSetup.RuleObject);
int age = Convert.ToInt32(rows[0]["AGE"]);
dataObject[rule.DataAttribute] = age;
string remark = dataObject["REMARK"] + $";{rule.DataAttribute} has been predicted by QCEngine;";
dataObject["REMARK"] = remark;
result.DataObject = dataObject.ToString();
result.DataType = rule.DataType;
result.SaveType = "Update";
result.IndexId = qcSetup.IndexId;
result.Status = "Passed";
}
else if (rows.Length > 1)
{
throw new Exception("PredictFormationOrder: Multiple occurences of formation not allowed");
}
return(result);
}
public static double?CalculateDepthUsingIdw(DataTable nb, QcRuleSetup qcSetup)
{
double?depth = null;
double prevLat = -99999.0;
double prevLon = -99999.0;
for (int k = 0; k < nb.Rows.Count; k++)
{
double currLat = Common.GetDataRowNumber(nb.Rows[k], "LATITUDE");
double currLon = Common.GetDataRowNumber(nb.Rows[k], "LONGITUDE");
double distance = 99999.0;
if (nb.Columns.Contains("DISTANCE"))
{
distance = Convert.ToDouble(nb.Rows[k]["DISTANCE"]);
}
Boolean deleteRow = false;
if (currLat == -99999.0 && currLon == -99999.0)
{
deleteRow = true;
}
if (Math.Abs(prevLat - currLat) < 0.0001 & Math.Abs(prevLon - currLon) < 0.0001)
{
deleteRow = true;
}
if (distance < 0.1)
{
deleteRow = true;
}
if (deleteRow)
{
nb.Rows[k].Delete();
}
prevLat = currLat;
prevLon = currLon;
}
nb.AcceptChanges();
JObject ruleObject = JObject.Parse(qcSetup.RuleObject);
string depthAttribute = ruleObject["DataAttribute"].ToString();
List <IdwPoint> idwPoints = new List <IdwPoint>();
foreach (DataRow row in nb.Rows)
{
double distance = Common.GetDataRowNumber(row, "DISTANCE");
double number = Common.GetDataRowNumber(row, "DEPTH");
if (number != -99999.0)
{
idwPoints.Add(new IdwPoint
{
Distance = distance,
Depth = number
});
}
}
if (idwPoints.Count > 2)
{
depth = IdwCalculate(idwPoints);
}
return(depth);
}
