public async Task <string> GetIndexData(string sourceName)
{
string result = "";
ConnectParameters connector = await Common.GetConnectParameters(azureConnectionString, sourceName);
if (connector.SourceType == "DataBase")
{
string jsonConnectDef = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
connector.DataAccessDefinition = jsonConnectDef;
}
else
{
Exception error = new Exception($"RuleManagement: data source must be a Database type");
throw error;
}
_dbConn.OpenConnection(connector);
string strProcedure = $"EXEC spGetNumberOfDescendants '/', 1";
string query = "";
DataTable qc = _dbConn.GetDataTable(strProcedure, query);
List <DmsIndex> index = ProcessAllChildren(qc);
result = JsonConvert.SerializeObject(index, Formatting.Indented);
_dbConn.CloseConnection();
return(result);
}
private async Task CreatePPDMModel(DataModelParameters dmParameters, ConnectParameters connector)
{
try
{
string sql = await _fileStorage.ReadFile(dmParameters.FileShare, dmParameters.FileName);
if (string.IsNullOrEmpty(sql))
{
Exception error = new Exception($"Empty data from {dmParameters.FileName}");
throw error;
}
else
{
DbUtilities dbConn = new DbUtilities();
dbConn.OpenConnection(connector);
dbConn.SQLExecute(sql);
dbConn.CloseConnection();
}
}
catch (Exception ex)
{
Exception error = new Exception("Create PPDM Model Error: ", ex);
throw error;
}
}
public async Task <string> GetPrediction(string name)
{
string result = "";
string ruleName = name + ".json";
result = await _fileStorage.ReadFile(ruleShare, ruleName);
if (string.IsNullOrEmpty(result))
{
Exception error = new Exception($"Empty data from {ruleName}");
throw error;
}
return(result);
}
public async Task <int> Initialize(ConnectParameters target, ConnectParameters source, string taxonomyFile, string filter)
{
string jsonTaxonomy = await _fileStorage.ReadFile("taxonomy", taxonomyFile);
int parentNodes = 0;
if (source.SourceType == "DataBase")
{
iBuilder = new IndexBuilder(new DBDataAccess());
}
else
{
source.ConnectionString = _azureConnectionString;
if (source.DataType == "Logs")
{
iBuilder = new IndexBuilder(new LASDataAccess(_fileStorage));
}
else
{
iBuilder = new IndexBuilder(new CSVDataAccess(_fileStorage));
}
}
iBuilder.InitializeIndex(target, source, jsonTaxonomy);
iBuilder.CreateRoot(source);
parentNodes = iBuilder.JsonIndexArray.Count;
return(parentNodes);
}
public async Task <ActionResult <string> > GetPipeline(string name)
{
SetStorageAccount();
string dataOpsFile = await fileStorageService.ReadFile(fileShare, name);
return(dataOpsFile);
}
public async Task ExecutePrediction(PredictionParameters parms)
{
string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
_accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson);
ConnectParameters connector = await GetConnector(parms.DataConnector);
databaseConnectionString = connector.ConnectionString;
_dbConn.OpenConnection(connector);
string sourceConnector = GetSource();
if (parms.DataConnector == sourceConnector)
{
syncPredictions = true;
}
else
{
syncPredictions = false;
}
RuleManagement rules = new RuleManagement(_azureConnectionString);
RuleModel rule = await rules.GetRuleAndFunction(parms.DataConnector, parms.PredictionId);
manageIndexTable = new ManageIndexTable(_accessDefs, connector.ConnectionString, rule.DataType, rule.FailRule);
manageIndexTable.InitQCFlags(false);
await MakePredictions(rule, connector);
_dbConn.CloseConnection();
manageIndexTable.SaveQCFlags();
}
public async Task <List <int> > ExecuteQcRule(DataQCParameters qcParms)
{
try
{
ConnectParameters connector = await GetConnector(qcParms.DataConnector);
RuleModel rule = await GetRuleAndFunctionInfo(qcParms.DataConnector, qcParms.RuleId);
string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
_accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson);
DbUtilities dbConn = new DbUtilities();
dbConn.OpenConnection(connector);
manageQCFlags = new ManageIndexTable(_accessDefs, connector.ConnectionString, rule.DataType);
List <int> failedObjects = await QualityCheckDataType(dbConn, rule, connector);
dbConn.CloseConnection();
return(failedObjects);
}
catch (Exception ex)
{
Exception error = new Exception($"DataQc: Could process rule {qcParms.RuleId}, {ex}");
throw error;
}
}
public async Task <DataTable> GetLASWellHeaders(ConnectParameters source, ConnectParameters target)
{
string accessJson = await fileStorageService.ReadFile("connectdefinition", "PPDMDataAccess.json");
_dataDef = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson);
//lasAccessJson = JObject.Parse(await fileStorageService.ReadFile("connectdefinition", "LASDataAccess.json"));
string lasMappingsStr = await fileStorageService.ReadFile("connectdefinition", "LASDataAccess.json");
lasMappings = JsonConvert.DeserializeObject <LASMappings>(lasMappingsStr);
List <string> files = new List <string>();
files = await GetLASFileNames(source.Catalog);
DataAccessDef dataType = _dataDef.First(x => x.DataType == "WellBore");
string select = dataType.Select;
string query = $" where 0 = 1";
_dbConn.OpenConnection(target);
DataTable dt = _dbConn.GetDataTable(select, query);
foreach (string file in files)
{
LASSections ls = await GetLASSections(source.Catalog, file);
lasSections.Add(ls);
GetVersionInfo(ls.versionInfo);
string json = GetHeaderInfo(ls.wellInfo);
DataRow row = dt.NewRow();
JObject jo = JObject.Parse(json);
foreach (JProperty property in jo.Properties())
{
string strValue = property.Value.ToString();
if (!string.IsNullOrEmpty(strValue))
{
if (row.Table.Columns.Contains(property.Name))
{
row[property.Name] = property.Value;
}
}
}
dt.Rows.Add(row);
}
return(dt);
}
public async Task <ConnectParameters> GetSourceParameters(string name)
{
if (String.IsNullOrEmpty(name))
{
Exception error = new Exception($"Sources: Source name is not set");
throw error;
}
var connector = _sourceData.GetSource(name);
if (connector.SourceType == "DataBase")
{
string jsonConnectDef = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
connector.DataAccessDefinition = jsonConnectDef;
}
return(connector);
}
public async Task CopyFiles(TransferParameters parms)
{
try
{
ConnectParameters sourceConnector = await Common.GetConnectParameters(_azureConnectionString, parms.SourceName);
ConnectParameters targetConnector = await Common.GetConnectParameters(_azureConnectionString, parms.TargetName);
string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
string referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json");
targetConnector.DataAccessDefinition = dataAccessDefinition;
if (sourceConnector.SourceType == "DataBase")
{
DatabaseLoader dl = new DatabaseLoader();
dl.CopyTable(parms, sourceConnector.ConnectionString, targetConnector, referenceJson);
}
else if (sourceConnector.SourceType == "File")
{
if (sourceConnector.DataType == "Logs")
{
LASLoader ls = new LASLoader(_fileStorage);
await ls.LoadLASFile(sourceConnector, targetConnector, parms.Table, referenceJson);
}
else
{
CSVLoader cl = new CSVLoader(_fileStorage);
await cl.LoadCSVFile(sourceConnector, targetConnector, parms.Table);
}
}
else
{
Exception error = new Exception($"DataTransfer: Not a valid source type for {sourceConnector.SourceName}");
throw error;
}
}
catch (Exception ex)
{
Exception error = new Exception($"DataTransfer: Problems transfer files/tables, {ex}");
throw error;
}
}
public async Task <List <PipeLine> > GetDataOpsPipe(string name)
{
List <PipeLine> dataOps = new List <PipeLine>();
List <string> result = await _fileStorage.ListFiles("dataops");
string dataOpsFile = await _fileStorage.ReadFile(fileShare, name);
dataOps = JsonConvert.DeserializeObject <List <PipeLine> >(dataOpsFile);
return(dataOps);
}
public async Task GetTransferConnector(string message)
{
try
{
TransferParameters transParms = JsonConvert.DeserializeObject <TransferParameters>(message);
_targetConnector = GetConnectionString(transParms.TargetName);
string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
_targetConnector.DataAccessDefinition = dataAccessDefinition;
target = _targetConnector.ConnectionString;
_logger.LogInformation($"Target connect string: {target}");
ConnectParameters sourceConnector = GetConnectionString(transParms.SourceName);
source = sourceConnector.ConnectionString;
_logger.LogInformation($"Source connect string: {source}");
transferQuery = transParms.TransferQuery;
queryType = transParms.QueryType;
_referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json");
}
catch (Exception ex)
{
_logger.LogInformation($"Error getting connector info {ex.ToString()}");
}
}
public async Task <string> GetAttributeInfo(string sourceName, string dataType)
{
string json = "";
ConnectParameters connector = await Common.GetConnectParameters(azureConnectionString, sourceName);
string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
List <DataAccessDef> accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson);
DataAccessDef dataAccess = accessDefs.First(x => x.DataType == dataType);
string sql = dataAccess.Select;
string table = Common.GetTable(sql);
string query = $" where 0 = 1";
DbUtilities dbConn = new DbUtilities();
dbConn.OpenWithConnectionString(connector.ConnectionString);
DataTable dt = dbConn.GetDataTable(sql, query);
AttributeInfo attributeInfo = new AttributeInfo();
attributeInfo.DataAttributes = dt.GetColumnTypes();
json = JsonConvert.SerializeObject(attributeInfo);
return(json);
}
public async Task <DataTable> GetCSVTable(ConnectParameters source, ConnectParameters target, string indexDataType)
{
DataTable result = new DataTable();
if (dt.Rows.Count == 0)
{
DateTime timeStart = DateTime.Now;
DateTime timeEnd;
TimeSpan diff;
connectionString = target.ConnectionString;
dt = new DataTable();
string accessJson = await fileStorageService.ReadFile("connectdefinition", "PPDMDataAccess.json");
_dataDef = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson);
string referenceJson = await fileStorageService.ReadFile("connectdefinition", "PPDMReferenceTables.json");
_references = JsonConvert.DeserializeObject <List <ReferenceTable> >(referenceJson);
string csvJson = await fileStorageService.ReadFile("connectdefinition", "CSVDataAccess.json");
_csvDef = JsonConvert.DeserializeObject <List <CSVAccessDef> >(csvJson);
//Console.WriteLine("start reading csv file");
string csvText = await fileStorageService.ReadFile(source.Catalog, source.FileName);
timeEnd = DateTime.Now;
diff = timeEnd - timeStart;
//Console.WriteLine($"Time span, read all definitions files: {diff}");
string dataType = source.DataType.Remove(source.DataType.Length - 1, 1);
dataAccess = _dataDef.First(x => x.DataType == dataType);
dt = NewDataTable(dataType);
tableDataType = dataType;
int headerLines = 1;
int wellCounter = 0;
CSVAccessDef csvAccess = _csvDef.First(x => x.DataType == dataType);
Dictionary <string, int> attributes = csvAccess.Mappings.ToDictionary();
Dictionary <string, string> constants = csvAccess.Constants.ToStringDictionary();
byte[] byteArray = Encoding.ASCII.GetBytes(csvText);
MemoryStream csvStream = new MemoryStream(byteArray);
using (TextFieldParser csvParser = new TextFieldParser(csvStream))
{
csvParser.CommentTokens = new string[] { "#" };
csvParser.SetDelimiters(new string[] { "," });
csvParser.HasFieldsEnclosedInQuotes = true;
for (int i = 0; i < headerLines; i++)
{
string line = csvParser.ReadLine();
}
while (!csvParser.EndOfData)
{
wellCounter++;
if ((wellCounter % 1000) == 0)
{
var tmp = wellCounter;
}
string[] fields = csvParser.ReadFields();
string key = GetDataKey(fields, attributes);
string duplicateKey = GetDuplicateKey(fields, attributes);
if (string.IsNullOrEmpty(key))
{
//Console.WriteLine($"Well {wellCounter} has an empty key");
}
else
{
if (duplicates.ContainsKey(duplicateKey))
{
DataRow[] rows = dt.Select(key);
rows[0] = InsertCSVRow(rows[0], attributes, fields, constants);
rows[0]["ROW_CHANGED_DATE"] = DateTime.Now.ToString("yyyy-MM-dd");
}
else
{
DataRow newRow = dt.NewRow();
newRow = InsertCSVRow(newRow, attributes, fields, constants);
newRow["ROW_CREATED_DATE"] = DateTime.Now.ToString("yyyy-MM-dd");
newRow["ROW_CHANGED_DATE"] = DateTime.Now.ToString("yyyy-MM-dd");
dt.Rows.Add(newRow);
duplicates.Add(duplicateKey, "");
}
}
}
}
timeEnd = DateTime.Now;
diff = timeEnd - timeStart;
//Console.WriteLine($"Time span, completion: {diff}");
}
if (tableDataType == indexDataType)
{
result = dt;
}
else
{
result = GetParentDataTable(indexDataType);
}
return(result);
}
