public async Task <string> GetIndexData(string sourceName) { string result = ""; ConnectParameters connector = await Common.GetConnectParameters(azureConnectionString, sourceName); if (connector.SourceType == "DataBase") { string jsonConnectDef = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); connector.DataAccessDefinition = jsonConnectDef; } else { Exception error = new Exception($"RuleManagement: data source must be a Database type"); throw error; } _dbConn.OpenConnection(connector); string strProcedure = $"EXEC spGetNumberOfDescendants '/', 1"; string query = ""; DataTable qc = _dbConn.GetDataTable(strProcedure, query); List <DmsIndex> index = ProcessAllChildren(qc); result = JsonConvert.SerializeObject(index, Formatting.Indented); _dbConn.CloseConnection(); return(result); }
private async Task CreatePPDMModel(DataModelParameters dmParameters, ConnectParameters connector) { try { string sql = await _fileStorage.ReadFile(dmParameters.FileShare, dmParameters.FileName); if (string.IsNullOrEmpty(sql)) { Exception error = new Exception($"Empty data from {dmParameters.FileName}"); throw error; } else { DbUtilities dbConn = new DbUtilities(); dbConn.OpenConnection(connector); dbConn.SQLExecute(sql); dbConn.CloseConnection(); } } catch (Exception ex) { Exception error = new Exception("Create PPDM Model Error: ", ex); throw error; } }
public async Task <string> GetPrediction(string name) { string result = ""; string ruleName = name + ".json"; result = await _fileStorage.ReadFile(ruleShare, ruleName); if (string.IsNullOrEmpty(result)) { Exception error = new Exception($"Empty data from {ruleName}"); throw error; } return(result); }
public async Task <int> Initialize(ConnectParameters target, ConnectParameters source, string taxonomyFile, string filter) { string jsonTaxonomy = await _fileStorage.ReadFile("taxonomy", taxonomyFile); int parentNodes = 0; if (source.SourceType == "DataBase") { iBuilder = new IndexBuilder(new DBDataAccess()); } else { source.ConnectionString = _azureConnectionString; if (source.DataType == "Logs") { iBuilder = new IndexBuilder(new LASDataAccess(_fileStorage)); } else { iBuilder = new IndexBuilder(new CSVDataAccess(_fileStorage)); } } iBuilder.InitializeIndex(target, source, jsonTaxonomy); iBuilder.CreateRoot(source); parentNodes = iBuilder.JsonIndexArray.Count; return(parentNodes); }
public async Task <ActionResult <string> > GetPipeline(string name) { SetStorageAccount(); string dataOpsFile = await fileStorageService.ReadFile(fileShare, name); return(dataOpsFile); }
public async Task ExecutePrediction(PredictionParameters parms) { string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); _accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson); ConnectParameters connector = await GetConnector(parms.DataConnector); databaseConnectionString = connector.ConnectionString; _dbConn.OpenConnection(connector); string sourceConnector = GetSource(); if (parms.DataConnector == sourceConnector) { syncPredictions = true; } else { syncPredictions = false; } RuleManagement rules = new RuleManagement(_azureConnectionString); RuleModel rule = await rules.GetRuleAndFunction(parms.DataConnector, parms.PredictionId); manageIndexTable = new ManageIndexTable(_accessDefs, connector.ConnectionString, rule.DataType, rule.FailRule); manageIndexTable.InitQCFlags(false); await MakePredictions(rule, connector); _dbConn.CloseConnection(); manageIndexTable.SaveQCFlags(); }
public async Task <List <int> > ExecuteQcRule(DataQCParameters qcParms) { try { ConnectParameters connector = await GetConnector(qcParms.DataConnector); RuleModel rule = await GetRuleAndFunctionInfo(qcParms.DataConnector, qcParms.RuleId); string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); _accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson); DbUtilities dbConn = new DbUtilities(); dbConn.OpenConnection(connector); manageQCFlags = new ManageIndexTable(_accessDefs, connector.ConnectionString, rule.DataType); List <int> failedObjects = await QualityCheckDataType(dbConn, rule, connector); dbConn.CloseConnection(); return(failedObjects); } catch (Exception ex) { Exception error = new Exception($"DataQc: Could process rule {qcParms.RuleId}, {ex}"); throw error; } }
public async Task <DataTable> GetLASWellHeaders(ConnectParameters source, ConnectParameters target) { string accessJson = await fileStorageService.ReadFile("connectdefinition", "PPDMDataAccess.json"); _dataDef = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson); //lasAccessJson = JObject.Parse(await fileStorageService.ReadFile("connectdefinition", "LASDataAccess.json")); string lasMappingsStr = await fileStorageService.ReadFile("connectdefinition", "LASDataAccess.json"); lasMappings = JsonConvert.DeserializeObject <LASMappings>(lasMappingsStr); List <string> files = new List <string>(); files = await GetLASFileNames(source.Catalog); DataAccessDef dataType = _dataDef.First(x => x.DataType == "WellBore"); string select = dataType.Select; string query = $" where 0 = 1"; _dbConn.OpenConnection(target); DataTable dt = _dbConn.GetDataTable(select, query); foreach (string file in files) { LASSections ls = await GetLASSections(source.Catalog, file); lasSections.Add(ls); GetVersionInfo(ls.versionInfo); string json = GetHeaderInfo(ls.wellInfo); DataRow row = dt.NewRow(); JObject jo = JObject.Parse(json); foreach (JProperty property in jo.Properties()) { string strValue = property.Value.ToString(); if (!string.IsNullOrEmpty(strValue)) { if (row.Table.Columns.Contains(property.Name)) { row[property.Name] = property.Value; } } } dt.Rows.Add(row); } return(dt); }
public async Task <ConnectParameters> GetSourceParameters(string name) { if (String.IsNullOrEmpty(name)) { Exception error = new Exception($"Sources: Source name is not set"); throw error; } var connector = _sourceData.GetSource(name); if (connector.SourceType == "DataBase") { string jsonConnectDef = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); connector.DataAccessDefinition = jsonConnectDef; } return(connector); }
public async Task CopyFiles(TransferParameters parms) { try { ConnectParameters sourceConnector = await Common.GetConnectParameters(_azureConnectionString, parms.SourceName); ConnectParameters targetConnector = await Common.GetConnectParameters(_azureConnectionString, parms.TargetName); string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); string referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json"); targetConnector.DataAccessDefinition = dataAccessDefinition; if (sourceConnector.SourceType == "DataBase") { DatabaseLoader dl = new DatabaseLoader(); dl.CopyTable(parms, sourceConnector.ConnectionString, targetConnector, referenceJson); } else if (sourceConnector.SourceType == "File") { if (sourceConnector.DataType == "Logs") { LASLoader ls = new LASLoader(_fileStorage); await ls.LoadLASFile(sourceConnector, targetConnector, parms.Table, referenceJson); } else { CSVLoader cl = new CSVLoader(_fileStorage); await cl.LoadCSVFile(sourceConnector, targetConnector, parms.Table); } } else { Exception error = new Exception($"DataTransfer: Not a valid source type for {sourceConnector.SourceName}"); throw error; } } catch (Exception ex) { Exception error = new Exception($"DataTransfer: Problems transfer files/tables, {ex}"); throw error; } }
public async Task <List <PipeLine> > GetDataOpsPipe(string name) { List <PipeLine> dataOps = new List <PipeLine>(); List <string> result = await _fileStorage.ListFiles("dataops"); string dataOpsFile = await _fileStorage.ReadFile(fileShare, name); dataOps = JsonConvert.DeserializeObject <List <PipeLine> >(dataOpsFile); return(dataOps); }
public async Task GetTransferConnector(string message) { try { TransferParameters transParms = JsonConvert.DeserializeObject <TransferParameters>(message); _targetConnector = GetConnectionString(transParms.TargetName); string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); _targetConnector.DataAccessDefinition = dataAccessDefinition; target = _targetConnector.ConnectionString; _logger.LogInformation($"Target connect string: {target}"); ConnectParameters sourceConnector = GetConnectionString(transParms.SourceName); source = sourceConnector.ConnectionString; _logger.LogInformation($"Source connect string: {source}"); transferQuery = transParms.TransferQuery; queryType = transParms.QueryType; _referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json"); } catch (Exception ex) { _logger.LogInformation($"Error getting connector info {ex.ToString()}"); } }
public async Task <string> GetAttributeInfo(string sourceName, string dataType) { string json = ""; ConnectParameters connector = await Common.GetConnectParameters(azureConnectionString, sourceName); string accessJson = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); List <DataAccessDef> accessDefs = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson); DataAccessDef dataAccess = accessDefs.First(x => x.DataType == dataType); string sql = dataAccess.Select; string table = Common.GetTable(sql); string query = $" where 0 = 1"; DbUtilities dbConn = new DbUtilities(); dbConn.OpenWithConnectionString(connector.ConnectionString); DataTable dt = dbConn.GetDataTable(sql, query); AttributeInfo attributeInfo = new AttributeInfo(); attributeInfo.DataAttributes = dt.GetColumnTypes(); json = JsonConvert.SerializeObject(attributeInfo); return(json); }
public async Task <DataTable> GetCSVTable(ConnectParameters source, ConnectParameters target, string indexDataType) { DataTable result = new DataTable(); if (dt.Rows.Count == 0) { DateTime timeStart = DateTime.Now; DateTime timeEnd; TimeSpan diff; connectionString = target.ConnectionString; dt = new DataTable(); string accessJson = await fileStorageService.ReadFile("connectdefinition", "PPDMDataAccess.json"); _dataDef = JsonConvert.DeserializeObject <List <DataAccessDef> >(accessJson); string referenceJson = await fileStorageService.ReadFile("connectdefinition", "PPDMReferenceTables.json"); _references = JsonConvert.DeserializeObject <List <ReferenceTable> >(referenceJson); string csvJson = await fileStorageService.ReadFile("connectdefinition", "CSVDataAccess.json"); _csvDef = JsonConvert.DeserializeObject <List <CSVAccessDef> >(csvJson); //Console.WriteLine("start reading csv file"); string csvText = await fileStorageService.ReadFile(source.Catalog, source.FileName); timeEnd = DateTime.Now; diff = timeEnd - timeStart; //Console.WriteLine($"Time span, read all definitions files: {diff}"); string dataType = source.DataType.Remove(source.DataType.Length - 1, 1); dataAccess = _dataDef.First(x => x.DataType == dataType); dt = NewDataTable(dataType); tableDataType = dataType; int headerLines = 1; int wellCounter = 0; CSVAccessDef csvAccess = _csvDef.First(x => x.DataType == dataType); Dictionary <string, int> attributes = csvAccess.Mappings.ToDictionary(); Dictionary <string, string> constants = csvAccess.Constants.ToStringDictionary(); byte[] byteArray = Encoding.ASCII.GetBytes(csvText); MemoryStream csvStream = new MemoryStream(byteArray); using (TextFieldParser csvParser = new TextFieldParser(csvStream)) { csvParser.CommentTokens = new string[] { "#" }; csvParser.SetDelimiters(new string[] { "," }); csvParser.HasFieldsEnclosedInQuotes = true; for (int i = 0; i < headerLines; i++) { string line = csvParser.ReadLine(); } while (!csvParser.EndOfData) { wellCounter++; if ((wellCounter % 1000) == 0) { var tmp = wellCounter; } string[] fields = csvParser.ReadFields(); string key = GetDataKey(fields, attributes); string duplicateKey = GetDuplicateKey(fields, attributes); if (string.IsNullOrEmpty(key)) { //Console.WriteLine($"Well {wellCounter} has an empty key"); } else { if (duplicates.ContainsKey(duplicateKey)) { DataRow[] rows = dt.Select(key); rows[0] = InsertCSVRow(rows[0], attributes, fields, constants); rows[0]["ROW_CHANGED_DATE"] = DateTime.Now.ToString("yyyy-MM-dd"); } else { DataRow newRow = dt.NewRow(); newRow = InsertCSVRow(newRow, attributes, fields, constants); newRow["ROW_CREATED_DATE"] = DateTime.Now.ToString("yyyy-MM-dd"); newRow["ROW_CHANGED_DATE"] = DateTime.Now.ToString("yyyy-MM-dd"); dt.Rows.Add(newRow); duplicates.Add(duplicateKey, ""); } } } } timeEnd = DateTime.Now; diff = timeEnd - timeStart; //Console.WriteLine($"Time span, completion: {diff}"); } if (tableDataType == indexDataType) { result = dt; } else { result = GetParentDataTable(indexDataType); } return(result); }