private string SerializeChilds(XElement parentChild) { List <DataInfoField> dataInfoFields = new List <DataInfoField>(); foreach (var child in parentChild.Elements().Where(q => !q.IsEmpty)) { DataInfoField dataInfoField = new DataInfoField(); dataInfoField.fieldname = child.Name.ToString(); if (child.Elements().Count() > 0) { if (parentChild.Elements().Where(q => q.Name.ToString() == child.Name.ToString()).Count() > 1) { dataInfoField.type = "List"; } dataInfoField.value = SerializeChilds(child); } else { if (parentChild.Elements().Where(q => q.Name.ToString() == child.Name.ToString()).Count() > 1) { dataInfoField.type = "List"; } dataInfoField.value = child.Value; } dataInfoFields.Add(dataInfoField); } DataInfo dataInfoItem = new DataInfo(); dataInfoItem.dataInfoFields = dataInfoFields; return(CustomJsoSerialice(dataInfoItem)); }
private async Task <bool> DatabaseProcess() { var connectionString = config.sourceOptions?.FirstOrDefault(p => p.Key == "connectionString").Value; var query = config.sourceOptions?.FirstOrDefault(p => p.Key == "query").Value; if (string.IsNullOrWhiteSpace(connectionString) || string.IsNullOrWhiteSpace(query)) { Logger.ErrorFormat("Database params from config is invalid."); return(false); } UniConnection connection = new UniConnection(connectionString); try { connection.Open(); UniCommand cmd = new UniCommand(query, connection); UniDataReader dataReader = cmd.ExecuteReader(); if (dataReader.HasRows) { while (dataReader.Read()) { List <DataInfoField> dataInfoFields = new List <DataInfoField>(); var dataId = string.Empty; for (int j = 0; j < dataReader.FieldCount; j++) { DataInfoField dataInfoField = new DataInfoField(); var columnName = dataReader.GetName(j); if (columnName == columnId) { dataId = dataReader.GetValue(j).ToString(); } else { dataInfoField.fieldname = columnName; dataInfoField.value = dataReader.GetValue(j).ToString(); dataInfoField.type = ""; dataInfoFields.Add(dataInfoField); } } DataInfo dataInfoItem = new DataInfo(); dataInfoItem.dataInfoFields = dataInfoFields; if (!historyStorage.Exits(dataId)) { if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId)) { historyStorage.InsertItem(dataId, 0, 1); } else { historyStorage.InsertItem(dataId, 1, 0); } } else { var faildAttems = historyStorage.IsFailed(dataId); if (faildAttems > 0 && faildAttems < maxAttemps) { // retry and updateElement if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId)) { historyStorage.UpdateItem(dataId, faildAttems, 1); } else { historyStorage.UpdateItem(dataId, faildAttems + 1, 0); } } } } } else { Logger.InfoFormat("Empty query result."); } } catch (Exception ex) { Logger.ErrorFormat("Unexpected error. Description: {0}", ex.Message); } finally { connection.Close(); } return(true); }
private async Task <bool> ExcelProcess() { var excelPath = config.sourceOptions?.FirstOrDefault(p => p.Key == "excelPath").Value; var excelSheet = config.sourceOptions?.FirstOrDefault(p => p.Key == "excelSheet").Value; if (string.IsNullOrWhiteSpace(excelPath) || string.IsNullOrWhiteSpace(excelSheet)) { Logger.ErrorFormat("Excel params from config is invalid."); return(false); } if (!Directory.Exists(excelPath)) { Logger.ErrorFormat("Failed open directory in: {0}", excelPath); } else { Logger.Info("Starting proccess excel"); DirectoryInfo di = new DirectoryInfo(excelPath); FileInfo[] files = di.GetFiles("*.bak"); if (files.Count() == 0) { Logger.InfoFormat("No excel files found!!!"); return(false); } if (string.IsNullOrEmpty(ApiTicket)) { return(false); } foreach (var file in files) { Stream excelStream = file.OpenRead(); using (excelStream) { if (excelStream != null) { Logger.InfoFormat("{0} Excel file loaded", file.Name.ToString()); ExcelEngine excelEngine = new ExcelEngine(); IWorkbook workbook = excelEngine.Excel.Workbooks.Open(excelStream); IWorksheet sheet = workbook.Worksheets.FirstOrDefault(p => p.Name == excelSheet); if (sheet == null) { Logger.ErrorFormat("{0} sheet not found in {1} excel file. File Skiped", excelSheet, file.Name); continue; } Logger.Info("Reading Excel content."); if (sheet.Rows.Count() > 1) { for (int i = 1; i < sheet.Rows.Count(); i++) { List <DataInfoField> dataInfoFields = new List <DataInfoField>(); var dataId = string.Empty; for (int j = 0; j < sheet.Columns.Count(); j++) { DataInfoField dataInfoField = new DataInfoField(); var columnName = sheet.GetValueRowCol(1, j + 1).ToString(); if (columnName == columnId) { dataId = sheet.GetValueRowCol(i + 1, j + 1).ToString(); } else { dataInfoField.fieldname = columnName; dataInfoField.value = sheet.GetValueRowCol(i + 1, j + 1).ToString(); dataInfoField.type = ""; dataInfoFields.Add(dataInfoField); } } DataInfo dataInfoItem = new DataInfo(); dataInfoItem.dataInfoFields = dataInfoFields; if (!historyStorage.Exits(dataId)) { if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId)) { historyStorage.InsertItem(dataId, 0, 1); } else { historyStorage.InsertItem(dataId, 1, 0); } } else { var faildAttems = historyStorage.IsFailed(dataId); if (faildAttems > 0 && faildAttems <= maxAttemps) { // retry and updateElement if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId)) { historyStorage.UpdateItem(dataId, faildAttems, 1); } else { historyStorage.UpdateItem(dataId, faildAttems + 1, 0); } } } } } else { Logger.Warn("This sheet of excel not contain informations."); } Logger.InfoFormat("{0} Excel file process complete.", file.Name.ToString()); } } } } return(true); }
private async Task <bool> XMLProcess() { if (string.IsNullOrEmpty(ApiTicket)) { Logger.ErrorFormat("Missing ApiTicket."); return(false); } var xmlPath = config.sourceOptions?.FirstOrDefault(p => p.Key == "xmlPath").Value; var complexNodes = config.sourceOptions?.FirstOrDefault(p => p.Key == "complexNodes").Value?.Split(','); if (string.IsNullOrWhiteSpace(xmlPath)) { Logger.ErrorFormat("XML params from config is invalid."); return(false); } if (!Directory.Exists(xmlPath)) { Logger.ErrorFormat("Failed open directory in: {0}", xmlPath); } else { Logger.Info("Starting proccess xml"); DirectoryInfo di = new DirectoryInfo(xmlPath); FileInfo[] files = di.GetFiles("*.xml"); if (files.Count() == 0) { Logger.InfoFormat("No xml files found!!!"); return(false); } foreach (var file in files) { Stream xmlStream = file.OpenRead(); var dataId = Path.GetFileNameWithoutExtension(file.FullName); bool processed = false; using (xmlStream) { if (xmlStream != null) { Logger.InfoFormat("{0} xml file loaded", file.Name.ToString()); XElement xelement = null; using (XmlReader xr = XmlReader.Create(xmlStream)) { xelement = XElement.Load(xr); var level1Elements = xelement.Elements(); foreach (var level1Element in level1Elements) { //Fill data process var level2Elements = level1Element.Elements().Where(q => !q.IsEmpty); List <DataInfoField> dataInfoFields = new List <DataInfoField>(); foreach (var level2Element in level2Elements) { DataInfoField dataInfoField = new DataInfoField(); //READ DATA INFO FIELD from XML dataInfoField.fieldname = level2Element.Name.ToString(); if (level2Element.Elements().Count() > 0) { if (complexNodes.Contains(dataInfoField.fieldname) || level1Element.Elements().Where(q => q.Name.ToString() == level2Element.Name.ToString()).Count() > 1) { dataInfoField.type = "List"; } dataInfoField.value = SerializeChilds(level2Element); } else { if (complexNodes.Contains(dataInfoField.fieldname) || level1Element.Elements().Where(q => q.Name.ToString() == level2Element.Name.ToString()).Count() > 1) { dataInfoField.type = "List"; } dataInfoField.value = level2Element.Value.ToString(); } dataInfoFields.Add(dataInfoField); } DataInfo dataInfoItem = new DataInfo(); dataInfoItem.dataInfoFields = dataInfoFields; if (!historyStorage.Exits(dataId)) { if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId)) { historyStorage.InsertItem(dataId, 0, 1); processed = true; } else { historyStorage.InsertItem(dataId, 1, 0); } } else { var faildAttems = historyStorage.IsFailed(dataId); if (faildAttems > 0 && faildAttems <= maxAttemps) { // retry and updateElement if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId)) { historyStorage.UpdateItem(dataId, faildAttems, 1); processed = true; } else { historyStorage.UpdateItem(dataId, faildAttems + 1, 0); } } } } } Logger.InfoFormat("{0} xml file process complete.", file.Name.ToString()); } } if (processed) { try { if (!Directory.Exists(file.DirectoryName + "\\procesados")) { Directory.CreateDirectory(file.DirectoryName + "\\procesados"); } file.MoveTo(file.DirectoryName + "\\procesados\\" + file.Name); } catch (Exception e) { Logger.ErrorFormat("Ocurrió un error al marcar como procesado el archivo {0}. Descripción del error: {1}", file.Name, e.Message); } } } } return(true); }