private string SerializeChilds(XElement parentChild)
{
List <DataInfoField> dataInfoFields = new List <DataInfoField>();
foreach (var child in parentChild.Elements().Where(q => !q.IsEmpty))
{
DataInfoField dataInfoField = new DataInfoField();
dataInfoField.fieldname = child.Name.ToString();
if (child.Elements().Count() > 0)
{
if (parentChild.Elements().Where(q => q.Name.ToString() == child.Name.ToString()).Count() > 1)
{
dataInfoField.type = "List";
}
dataInfoField.value = SerializeChilds(child);
}
else
{
if (parentChild.Elements().Where(q => q.Name.ToString() == child.Name.ToString()).Count() > 1)
{
dataInfoField.type = "List";
}
dataInfoField.value = child.Value;
}
dataInfoFields.Add(dataInfoField);
}
DataInfo dataInfoItem = new DataInfo();
dataInfoItem.dataInfoFields = dataInfoFields;
return(CustomJsoSerialice(dataInfoItem));
}
private async Task <bool> DatabaseProcess()
{
var connectionString = config.sourceOptions?.FirstOrDefault(p => p.Key == "connectionString").Value;
var query = config.sourceOptions?.FirstOrDefault(p => p.Key == "query").Value;
if (string.IsNullOrWhiteSpace(connectionString) ||
string.IsNullOrWhiteSpace(query))
{
Logger.ErrorFormat("Database params from config is invalid.");
return(false);
}
UniConnection connection = new UniConnection(connectionString);
try
{
connection.Open();
UniCommand cmd = new UniCommand(query, connection);
UniDataReader dataReader = cmd.ExecuteReader();
if (dataReader.HasRows)
{
while (dataReader.Read())
{
List <DataInfoField> dataInfoFields = new List <DataInfoField>();
var dataId = string.Empty;
for (int j = 0; j < dataReader.FieldCount; j++)
{
DataInfoField dataInfoField = new DataInfoField();
var columnName = dataReader.GetName(j);
if (columnName == columnId)
{
dataId = dataReader.GetValue(j).ToString();
}
else
{
dataInfoField.fieldname = columnName;
dataInfoField.value = dataReader.GetValue(j).ToString();
dataInfoField.type = "";
dataInfoFields.Add(dataInfoField);
}
}
DataInfo dataInfoItem = new DataInfo();
dataInfoItem.dataInfoFields = dataInfoFields;
if (!historyStorage.Exits(dataId))
{
if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId))
{
historyStorage.InsertItem(dataId, 0, 1);
}
else
{
historyStorage.InsertItem(dataId, 1, 0);
}
}
else
{
var faildAttems = historyStorage.IsFailed(dataId);
if (faildAttems > 0 && faildAttems < maxAttemps)
{
// retry and updateElement
if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId))
{
historyStorage.UpdateItem(dataId, faildAttems, 1);
}
else
{
historyStorage.UpdateItem(dataId, faildAttems + 1, 0);
}
}
}
}
}
else
{
Logger.InfoFormat("Empty query result.");
}
}
catch (Exception ex)
{
Logger.ErrorFormat("Unexpected error. Description: {0}", ex.Message);
}
finally
{
connection.Close();
}
return(true);
}
private async Task <bool> ExcelProcess()
{
var excelPath = config.sourceOptions?.FirstOrDefault(p => p.Key == "excelPath").Value;
var excelSheet = config.sourceOptions?.FirstOrDefault(p => p.Key == "excelSheet").Value;
if (string.IsNullOrWhiteSpace(excelPath) || string.IsNullOrWhiteSpace(excelSheet))
{
Logger.ErrorFormat("Excel params from config is invalid.");
return(false);
}
if (!Directory.Exists(excelPath))
{
Logger.ErrorFormat("Failed open directory in: {0}", excelPath);
}
else
{
Logger.Info("Starting proccess excel");
DirectoryInfo di = new DirectoryInfo(excelPath);
FileInfo[] files = di.GetFiles("*.bak");
if (files.Count() == 0)
{
Logger.InfoFormat("No excel files found!!!");
return(false);
}
if (string.IsNullOrEmpty(ApiTicket))
{
return(false);
}
foreach (var file in files)
{
Stream excelStream = file.OpenRead();
using (excelStream)
{
if (excelStream != null)
{
Logger.InfoFormat("{0} Excel file loaded", file.Name.ToString());
ExcelEngine excelEngine = new ExcelEngine();
IWorkbook workbook = excelEngine.Excel.Workbooks.Open(excelStream);
IWorksheet sheet = workbook.Worksheets.FirstOrDefault(p => p.Name == excelSheet);
if (sheet == null)
{
Logger.ErrorFormat("{0} sheet not found in {1} excel file. File Skiped", excelSheet, file.Name);
continue;
}
Logger.Info("Reading Excel content.");
if (sheet.Rows.Count() > 1)
{
for (int i = 1; i < sheet.Rows.Count(); i++)
{
List <DataInfoField> dataInfoFields = new List <DataInfoField>();
var dataId = string.Empty;
for (int j = 0; j < sheet.Columns.Count(); j++)
{
DataInfoField dataInfoField = new DataInfoField();
var columnName = sheet.GetValueRowCol(1, j + 1).ToString();
if (columnName == columnId)
{
dataId = sheet.GetValueRowCol(i + 1, j + 1).ToString();
}
else
{
dataInfoField.fieldname = columnName;
dataInfoField.value = sheet.GetValueRowCol(i + 1, j + 1).ToString();
dataInfoField.type = "";
dataInfoFields.Add(dataInfoField);
}
}
DataInfo dataInfoItem = new DataInfo();
dataInfoItem.dataInfoFields = dataInfoFields;
if (!historyStorage.Exits(dataId))
{
if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId))
{
historyStorage.InsertItem(dataId, 0, 1);
}
else
{
historyStorage.InsertItem(dataId, 1, 0);
}
}
else
{
var faildAttems = historyStorage.IsFailed(dataId);
if (faildAttems > 0 && faildAttems <= maxAttemps)
{
// retry and updateElement
if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId))
{
historyStorage.UpdateItem(dataId, faildAttems, 1);
}
else
{
historyStorage.UpdateItem(dataId, faildAttems + 1, 0);
}
}
}
}
}
else
{
Logger.Warn("This sheet of excel not contain informations.");
}
Logger.InfoFormat("{0} Excel file process complete.", file.Name.ToString());
}
}
}
}
return(true);
}
private async Task <bool> XMLProcess()
{
if (string.IsNullOrEmpty(ApiTicket))
{
Logger.ErrorFormat("Missing ApiTicket.");
return(false);
}
var xmlPath = config.sourceOptions?.FirstOrDefault(p => p.Key == "xmlPath").Value;
var complexNodes = config.sourceOptions?.FirstOrDefault(p => p.Key == "complexNodes").Value?.Split(',');
if (string.IsNullOrWhiteSpace(xmlPath))
{
Logger.ErrorFormat("XML params from config is invalid.");
return(false);
}
if (!Directory.Exists(xmlPath))
{
Logger.ErrorFormat("Failed open directory in: {0}", xmlPath);
}
else
{
Logger.Info("Starting proccess xml");
DirectoryInfo di = new DirectoryInfo(xmlPath);
FileInfo[] files = di.GetFiles("*.xml");
if (files.Count() == 0)
{
Logger.InfoFormat("No xml files found!!!");
return(false);
}
foreach (var file in files)
{
Stream xmlStream = file.OpenRead();
var dataId = Path.GetFileNameWithoutExtension(file.FullName);
bool processed = false;
using (xmlStream)
{
if (xmlStream != null)
{
Logger.InfoFormat("{0} xml file loaded", file.Name.ToString());
XElement xelement = null;
using (XmlReader xr = XmlReader.Create(xmlStream))
{
xelement = XElement.Load(xr);
var level1Elements = xelement.Elements();
foreach (var level1Element in level1Elements)
{
//Fill data process
var level2Elements = level1Element.Elements().Where(q => !q.IsEmpty);
List <DataInfoField> dataInfoFields = new List <DataInfoField>();
foreach (var level2Element in level2Elements)
{
DataInfoField dataInfoField = new DataInfoField();
//READ DATA INFO FIELD from XML
dataInfoField.fieldname = level2Element.Name.ToString();
if (level2Element.Elements().Count() > 0)
{
if (complexNodes.Contains(dataInfoField.fieldname) || level1Element.Elements().Where(q => q.Name.ToString() == level2Element.Name.ToString()).Count() > 1)
{
dataInfoField.type = "List";
}
dataInfoField.value = SerializeChilds(level2Element);
}
else
{
if (complexNodes.Contains(dataInfoField.fieldname) || level1Element.Elements().Where(q => q.Name.ToString() == level2Element.Name.ToString()).Count() > 1)
{
dataInfoField.type = "List";
}
dataInfoField.value = level2Element.Value.ToString();
}
dataInfoFields.Add(dataInfoField);
}
DataInfo dataInfoItem = new DataInfo();
dataInfoItem.dataInfoFields = dataInfoFields;
if (!historyStorage.Exits(dataId))
{
if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId))
{
historyStorage.InsertItem(dataId, 0, 1);
processed = true;
}
else
{
historyStorage.InsertItem(dataId, 1, 0);
}
}
else
{
var faildAttems = historyStorage.IsFailed(dataId);
if (faildAttems > 0 && faildAttems <= maxAttemps)
{
// retry and updateElement
if (await StartProcess(dataInfoItem, processId, communityId, config.initWF, config.superAdmin, dataId))
{
historyStorage.UpdateItem(dataId, faildAttems, 1);
processed = true;
}
else
{
historyStorage.UpdateItem(dataId, faildAttems + 1, 0);
}
}
}
}
}
Logger.InfoFormat("{0} xml file process complete.", file.Name.ToString());
}
}
if (processed)
{
try
{
if (!Directory.Exists(file.DirectoryName + "\\procesados"))
{
Directory.CreateDirectory(file.DirectoryName + "\\procesados");
}
file.MoveTo(file.DirectoryName + "\\procesados\\" + file.Name);
}
catch (Exception e)
{
Logger.ErrorFormat("Ocurrió un error al marcar como procesado el archivo {0}. Descripción del error: {1}", file.Name, e.Message);
}
}
}
}
return(true);
}
