private void ImportRegistersStruct(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db)
{
Int32 resourcePluginCol = jData.GetKeyIndex("resource_plugin");
Int32 pkgCol = jData.GetKeyIndex("package");
if (resourcePluginCol == -1)
{
TextLog.Log("Inbound", "\t[ImportStruct] Erro on find column 'resource_plugin' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (pkgCol == -1)
{
TextLog.Log("Inbound", "\t[ImportStruct] Erro on find column 'package' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
//Realiza a importação no modelo BulkInsert por melhor desempenho do banco
DataTable dtBulk = new DataTable();
dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime)));
dtBulk.Columns.Add(new DataColumn("file_name", typeof(String)));
dtBulk.Columns.Add(new DataColumn("resource_plugin", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("import_id", typeof(String)));
dtBulk.Columns.Add(new DataColumn("package_id", typeof(String)));
dtBulk.Columns.Add(new DataColumn("package", typeof(String)));
foreach (String[] dr in jData.data)
{
PluginConnectorBaseImportPackageStruct pkg = JSON.DeserializeFromBase64 <PluginConnectorBaseImportPackageStruct>(dr[pkgCol]);
dtBulk.Rows.Add(new Object[] { DateTime.Now, f.Name, dr[resourcePluginCol], pkg.importId, pkg.pkgId, JSON.Serialize2(pkg) });
}
db.BulkCopy(dtBulk, "collector_imports_struct");
//Atualiza os registros importados deste arquivo para liberar o processamento
//Isso avisa o sistema que estes registros estão livres para processamento
db.ExecuteNonQuery("update collector_imports_struct set status = 'F' where [file_name] = '" + f.Name + "'", CommandType.Text, null);
#if DEBUG
TextLog.Log("Inbound", "\t[ImportStruct] Imported " + dtBulk.Rows.Count + " registers for enterprise " + req.enterpriseid + " and proxy " + req.host);
#endif
dtBulk.Dispose();
dtBulk = null;
jData = null;
}
private void ImportRegistersV2(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db)
{
Int32 resourcePluginCol = jData.GetKeyIndex("resource_plugin");
Int32 pkgCol = jData.GetKeyIndex("package");
if (resourcePluginCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegistersV2] Erro on find column 'resource_plugin' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (pkgCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegistersV2] Erro on find column 'package' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
//Realiza a importação no modelo BulkInsert por melhor desempenho do banco
DataTable dtBulk = new DataTable();
dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime)));
dtBulk.Columns.Add(new DataColumn("file_name", typeof(String)));
dtBulk.Columns.Add(new DataColumn("resource_plugin", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("import_id", typeof(String)));
dtBulk.Columns.Add(new DataColumn("package_id", typeof(String)));
dtBulk.Columns.Add(new DataColumn("package", typeof(String)));
dtBulk.Columns.Add(new DataColumn("status", typeof(String)));
foreach (String[] dr in jData.data)
{
PluginConnectorBaseImportPackageUser pkg = JSON.DeserializeFromBase64 <PluginConnectorBaseImportPackageUser>(dr[pkgCol]);
dtBulk.Rows.Add(new Object[] { DateTime.Now, f.Name, dr[resourcePluginCol], pkg.importId, pkg.pkgId, JSON.Serialize2(pkg), 'F' });
try
{
DbParameterCollection par = new DbParameterCollection();
par.Add("@date", typeof(DateTime)).Value = pkg.GetBuildDate();
par.Add("@package_id", typeof(String), pkg.pkgId.Length).Value = pkg.pkgId;
Int64 trackId = db.ExecuteScalar <Int64>("select id from st_package_track where flow = 'inbound' and date = @date and package_id = @package_id", System.Data.CommandType.Text, par, null);
db.AddPackageTrack(trackId, "inbound", "Package imported to process queue");
}
catch { }
}
db.BulkCopy(dtBulk, "collector_imports");
//Apaga todos os registros da tabela temporaria
/*
* Procedimento desabiliato em 2018-08-29 por suspeita de problema
* db.ExecuteNonQuery("delete from collector_imports_temp", System.Data.CommandType.Text, null, null);
*
* db.BulkCopy(dtBulk, "collector_imports_temp");
*
* //Proteção contra reimportação de pacotes (loop)
* db.ExecuteNonQuery("delete from collector_imports_temp where exists (select 1 from collector_imports_old o where o.date >= dateadd(day,-1,getdate()) and o.file_name = file_name and o.resource_plugin_id = resource_plugin_id and o.import_id = import_id and o.package_id = package_id)", System.Data.CommandType.Text, null, null);
* db.ExecuteNonQuery("delete from collector_imports_temp where exists (select 1 from collector_imports o where o.date >= dateadd(day,-1,getdate()) and o.file_name = file_name and o.resource_plugin_id = resource_plugin_id and o.import_id = import_id and o.package_id = package_id)", System.Data.CommandType.Text, null, null);
*
* db.ExecuteNonQuery("insert into collector_imports select * from collector_imports_temp", System.Data.CommandType.Text, null, null);
* db.ExecuteNonQuery("delete from collector_imports_temp", System.Data.CommandType.Text, null, null);
* */
//Atualiza os registros importados deste arquivo para liberar o processamento
//Isso avisa o sistema que estes registros estão livres para processamento
//*** Desabilitado essa funç~~ao em 2018-03-08, e colocado o registro para ser importado diretamente com o Status 'F'
//db.ExecuteNonQuery("update collector_imports set status = 'F' where [file_name] = '" + f.Name + "'", CommandType.Text, null);
//Realiza o rebuild do indice desta tabela para agilizar no engine
//Este processo será executado somente uma vez pelo objeto pai
//db.ExecuteNonQuery("sp_reindex_imports", CommandType.StoredProcedure, null);
#if DEBUG
TextLog.Log("Inbound", "\t[ImportRegistersV2] Imported " + dtBulk.Rows.Count + " registers for enterprise " + req.enterpriseid + " and proxy " + req.host);
#endif
dtBulk.Dispose();
dtBulk = null;
jData = null;
}
private void ImportNotify(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db)
{
Int32 resourceCol = jData.GetKeyIndex("resource");
Int32 sourceCol = jData.GetKeyIndex("source");
Int32 uriCol = jData.GetKeyIndex("uri");
Int32 entityIdCol = jData.GetKeyIndex("entityid");
if (resourceCol == -1)
{
TextLog.Log("Inbound", "\t[ImportNotify] Erro on find column 'resource' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (sourceCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'source' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (uriCol == -1)
{
TextLog.Log("Inbound", "\t[ImportNotify] Erro on find column 'uri' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (entityIdCol == -1)
{
TextLog.Log("Inbound", "\t[ImportNotify] Erro on find column 'entityId' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
DateTime date = DateTime.Now;
//Realiza a importação no modelo BulkInsert por melhor desempenho do banco
DataTable dtBulk = new DataTable();
dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime)));
dtBulk.Columns.Add(new DataColumn("source", typeof(String)));
dtBulk.Columns.Add(new DataColumn("plugin_uri", typeof(String)));
dtBulk.Columns.Add(new DataColumn("resource_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("entity_id", typeof(Int64)));
foreach (String[] dr in jData.data)
{
dtBulk.Rows.Add(new Object[] { date, dr[sourceCol], dr[uriCol], Int64.Parse(dr[resourceCol]), Int64.Parse(dr[entityIdCol]) });
}
db.BulkCopy(dtBulk, "notify_imports");
#if DEBUG
TextLog.Log("Inbound", "\t[ImportNotify] Imported " + dtBulk.Rows.Count + " notify for enterprise " + req.enterpriseid + " and proxy " + req.host);
#endif
dtBulk.Dispose();
dtBulk = null;
jData = null;
}
private void ImportRegistersOLD(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db)
{
Int32 resourceCol = jData.GetKeyIndex("resource");
Int32 uriCol = jData.GetKeyIndex("uri");
Int32 importidCol = jData.GetKeyIndex("importid");
Int32 registryidCol = jData.GetKeyIndex("registryid");
Int32 datanameCol = jData.GetKeyIndex("dataname");
Int32 datavalueCol = jData.GetKeyIndex("datavalue");
Int32 datatypeCol = jData.GetKeyIndex("datatype");
if (resourceCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'resource' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (uriCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'uri' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (importidCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'importid' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (registryidCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'registryid' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (datanameCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'dataname' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (datavalueCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'datavalue' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (datatypeCol == -1)
{
TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'datatype' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
DateTime date = DateTime.Now;
//Realiza a importação no modelo BulkInsert por melhor desempenho do banco
DataTable dtBulk = new DataTable();
dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime)));
dtBulk.Columns.Add(new DataColumn("file_name", typeof(String)));
dtBulk.Columns.Add(new DataColumn("plugin_uri", typeof(String)));
dtBulk.Columns.Add(new DataColumn("resource_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("import_id", typeof(String)));
dtBulk.Columns.Add(new DataColumn("registry_id", typeof(String)));
dtBulk.Columns.Add(new DataColumn("data_name", typeof(String)));
dtBulk.Columns.Add(new DataColumn("data_value", typeof(String)));
dtBulk.Columns.Add(new DataColumn("data_type", typeof(String)));
foreach (String[] dr in jData.data)
{
dtBulk.Rows.Add(new Object[] { date, f.Name, dr[uriCol], Int64.Parse(dr[resourceCol]), dr[importidCol], dr[registryidCol], dr[datanameCol], dr[datavalueCol], dr[datatypeCol] });
}
db.BulkCopy(dtBulk, "collector_imports");
//Atualiza os registros importados deste arquivo para liberar o processamento
//Isso avisa o sistema que estes registros estão livres para processamento
db.ExecuteNonQuery("update collector_imports set status = 'F' where [file_name] = '" + f.Name + "'", CommandType.Text, null);
//Realiza o rebuild do indice desta tabela para agilizar no engine
//Este processo será executado somente uma vez pelo objeto pai
//db.ExecuteNonQuery("sp_reindex_imports", CommandType.StoredProcedure, null);
#if DEBUG
TextLog.Log("Inbound", "\t[ImportRegisters] Imported " + dtBulk.Rows.Count + " registers for enterprise " + req.enterpriseid + " and proxy " + req.host);
#endif
dtBulk.Dispose();
dtBulk = null;
jData = null;
}
private void ImportLogs(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db)
{
Int32 resourceCol = jData.GetKeyIndex("resource");
Int32 dateCol = jData.GetKeyIndex("date");
Int32 sourceCol = jData.GetKeyIndex("source");
Int32 keyCol = jData.GetKeyIndex("key");
Int32 uriCol = jData.GetKeyIndex("uri");
Int32 typeCol = jData.GetKeyIndex("type");
Int32 entityIdCol = jData.GetKeyIndex("entityid");
Int32 identityIdCol = jData.GetKeyIndex("identityid");
Int32 textCol = jData.GetKeyIndex("text");
Int32 additionaldataCol = jData.GetKeyIndex("additionaldata");
if (resourceCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'resource' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (sourceCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'source' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (keyCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'key' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (uriCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'uri' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (entityIdCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'entityId' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (identityIdCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'identityId' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
if (textCol == -1)
{
TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'text' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host);
return;
}
DateTime date = DateTime.Now;
//Realiza a importação no modelo BulkInsert por melhor desempenho do banco
DataTable dtBulk = new DataTable();
dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime)));
dtBulk.Columns.Add(new DataColumn("source", typeof(String)));
dtBulk.Columns.Add(new DataColumn("key", typeof(Int32)));
dtBulk.Columns.Add(new DataColumn("enterprise_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("proxy_name", typeof(String)));
dtBulk.Columns.Add(new DataColumn("proxy_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("plugin_uri", typeof(String)));
dtBulk.Columns.Add(new DataColumn("plugin_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("resource_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("entity_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("identity_id", typeof(Int64)));
dtBulk.Columns.Add(new DataColumn("type", typeof(String)));
dtBulk.Columns.Add(new DataColumn("text", typeof(String)));
dtBulk.Columns.Add(new DataColumn("additional_data", typeof(String)));
foreach (String[] dr in jData.data)
{
try
{
//Console.WriteLine(f.Name + " - " + dr[entityIdCol] + " ==> " + dr[textCol]);
//Console.WriteLine(dr[additionaldataCol]);
//Console.WriteLine("");
dtBulk.Rows.Add(new Object[] { (dateCol >= 0 ? DateTime.Parse(dr[dateCol]) : date), dr[sourceCol], dr[keyCol], req.enterpriseid, req.host, 0, dr[uriCol], 0, Int64.Parse(dr[resourceCol]), Int64.Parse(dr[entityIdCol]), Int64.Parse(dr[identityIdCol]), dr[typeCol], dr[textCol], (additionaldataCol >= 0 ? dr[additionaldataCol] : "") });
}
catch (Exception ex)
{
throw ex;
}
}
db.BulkCopy(dtBulk, "logs_imports");
//Procedure que processa os logs e importa para a tabela definitiva
db.ExecuteNonQuery("sp_process_logs", CommandType.StoredProcedure, null);
#if debug
db.AddUserLog(LogKey.Import, null, "Inbound", UserLogLevel.Info, 0, 0, 0, 0, 0, 0, 0, "Imported " + dtBulk.Rows.Count + " logs for enterprise " + req.enterpriseid + " and proxy " + req.host + " from file " + f.Name);
TextLog.Log("Inbound", "\t[ImportLogs] Imported " + dtBulk.Rows.Count + " logs for enterprise " + req.enterpriseid + " and proxy " + req.host);
#endif
dtBulk.Dispose();
dtBulk = null;
jData = null;
}