private void ImportRegistersStruct(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db) { Int32 resourcePluginCol = jData.GetKeyIndex("resource_plugin"); Int32 pkgCol = jData.GetKeyIndex("package"); if (resourcePluginCol == -1) { TextLog.Log("Inbound", "\t[ImportStruct] Erro on find column 'resource_plugin' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (pkgCol == -1) { TextLog.Log("Inbound", "\t[ImportStruct] Erro on find column 'package' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } //Realiza a importação no modelo BulkInsert por melhor desempenho do banco DataTable dtBulk = new DataTable(); dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime))); dtBulk.Columns.Add(new DataColumn("file_name", typeof(String))); dtBulk.Columns.Add(new DataColumn("resource_plugin", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("import_id", typeof(String))); dtBulk.Columns.Add(new DataColumn("package_id", typeof(String))); dtBulk.Columns.Add(new DataColumn("package", typeof(String))); foreach (String[] dr in jData.data) { PluginConnectorBaseImportPackageStruct pkg = JSON.DeserializeFromBase64 <PluginConnectorBaseImportPackageStruct>(dr[pkgCol]); dtBulk.Rows.Add(new Object[] { DateTime.Now, f.Name, dr[resourcePluginCol], pkg.importId, pkg.pkgId, JSON.Serialize2(pkg) }); } db.BulkCopy(dtBulk, "collector_imports_struct"); //Atualiza os registros importados deste arquivo para liberar o processamento //Isso avisa o sistema que estes registros estão livres para processamento db.ExecuteNonQuery("update collector_imports_struct set status = 'F' where [file_name] = '" + f.Name + "'", CommandType.Text, null); #if DEBUG TextLog.Log("Inbound", "\t[ImportStruct] Imported " + dtBulk.Rows.Count + " registers for enterprise " + req.enterpriseid + " and proxy " + req.host); #endif dtBulk.Dispose(); dtBulk = null; jData = null; }
private void ImportRegistersV2(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db) { Int32 resourcePluginCol = jData.GetKeyIndex("resource_plugin"); Int32 pkgCol = jData.GetKeyIndex("package"); if (resourcePluginCol == -1) { TextLog.Log("Inbound", "\t[ImportRegistersV2] Erro on find column 'resource_plugin' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (pkgCol == -1) { TextLog.Log("Inbound", "\t[ImportRegistersV2] Erro on find column 'package' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } //Realiza a importação no modelo BulkInsert por melhor desempenho do banco DataTable dtBulk = new DataTable(); dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime))); dtBulk.Columns.Add(new DataColumn("file_name", typeof(String))); dtBulk.Columns.Add(new DataColumn("resource_plugin", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("import_id", typeof(String))); dtBulk.Columns.Add(new DataColumn("package_id", typeof(String))); dtBulk.Columns.Add(new DataColumn("package", typeof(String))); dtBulk.Columns.Add(new DataColumn("status", typeof(String))); foreach (String[] dr in jData.data) { PluginConnectorBaseImportPackageUser pkg = JSON.DeserializeFromBase64 <PluginConnectorBaseImportPackageUser>(dr[pkgCol]); dtBulk.Rows.Add(new Object[] { DateTime.Now, f.Name, dr[resourcePluginCol], pkg.importId, pkg.pkgId, JSON.Serialize2(pkg), 'F' }); try { DbParameterCollection par = new DbParameterCollection(); par.Add("@date", typeof(DateTime)).Value = pkg.GetBuildDate(); par.Add("@package_id", typeof(String), pkg.pkgId.Length).Value = pkg.pkgId; Int64 trackId = db.ExecuteScalar <Int64>("select id from st_package_track where flow = 'inbound' and date = @date and package_id = @package_id", System.Data.CommandType.Text, par, null); db.AddPackageTrack(trackId, "inbound", "Package imported to process queue"); } catch { } } db.BulkCopy(dtBulk, "collector_imports"); //Apaga todos os registros da tabela temporaria /* * Procedimento desabiliato em 2018-08-29 por suspeita de problema * db.ExecuteNonQuery("delete from collector_imports_temp", System.Data.CommandType.Text, null, null); * * db.BulkCopy(dtBulk, "collector_imports_temp"); * * //Proteção contra reimportação de pacotes (loop) * db.ExecuteNonQuery("delete from collector_imports_temp where exists (select 1 from collector_imports_old o where o.date >= dateadd(day,-1,getdate()) and o.file_name = file_name and o.resource_plugin_id = resource_plugin_id and o.import_id = import_id and o.package_id = package_id)", System.Data.CommandType.Text, null, null); * db.ExecuteNonQuery("delete from collector_imports_temp where exists (select 1 from collector_imports o where o.date >= dateadd(day,-1,getdate()) and o.file_name = file_name and o.resource_plugin_id = resource_plugin_id and o.import_id = import_id and o.package_id = package_id)", System.Data.CommandType.Text, null, null); * * db.ExecuteNonQuery("insert into collector_imports select * from collector_imports_temp", System.Data.CommandType.Text, null, null); * db.ExecuteNonQuery("delete from collector_imports_temp", System.Data.CommandType.Text, null, null); * */ //Atualiza os registros importados deste arquivo para liberar o processamento //Isso avisa o sistema que estes registros estão livres para processamento //*** Desabilitado essa funç~~ao em 2018-03-08, e colocado o registro para ser importado diretamente com o Status 'F' //db.ExecuteNonQuery("update collector_imports set status = 'F' where [file_name] = '" + f.Name + "'", CommandType.Text, null); //Realiza o rebuild do indice desta tabela para agilizar no engine //Este processo será executado somente uma vez pelo objeto pai //db.ExecuteNonQuery("sp_reindex_imports", CommandType.StoredProcedure, null); #if DEBUG TextLog.Log("Inbound", "\t[ImportRegistersV2] Imported " + dtBulk.Rows.Count + " registers for enterprise " + req.enterpriseid + " and proxy " + req.host); #endif dtBulk.Dispose(); dtBulk = null; jData = null; }
private void ImportNotify(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db) { Int32 resourceCol = jData.GetKeyIndex("resource"); Int32 sourceCol = jData.GetKeyIndex("source"); Int32 uriCol = jData.GetKeyIndex("uri"); Int32 entityIdCol = jData.GetKeyIndex("entityid"); if (resourceCol == -1) { TextLog.Log("Inbound", "\t[ImportNotify] Erro on find column 'resource' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (sourceCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'source' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (uriCol == -1) { TextLog.Log("Inbound", "\t[ImportNotify] Erro on find column 'uri' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (entityIdCol == -1) { TextLog.Log("Inbound", "\t[ImportNotify] Erro on find column 'entityId' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } DateTime date = DateTime.Now; //Realiza a importação no modelo BulkInsert por melhor desempenho do banco DataTable dtBulk = new DataTable(); dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime))); dtBulk.Columns.Add(new DataColumn("source", typeof(String))); dtBulk.Columns.Add(new DataColumn("plugin_uri", typeof(String))); dtBulk.Columns.Add(new DataColumn("resource_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("entity_id", typeof(Int64))); foreach (String[] dr in jData.data) { dtBulk.Rows.Add(new Object[] { date, dr[sourceCol], dr[uriCol], Int64.Parse(dr[resourceCol]), Int64.Parse(dr[entityIdCol]) }); } db.BulkCopy(dtBulk, "notify_imports"); #if DEBUG TextLog.Log("Inbound", "\t[ImportNotify] Imported " + dtBulk.Rows.Count + " notify for enterprise " + req.enterpriseid + " and proxy " + req.host); #endif dtBulk.Dispose(); dtBulk = null; jData = null; }
private void ImportRegistersOLD(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db) { Int32 resourceCol = jData.GetKeyIndex("resource"); Int32 uriCol = jData.GetKeyIndex("uri"); Int32 importidCol = jData.GetKeyIndex("importid"); Int32 registryidCol = jData.GetKeyIndex("registryid"); Int32 datanameCol = jData.GetKeyIndex("dataname"); Int32 datavalueCol = jData.GetKeyIndex("datavalue"); Int32 datatypeCol = jData.GetKeyIndex("datatype"); if (resourceCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'resource' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (uriCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'uri' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (importidCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'importid' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (registryidCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'registryid' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (datanameCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'dataname' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (datavalueCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'datavalue' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (datatypeCol == -1) { TextLog.Log("Inbound", "\t[ImportRegisters] Erro on find column 'datatype' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } DateTime date = DateTime.Now; //Realiza a importação no modelo BulkInsert por melhor desempenho do banco DataTable dtBulk = new DataTable(); dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime))); dtBulk.Columns.Add(new DataColumn("file_name", typeof(String))); dtBulk.Columns.Add(new DataColumn("plugin_uri", typeof(String))); dtBulk.Columns.Add(new DataColumn("resource_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("import_id", typeof(String))); dtBulk.Columns.Add(new DataColumn("registry_id", typeof(String))); dtBulk.Columns.Add(new DataColumn("data_name", typeof(String))); dtBulk.Columns.Add(new DataColumn("data_value", typeof(String))); dtBulk.Columns.Add(new DataColumn("data_type", typeof(String))); foreach (String[] dr in jData.data) { dtBulk.Rows.Add(new Object[] { date, f.Name, dr[uriCol], Int64.Parse(dr[resourceCol]), dr[importidCol], dr[registryidCol], dr[datanameCol], dr[datavalueCol], dr[datatypeCol] }); } db.BulkCopy(dtBulk, "collector_imports"); //Atualiza os registros importados deste arquivo para liberar o processamento //Isso avisa o sistema que estes registros estão livres para processamento db.ExecuteNonQuery("update collector_imports set status = 'F' where [file_name] = '" + f.Name + "'", CommandType.Text, null); //Realiza o rebuild do indice desta tabela para agilizar no engine //Este processo será executado somente uma vez pelo objeto pai //db.ExecuteNonQuery("sp_reindex_imports", CommandType.StoredProcedure, null); #if DEBUG TextLog.Log("Inbound", "\t[ImportRegisters] Imported " + dtBulk.Rows.Count + " registers for enterprise " + req.enterpriseid + " and proxy " + req.host); #endif dtBulk.Dispose(); dtBulk = null; jData = null; }
private void ImportLogs(ProxyConfig config, JsonGeneric jData, FileInfo f, JSONRequest req, IAMDatabase db) { Int32 resourceCol = jData.GetKeyIndex("resource"); Int32 dateCol = jData.GetKeyIndex("date"); Int32 sourceCol = jData.GetKeyIndex("source"); Int32 keyCol = jData.GetKeyIndex("key"); Int32 uriCol = jData.GetKeyIndex("uri"); Int32 typeCol = jData.GetKeyIndex("type"); Int32 entityIdCol = jData.GetKeyIndex("entityid"); Int32 identityIdCol = jData.GetKeyIndex("identityid"); Int32 textCol = jData.GetKeyIndex("text"); Int32 additionaldataCol = jData.GetKeyIndex("additionaldata"); if (resourceCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'resource' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (sourceCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'source' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (keyCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'key' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (uriCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'uri' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (entityIdCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'entityId' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (identityIdCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'identityId' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } if (textCol == -1) { TextLog.Log("Inbound", "\t[ImportLogs] Erro on find column 'text' in " + f.Name + " enterprise " + req.enterpriseid + " and proxy " + req.host); return; } DateTime date = DateTime.Now; //Realiza a importação no modelo BulkInsert por melhor desempenho do banco DataTable dtBulk = new DataTable(); dtBulk.Columns.Add(new DataColumn("date", typeof(DateTime))); dtBulk.Columns.Add(new DataColumn("source", typeof(String))); dtBulk.Columns.Add(new DataColumn("key", typeof(Int32))); dtBulk.Columns.Add(new DataColumn("enterprise_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("proxy_name", typeof(String))); dtBulk.Columns.Add(new DataColumn("proxy_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("plugin_uri", typeof(String))); dtBulk.Columns.Add(new DataColumn("plugin_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("resource_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("entity_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("identity_id", typeof(Int64))); dtBulk.Columns.Add(new DataColumn("type", typeof(String))); dtBulk.Columns.Add(new DataColumn("text", typeof(String))); dtBulk.Columns.Add(new DataColumn("additional_data", typeof(String))); foreach (String[] dr in jData.data) { try { //Console.WriteLine(f.Name + " - " + dr[entityIdCol] + " ==> " + dr[textCol]); //Console.WriteLine(dr[additionaldataCol]); //Console.WriteLine(""); dtBulk.Rows.Add(new Object[] { (dateCol >= 0 ? DateTime.Parse(dr[dateCol]) : date), dr[sourceCol], dr[keyCol], req.enterpriseid, req.host, 0, dr[uriCol], 0, Int64.Parse(dr[resourceCol]), Int64.Parse(dr[entityIdCol]), Int64.Parse(dr[identityIdCol]), dr[typeCol], dr[textCol], (additionaldataCol >= 0 ? dr[additionaldataCol] : "") }); } catch (Exception ex) { throw ex; } } db.BulkCopy(dtBulk, "logs_imports"); //Procedure que processa os logs e importa para a tabela definitiva db.ExecuteNonQuery("sp_process_logs", CommandType.StoredProcedure, null); #if debug db.AddUserLog(LogKey.Import, null, "Inbound", UserLogLevel.Info, 0, 0, 0, 0, 0, 0, 0, "Imported " + dtBulk.Rows.Count + " logs for enterprise " + req.enterpriseid + " and proxy " + req.host + " from file " + f.Name); TextLog.Log("Inbound", "\t[ImportLogs] Imported " + dtBulk.Rows.Count + " logs for enterprise " + req.enterpriseid + " and proxy " + req.host); #endif dtBulk.Dispose(); dtBulk = null; jData = null; }