private static void processaCsvZipado(SqliteDB db, string zipName, Func <string, bool> fileFilter, Func <string[], ReceitasModel> extractionFilter) { var zip = new LeitorZipTransparencia(zipName); // Receitas, Brasil, agnóstico ao ano zip.ShouldProcessFile = fileFilter; zip.InicioLeituraArquivo += (s, e) => Console.WriteLine(e); zip.IgnoreFirstLine = true; var zipLines = zip.ReadLines(); var rows = CSVHelper.DelimiterSplit(zipLines, ';'); var buffer = new DataBuffer <ReceitasModel>(30000, data => { db.BulkInsert(data, addReplace: true); }); int qtd = 0; foreach (var row in rows) { var cad = extractionFilter(row); buffer.Add(cad); qtd++; if (qtd % buffer.Quantity == 0) { Console.WriteLine($"{DateTime.Now.ToLongTimeString()} Processado: {qtd:N0} {cad.NomeDoadorRFB}"); } } buffer.Flush(); }
private static void processaArquivo(string zipName, bool militar, Simple.Sqlite.SqliteDB db) { var zip = new LeitorZipTransparencia(zipName); zip.ShouldProcessFile = n => n.Contains("_Cadastro"); zip.IgnoreFirstLine = true; zip.InicioLeituraArquivo += (s, a) => Console.WriteLine($"Lendo {a}"); var zipLines = zip.ReadLines(); var rows = CSVHelper.DelimiterSplit(zipLines, ';'); var buffer = new DataBuffer <ServidoresCadastroModel>(10000, data => db.BulkInsert(data, addReplace: true)); foreach (var row in rows) { var cad = new ServidoresCadastroModel() { ServidorCadastroMilitar = militar, ID_ServidorPortal = row[0].ToInt(), Nome = row[1], CPF_6D = row[2], Matricula_3D = row[3], CodigoOrgaoSuperiorLotacao = row[19].ToInt(), CodigoOrgaoLotacao = row[17].ToInt(), DocumentoIngressoServicoPublico = row[36], TipoVinculo = row[27].ToInt(), }; if (uids.Contains(cad.UIID)) { continue; } uids.Add(cad.UIID); buffer.Add(cad); qtd++; if (qtd % buffer.Quantity == 0) { Console.WriteLine($"{DateTime.Now.ToLongTimeString()} Processado: {qtd}"); } } buffer.Flush(); }
private static void processaArquivo(string zipName, Simple.Sqlite.SqliteDB db) { var zip = new LeitorZipTransparencia(zipName); //zip.ShouldProcessFile = n => n.Contains("_Cadastro"); zip.IgnoreFirstLine = true; zip.InicioLeituraArquivo += (s, a) => Console.WriteLine($"Lendo {a}"); var zipLines = zip.ReadLines(); var rows = CSVHelper.DelimiterSplit(zipLines, ';'); var buffer = new DataBuffer <AuxilioModel>(20000, data => { db.BulkInsert(data, addReplace: true); Console.WriteLine($"# Data Write "); }); foreach (var row in rows) { var cad = new AuxilioModel() { UF = row[1], Municipio = row[2].ToInt(0), NIS = row[4], CPF_6D = row[5], Nome = row[6], NIS_Responsavel = row[7], CPF_6D_Responsavel = row[8], Nome_Responsavel = row[9], Enquadramento = row[10], }; if (cad.NIS == "00000000000") { cad.NIS = null; } if (cad.NIS_Responsavel == "-2") { cad.NIS_Responsavel = null; } if (cad.Nome_Responsavel == "Não se aplica") { cad.Nome_Responsavel = null; } if (cad.NIS_Responsavel == cad.NIS) { cad.NIS_Responsavel = null; } if (cad.CPF_6D_Responsavel == cad.NIS) { cad.CPF_6D_Responsavel = null; } if (cad.Nome_Responsavel == cad.Nome) { cad.Nome_Responsavel = null; } qtd++; if (qtd % buffer.Quantity == 0) { Console.WriteLine($"{DateTime.Now.ToLongTimeString()} Processado: {qtd:N0} {cad.Nome}"); } if (uids.Contains(cad.Key())) { continue; } uids.Add(cad.Key()); buffer.Add(cad); } buffer.Flush(); }