private static void processaCsvZipado(SqliteDB db, string zipName, Func <string, bool> fileFilter, Func <string[], ReceitasModel> extractionFilter)
{
var zip = new LeitorZipTransparencia(zipName);
// Receitas, Brasil, agnóstico ao ano
zip.ShouldProcessFile = fileFilter;
zip.InicioLeituraArquivo += (s, e) => Console.WriteLine(e);
zip.IgnoreFirstLine = true;
var zipLines = zip.ReadLines();
var rows = CSVHelper.DelimiterSplit(zipLines, ';');
var buffer = new DataBuffer <ReceitasModel>(30000, data =>
{
db.BulkInsert(data, addReplace: true);
});
int qtd = 0;
foreach (var row in rows)
{
var cad = extractionFilter(row);
buffer.Add(cad);
qtd++;
if (qtd % buffer.Quantity == 0)
{
Console.WriteLine($"{DateTime.Now.ToLongTimeString()} Processado: {qtd:N0} {cad.NomeDoadorRFB}");
}
}
buffer.Flush();
}
public void CSVHelpers_DelimiterSplitTests_Base()
{
string delimiter = "abc;\"def\";ghi\n123;'456';\"789\"";
var lines = delimiter.Split('\n');
var items = CSVHelper.DelimiterSplit(lines, ';').ToArray();
Assert.Equal(2, items.Length);
Assert.Equal(new string[] { "abc", "def", "ghi" }, items[0]);
Assert.Equal(new string[] { "123", "'456'", "789" }, items[1]);
}
private static void processaArquivo(string zipName, bool militar, Simple.Sqlite.SqliteDB db)
{
var zip = new LeitorZipTransparencia(zipName);
zip.ShouldProcessFile = n => n.Contains("_Cadastro");
zip.IgnoreFirstLine = true;
zip.InicioLeituraArquivo += (s, a) => Console.WriteLine($"Lendo {a}");
var zipLines = zip.ReadLines();
var rows = CSVHelper.DelimiterSplit(zipLines, ';');
var buffer = new DataBuffer <ServidoresCadastroModel>(10000, data => db.BulkInsert(data, addReplace: true));
foreach (var row in rows)
{
var cad = new ServidoresCadastroModel()
{
ServidorCadastroMilitar = militar,
ID_ServidorPortal = row[0].ToInt(),
Nome = row[1],
CPF_6D = row[2],
Matricula_3D = row[3],
CodigoOrgaoSuperiorLotacao = row[19].ToInt(),
CodigoOrgaoLotacao = row[17].ToInt(),
DocumentoIngressoServicoPublico = row[36],
TipoVinculo = row[27].ToInt(),
};
if (uids.Contains(cad.UIID))
{
continue;
}
uids.Add(cad.UIID);
buffer.Add(cad);
qtd++;
if (qtd % buffer.Quantity == 0)
{
Console.WriteLine($"{DateTime.Now.ToLongTimeString()} Processado: {qtd}");
}
}
buffer.Flush();
}
private static void processaArquivo(string zipName, Simple.Sqlite.SqliteDB db)
{
var zip = new LeitorZipTransparencia(zipName);
//zip.ShouldProcessFile = n => n.Contains("_Cadastro");
zip.IgnoreFirstLine = true;
zip.InicioLeituraArquivo += (s, a) => Console.WriteLine($"Lendo {a}");
var zipLines = zip.ReadLines();
var rows = CSVHelper.DelimiterSplit(zipLines, ';');
var buffer = new DataBuffer <AuxilioModel>(20000, data =>
{
db.BulkInsert(data, addReplace: true);
Console.WriteLine($"# Data Write ");
});
foreach (var row in rows)
{
var cad = new AuxilioModel()
{
UF = row[1],
Municipio = row[2].ToInt(0),
NIS = row[4],
CPF_6D = row[5],
Nome = row[6],
NIS_Responsavel = row[7],
CPF_6D_Responsavel = row[8],
Nome_Responsavel = row[9],
Enquadramento = row[10],
};
if (cad.NIS == "00000000000")
{
cad.NIS = null;
}
if (cad.NIS_Responsavel == "-2")
{
cad.NIS_Responsavel = null;
}
if (cad.Nome_Responsavel == "Não se aplica")
{
cad.Nome_Responsavel = null;
}
if (cad.NIS_Responsavel == cad.NIS)
{
cad.NIS_Responsavel = null;
}
if (cad.CPF_6D_Responsavel == cad.NIS)
{
cad.CPF_6D_Responsavel = null;
}
if (cad.Nome_Responsavel == cad.Nome)
{
cad.Nome_Responsavel = null;
}
qtd++;
if (qtd % buffer.Quantity == 0)
{
Console.WriteLine($"{DateTime.Now.ToLongTimeString()} Processado: {qtd:N0} {cad.Nome}");
}
if (uids.Contains(cad.Key()))
{
continue;
}
uids.Add(cad.Key());
buffer.Add(cad);
}
buffer.Flush();
}