private FileObjectCollection PopulateCollection <T>(FileObject fileObject, IList <T> records) where T : IRecord
{
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
TotalFilesFormatted++;
TotalRecords += records.Count;
if (Config.IsKustoConfigured())
{
// kusto native format is Csv
// kusto json ingest is 2 to 3 times slower and does *not* use standard json format. uses json document per line no comma
// using csv and compression for best performance
collection = SerializeCsv(fileObject, records);
if (Config.KustoCompressed)
{
collection.ForEach(x => x.Stream.Compress());
}
}
if (Config.IsLogAnalyticsConfigured())
{
// la is kusto based but only accepts non compressed json format ingest
collection = SerializeJson(fileObject, records);
}
collection.ForEach(x => SaveToCache(x));
records.Clear();
return(collection);
}
public void ImportJson(FileObjectCollection fileObjectCollection)
{
fileObjectCollection.ForEach(x => ImportJson(x));
}
public void IngestMultipleFiles(FileObjectCollection fileObjectCollection)
{
fileObjectCollection.ForEach(x => IngestSingleFile(x));
}