private static void UploadEntireFileInBatches(string fileFullName, XmlCreationMechanism creationMechanism, int batchCount = 200)
{
WindowsEventPayload payload = GetNewPayloadObject();
bool useEventIngest = false;
// Set the ResourceId for upload
ResourceId = payload.GetLogAnalyticsResourceId(SentinelApiConfig.WorkspaceId);
Stopwatch fileStopwatch = new Stopwatch();
Stopwatch uploaderStopwatch = Stopwatch.StartNew();
try
{
fileStopwatch.Start();
var log = EvtxEnumerable.ReadEvtxFile(fileFullName);
Parallel.ForEach(log, new ParallelOptions
{
MaxDegreeOfParallelism = 8,
},
eventRecord => { payload.AddEvent(eventRecord, useEventIngest, creationMechanism); });
fileStopwatch.Stop();
if (useEventIngest)
{
//Console.WriteLine($"\tRecordCount: {payload.Uploader.ItemCount:N0}");
//Console.WriteLine(
// $"\tEPS for Conversion: {payload.Uploader.ItemCount / fileStopwatch.Elapsed.TotalSeconds:N3}");
//// Wait for upload to complete, and report
//payload.Uploader.OnCompleted();
//uploaderStopwatch.Stop();
//Console.WriteLine($"Upload Completed...");
//Console.WriteLine($"\tEPS for Upload with Event.Ingest to MMA-API: {payload.Uploader.ItemCount / uploaderStopwatch.Elapsed.TotalSeconds:N3}");
//Console.WriteLine($"\t Average for batch with Event.Ingest to MMA-API: {payload.BatchItemCount / payload.BatchTimeSpan.TotalSeconds:N3}");
}
else
{
Console.WriteLine($"\tRecordCount: {payload.DataItems.Count:N0}");
string output =
$"\tEPS for Conversion: {payload.DataItems.Count / fileStopwatch.Elapsed.TotalSeconds:N3}";
Console.WriteLine(output);
}
// Split into upload chunks
var splitLIsts = payload.SplitListIntoChunks <string>(batchCount);
fileStopwatch.Restart();
Parallel.ForEach(splitLIsts, new ParallelOptions
{
MaxDegreeOfParallelism = 8,
},
singleBatch => { UploadBatchToLogAnalytics(payload.GetUploadBatch(singleBatch), AuthX509Certificate2); });
fileStopwatch.Stop();
Console.WriteLine($"\tEPS for Upload to MMA-API: {payload.DataItems.Count / fileStopwatch.Elapsed.TotalSeconds:N3}");
}
catch (Exception e)
{
GlobalLog.WriteToStringBuilderLog(e.ToString(), 14008);
}
}
