private static void UploadEntireFileInBatches(string fileFullName, X509Certificate2 cert,
XmlCreationMechanism creationMechanism, int batchCount = 200)
{
var payload = GetNewPayloadObject();
var useEventIngest = false;
// Set the ResourceId for upload
ResourceId = payload.GetLogAnalyticsResourceId(SentinelApiConfig.WorkspaceId);
var fileStopwatch = new Stopwatch();
var uploaderStopwatch = Stopwatch.StartNew();
try
{
fileStopwatch.Start();
var log = EvtxEnumerable.ReadEvtxFile(fileFullName);
Parallel.ForEach(log, new ParallelOptions
{
MaxDegreeOfParallelism = 8
},
eventRecord => { payload.AddEvent(eventRecord, useEventIngest, creationMechanism); });
fileStopwatch.Stop();
Console.WriteLine($"\tRecordCount: {payload.DataItems.Count:N0}");
var output =
$"\tEPS for Conversion: {payload.DataItems.Count / fileStopwatch.Elapsed.TotalSeconds:N3}";
Console.WriteLine(output);
// Split into upload chunks
var splitLIsts = payload.SplitListIntoChunks <string>(batchCount);
fileStopwatch.Restart();
Parallel.ForEach(splitLIsts, new ParallelOptions
{
MaxDegreeOfParallelism = 8
},
singleBatch => { UploadBatchToLogAnalytics(payload.GetUploadBatch(singleBatch), cert); });
fileStopwatch.Stop();
Console.WriteLine(
$"\tEPS for Upload to MMA-API: {payload.DataItems.Count / fileStopwatch.Elapsed.TotalSeconds:N3}");
}
catch (Exception e)
{
GlobalLog.WriteToStringBuilderLog(e.ToString(), 14008);
}
}
public static void WindowsEventsFolderContents()
{
DirectoryInfo d = new DirectoryInfo($@"D:\OSSCWec\LAFiles");
XmlCreationMechanism createMechanism = XmlCreationMechanism.XmlWriter;
FileInfo[] Files = d.GetFiles("Archive*.evtx"); //Getting Text files
Console.WriteLine($"Attempting to upload {Files.Length}");
foreach (FileInfo file in Files)
{
Console.WriteLine($"FileName: {file.FullName}");
Console.WriteLine($"\tUploading file with : {createMechanism.ToString()}", 10003);
UploadEntireFileInBatches(file.FullName, createMechanism);
if (File.Exists(file.FullName))
{
Console.WriteLine($"\tDeleting File: {file.FullName}");
File.Delete(file.FullName);
}
}
}
public void AddEvent(EventRecord eventRecord, bool useEventIngest, XmlCreationMechanism xmlCreationMechanism)
{
switch (xmlCreationMechanism)
{
case XmlCreationMechanism.StringReplacement:
DateTime timeCreated = (DateTime)eventRecord.TimeCreated;
string tempWinEvent = DataItemTemplate;
tempWinEvent = tempWinEvent.Replace("{WorkspaceId}", WorkspaceId);
tempWinEvent = tempWinEvent.Replace("{ProviderGuid}", (eventRecord.ProviderId ?? Guid.Empty).ToString());
tempWinEvent = tempWinEvent.Replace("{Provider}", eventRecord.ProviderName);
tempWinEvent = tempWinEvent.Replace("{EventSource}", eventRecord.ProviderName);
tempWinEvent = tempWinEvent.Replace("{Channel}", eventRecord.LogName ?? "Unknown");
tempWinEvent = tempWinEvent.Replace("{Computer}", eventRecord.MachineName);
tempWinEvent = tempWinEvent.Replace("{EventId}", eventRecord.Id.ToString());
tempWinEvent = tempWinEvent.Replace("{EventCategory}", (eventRecord.Task ?? 0).ToString());
tempWinEvent = tempWinEvent.Replace("{EventLevel}", (eventRecord.Level ?? 0).ToString());
tempWinEvent = tempWinEvent.Replace("{EventTimeUTC}", $"{timeCreated.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.ffffffZ}");
tempWinEvent = tempWinEvent.Replace("{EventData}", WinLog.LogReader.RetrieveExtendedData(eventRecord.ToXml()));
AddToPayload(tempWinEvent, useEventIngest);
break;
case XmlCreationMechanism.XElement:
//var taskReturnValue = TestWriter(eventRecord);
var returnValue = LinqXElementWriter(eventRecord);
returnValue = returnValue.Replace("<", "<").Replace(">", ">");
AddToPayload(returnValue, useEventIngest);
break;
case XmlCreationMechanism.XmlWriter:
var returnXmlWriterValue = XmlWriterEventRecord(eventRecord);
AddToPayload(returnXmlWriterValue, useEventIngest);
break;
default:
throw new ArgumentOutOfRangeException(nameof(xmlCreationMechanism), xmlCreationMechanism, null);
}
}
private static void UploadEntireFileInBatches(string fileFullName, XmlCreationMechanism creationMechanism, int batchCount = 200)
{
WindowsEventPayload payload = GetNewPayloadObject();
bool useEventIngest = false;
// Set the ResourceId for upload
ResourceId = payload.GetLogAnalyticsResourceId(SentinelApiConfig.WorkspaceId);
Stopwatch fileStopwatch = new Stopwatch();
Stopwatch uploaderStopwatch = Stopwatch.StartNew();
try
{
fileStopwatch.Start();
var log = EvtxEnumerable.ReadEvtxFile(fileFullName);
Parallel.ForEach(log, new ParallelOptions
{
MaxDegreeOfParallelism = 8,
},
eventRecord => { payload.AddEvent(eventRecord, useEventIngest, creationMechanism); });
fileStopwatch.Stop();
if (useEventIngest)
{
//Console.WriteLine($"\tRecordCount: {payload.Uploader.ItemCount:N0}");
//Console.WriteLine(
// $"\tEPS for Conversion: {payload.Uploader.ItemCount / fileStopwatch.Elapsed.TotalSeconds:N3}");
//// Wait for upload to complete, and report
//payload.Uploader.OnCompleted();
//uploaderStopwatch.Stop();
//Console.WriteLine($"Upload Completed...");
//Console.WriteLine($"\tEPS for Upload with Event.Ingest to MMA-API: {payload.Uploader.ItemCount / uploaderStopwatch.Elapsed.TotalSeconds:N3}");
//Console.WriteLine($"\t Average for batch with Event.Ingest to MMA-API: {payload.BatchItemCount / payload.BatchTimeSpan.TotalSeconds:N3}");
}
else
{
Console.WriteLine($"\tRecordCount: {payload.DataItems.Count:N0}");
string output =
$"\tEPS for Conversion: {payload.DataItems.Count / fileStopwatch.Elapsed.TotalSeconds:N3}";
Console.WriteLine(output);
}
// Split into upload chunks
var splitLIsts = payload.SplitListIntoChunks <string>(batchCount);
fileStopwatch.Restart();
Parallel.ForEach(splitLIsts, new ParallelOptions
{
MaxDegreeOfParallelism = 8,
},
singleBatch => { UploadBatchToLogAnalytics(payload.GetUploadBatch(singleBatch), AuthX509Certificate2); });
fileStopwatch.Stop();
Console.WriteLine($"\tEPS for Upload to MMA-API: {payload.DataItems.Count / fileStopwatch.Elapsed.TotalSeconds:N3}");
}
catch (Exception e)
{
GlobalLog.WriteToStringBuilderLog(e.ToString(), 14008);
}
}