private static BlockingKustoUploader CreateUploader(
TimeSpan flushDuration,
string _outputFileName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
var ku = new BlockingKustoUploader(
_outputFileName, kscbAdmin, kscbIngest, _demoMode, _tableName, 10000, flushDuration, _resetTable);
return(ku);
}
private static void RunUploader(BlockingKustoUploader ku, IObservable <IDictionary <string, object> > etw, string _queryFile)
{
if (_queryFile == null)
{
using (etw.Subscribe(ku))
{
ku.Completed.WaitOne();
}
}
else
{
KqlNode preProcessor = new KqlNode();
preProcessor.KqlKqlQueryFailed += PreProcessor_KqlKqlQueryFailed;
ScalarFunctionFactory.AddFunctions(typeof(CustomScalarFunctions));
preProcessor.AddCslFile(_queryFile);
if (preProcessor.FailedKqlQueryList.Count > 0)
{
foreach (var failedDetection in preProcessor.FailedKqlQueryList)
{
Console.WriteLine($"Message: {failedDetection.Message}");
}
}
// If we have atleast one valid detection there is a point in waiting otherwise exit
if (preProcessor.KqlQueryList.Count > 0)
{
var processed = preProcessor.Output.Select(e => e.Output);
using (processed.Subscribe(ku))
{
using (etw.Subscribe(preProcessor))
{
ku.Completed.WaitOne();
}
}
}
else
{
Console.WriteLine("No Queries are running. Press Enter to terminate");
}
}
}
static void ProcessSyslogRealTime(
string logFileName,
string listenerAdapterName,
int listenerUdpPort,
string queryFile,
string consoleLogOption,
string outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool directIngest,
string tableName,
bool resetTable)
{
SyslogListener listener = null;
SyslogFileListener fileListener = null;
bool fileReadMode = false;
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
var parser = CreateSIEMfxSyslogParser();
var _converter = new SyslogEntryToRecordConverter();
// input
if (string.IsNullOrEmpty(logFileName))
{
// reading from local port
IPAddress localIp = null;
if (!string.IsNullOrEmpty(listenerAdapterName))
{
localIp = GetLocalIp(listenerAdapterName);
}
localIp ??= IPAddress.IPv6Any;
var endPoint = new IPEndPoint(localIp, listenerUdpPort);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
listener = new SyslogListener(parser, PortListener);
var filter = new SyslogFilter();
if (filter != null)
{
listener.Filter = filter.Allow;
}
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
listener.Subscribe(_converter);
listener.Start();
}
else
{
// reading from local file
var fileStream = new FileStream(logFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
fileListener = new SyslogFileListener(parser, fileStream);
fileReadMode = true;
var filter = new SyslogFilter();
if (filter != null)
{
fileListener.Filter = filter.Allow;
}
fileListener.Error += FileListener_Error;
fileListener.EntryReceived += FileListener_EntryReceived;
fileListener.Subscribe(_converter);
fileListener.Start();
}
Console.WriteLine();
Console.WriteLine("Listening to Syslog events. Press any key to terminate");
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, directIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, _converter, queryFile);
});
}
else if (!string.IsNullOrEmpty(outputFileName))
{
// output to file
fileOutput = new FileOutput(outputFileName);
RunFileOutput(fileOutput, _converter, queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, _converter, queryFile);
}
string readline = Console.ReadLine();
// clean up
if (!fileReadMode)
{
listener.Stop();
listener.Dispose();
listener = null;
}
else
{
fileListener.Stop();
fileListener.Dispose();
fileListener = null;
}
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
static void UploadRealTime(
string _sessionName,
string _queryFile,
string consoleLogOption,
string _outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
var etw = Tx.Windows.EtwTdhObservable.FromSession(_sessionName);
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
Console.WriteLine();
Console.WriteLine("Listening to real-time session '{0}'. Press Enter to terminate", _sessionName);
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, etw, _queryFile);
});
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
// output to file
fileOutput = new FileOutput(_outputFileName);
RunFileOutput(fileOutput, etw, _queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, etw, _queryFile);
}
string readline = Console.ReadLine();
// clean up
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
private static void UploadUsingWecFile(
string _wecFile,
string _logName,
string _queryFile,
bool _readExisting,
string consoleLogOption,
string _outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
IObservable <IDictionary <string, object> > etw;
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
if (!string.IsNullOrEmpty(_wecFile))
{
if (!File.Exists(_wecFile))
{
Console.WriteLine("Wec File doesnt exist!");
return;
}
string _wecFileContent = File.ReadAllText(_wecFile);
etw = Tx.Windows.EvtxObservable.FromLog(_logName, _wecFileContent, _readExisting, null).Select(x => x.Deserialize());
Console.WriteLine();
Console.WriteLine("Listening using WecFile '{0}'. Press Enter to terminate", _wecFile);
}
else
{
etw = Tx.Windows.EvtxObservable.FromLog(_logName, null, _readExisting).Select(x => x.Deserialize());
Console.WriteLine();
Console.WriteLine("Listening using Log Name '{0}'. Press Enter to terminate", _logName);
}
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, etw, _queryFile);
});
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
// output to file
fileOutput = new FileOutput(_outputFileName);
RunFileOutput(fileOutput, etw, _queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, etw, _queryFile);
}
string readline = Console.ReadLine();
// clean up
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
private static void ProcessCsvRealTime(
string fileName,
string queryFile,
string consoleLogOption,
string outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool directIngest,
string tableName,
bool resetTable)
{
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
// initiating simple csv reader
SimpleCsvParser simpleCsvParser = new SimpleCsvParser(fileName);
// initiating output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, directIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, simpleCsvParser, queryFile);
});
}
else if (!string.IsNullOrEmpty(outputFileName))
{
// output to file
fileOutput = new FileOutput(outputFileName);
RunFileOutput(fileOutput, simpleCsvParser, queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, simpleCsvParser, queryFile);
}
// starting simple csv reader
simpleCsvParser.Start();
string readline = Console.ReadLine();
// clean up
simpleCsvParser.Stop();
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
