static void ProcessSyslogRealTime(
string logFileName,
string listenerAdapterName,
int listenerUdpPort,
string queryFile,
string consoleLogOption,
string outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool directIngest,
string tableName,
bool resetTable)
{
SyslogListener listener = null;
SyslogFileListener fileListener = null;
bool fileReadMode = false;
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
var parser = CreateSIEMfxSyslogParser();
var _converter = new SyslogEntryToRecordConverter();
// input
if (string.IsNullOrEmpty(logFileName))
{
// reading from local port
IPAddress localIp = null;
if (!string.IsNullOrEmpty(listenerAdapterName))
{
localIp = GetLocalIp(listenerAdapterName);
}
localIp ??= IPAddress.IPv6Any;
var endPoint = new IPEndPoint(localIp, listenerUdpPort);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
listener = new SyslogListener(parser, PortListener);
var filter = new SyslogFilter();
if (filter != null)
{
listener.Filter = filter.Allow;
}
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
listener.Subscribe(_converter);
listener.Start();
}
else
{
// reading from local file
var fileStream = new FileStream(logFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
fileListener = new SyslogFileListener(parser, fileStream);
fileReadMode = true;
var filter = new SyslogFilter();
if (filter != null)
{
fileListener.Filter = filter.Allow;
}
fileListener.Error += FileListener_Error;
fileListener.EntryReceived += FileListener_EntryReceived;
fileListener.Subscribe(_converter);
fileListener.Start();
}
Console.WriteLine();
Console.WriteLine("Listening to Syslog events. Press any key to terminate");
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, directIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, _converter, queryFile);
});
}
else if (!string.IsNullOrEmpty(outputFileName))
{
// output to file
fileOutput = new FileOutput(outputFileName);
RunFileOutput(fileOutput, _converter, queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, _converter, queryFile);
}
string readline = Console.ReadLine();
// clean up
if (!fileReadMode)
{
listener.Stop();
listener.Dispose();
listener = null;
}
else
{
fileListener.Stop();
fileListener.Dispose();
fileListener = null;
}
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
static void UploadRealTime(
string _sessionName,
string _queryFile,
string consoleLogOption,
string _outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
var etw = Tx.Windows.EtwTdhObservable.FromSession(_sessionName);
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
Console.WriteLine();
Console.WriteLine("Listening to real-time session '{0}'. Press Enter to terminate", _sessionName);
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, etw, _queryFile);
});
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
// output to file
fileOutput = new FileOutput(_outputFileName);
RunFileOutput(fileOutput, etw, _queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, etw, _queryFile);
}
string readline = Console.ReadLine();
// clean up
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
private static void UploadUsingWecFile(
string _wecFile,
string _logName,
string _queryFile,
bool _readExisting,
string consoleLogOption,
string _outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
IObservable <IDictionary <string, object> > etw;
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
if (!string.IsNullOrEmpty(_wecFile))
{
if (!File.Exists(_wecFile))
{
Console.WriteLine("Wec File doesnt exist!");
return;
}
string _wecFileContent = File.ReadAllText(_wecFile);
etw = Tx.Windows.EvtxObservable.FromLog(_logName, _wecFileContent, _readExisting, null).Select(x => x.Deserialize());
Console.WriteLine();
Console.WriteLine("Listening using WecFile '{0}'. Press Enter to terminate", _wecFile);
}
else
{
etw = Tx.Windows.EvtxObservable.FromLog(_logName, null, _readExisting).Select(x => x.Deserialize());
Console.WriteLine();
Console.WriteLine("Listening using Log Name '{0}'. Press Enter to terminate", _logName);
}
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, etw, _queryFile);
});
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
// output to file
fileOutput = new FileOutput(_outputFileName);
RunFileOutput(fileOutput, etw, _queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, etw, _queryFile);
}
string readline = Console.ReadLine();
// clean up
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
static void UploadEtlFiles(
string _filePattern,
string _queryFile,
string consoleLogOption,
string _outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool _demoMode,
string _tableName,
bool _resetTable)
{
string[] files;
if (Path.IsPathRooted(_filePattern))
{
// Get directory and file parts of complete relative pattern
string pattern = Path.GetFileName(_filePattern);
string relDir = _filePattern.Substring(0, _filePattern.Length - pattern.Length);
// Get absolute path (root+relative)
string rootDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string absPath = Path.GetFullPath(Path.Combine(rootDir, relDir));
// Search files mathing the pattern
files = Directory.GetFiles(absPath, pattern, SearchOption.TopDirectoryOnly);
}
else
{
// input
string rootDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
// Get directory and file parts of complete relative pattern
string pattern = Path.GetFileName(_filePattern);
string relDir = pattern.Substring(0, _filePattern.Length - pattern.Length);
// Get absolute path (root+relative)
string absPath = Path.GetFullPath(Path.Combine(rootDir, relDir));
// Search files mathing the pattern
files = Directory.GetFiles(absPath, pattern, SearchOption.TopDirectoryOnly);
}
if (files != null && files.Length > 0)
{
var etw = Tx.Windows.EtwTdhObservable.FromFiles(files);
if (kscbAdmin != null)
{
// output to kusto
var ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, _demoMode, _tableName, _resetTable);
RunUploader(ku, etw, _queryFile);
}
else if (!string.IsNullOrEmpty(_outputFileName))
{
// output to file
var fileOutput = new FileOutput(_outputFileName);
RunFileOutput(fileOutput, etw, _queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
var consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, etw, _queryFile);
}
}
}
private static void ProcessCsvRealTime(
string fileName,
string queryFile,
string consoleLogOption,
string outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool directIngest,
string tableName,
bool resetTable)
{
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
// initiating simple csv reader
SimpleCsvParser simpleCsvParser = new SimpleCsvParser(fileName);
// initiating output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, directIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, simpleCsvParser, queryFile);
});
}
else if (!string.IsNullOrEmpty(outputFileName))
{
// output to file
fileOutput = new FileOutput(outputFileName);
RunFileOutput(fileOutput, simpleCsvParser, queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, simpleCsvParser, queryFile);
}
// starting simple csv reader
simpleCsvParser.Start();
string readline = Console.ReadLine();
// clean up
simpleCsvParser.Stop();
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
