static void UploadSyslogRealTime(
string listenerAdapterName,
int listenerUdpPort,
string queryFile,
string outputFileName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool quickIngest,
string tableName,
bool resetTable)
{
var parser = CreateSIEMfxSyslogParser();
IPAddress localIp = null;
if (!string.IsNullOrEmpty(listenerAdapterName))
{
localIp = GetLocalIp(listenerAdapterName);
}
localIp ??= IPAddress.IPv6Any;
var endPoint = new IPEndPoint(localIp, listenerUdpPort);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
using var listener = new SyslogListener(parser, PortListener);
var filter = new SyslogFilter();
if (filter != null)
{
listener.Filter = filter.Allow;
}
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
var _converter = new SyslogEntryToRecordConverter();
listener.Subscribe(_converter);
listener.Start();
Console.WriteLine();
Console.WriteLine("Listening to Syslog events. Press any key to terminate");
var ku = CreateUploader(UploadTimespan, outputFileName, kscbAdmin, kscbIngest, quickIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, _converter, queryFile);
});
string readline = Console.ReadLine();
listener.Stop();
ku.OnCompleted();
}
public void Start()
{
var nodeConfigString = File.ReadAllText("config.json");
var settings = new Newtonsoft.Json.JsonSerializerSettings
{
TypeNameHandling = Newtonsoft.Json.TypeNameHandling.Auto,
};
var config = Newtonsoft.Json.JsonConvert.DeserializeObject <JSONConfig>(nodeConfigString, settings);
// config Syslog listener
if (config.syslog.listenerEnabled)
{
var listener = new SyslogListener();
Action <int> startListener = listener.Start;
startListener.BeginInvoke(config.syslog.listenerPort, null, null);
}
// config PollEngine
foreach (var node in (config.nodes ?? new JSONConfigNode[0]))
{
var pollNode = Activator.CreateInstance(typeof(PollingEngine).Assembly.GetType(node.type), new object[] { node.name, node.groups ?? new string[0], node.settings }) as PollNode;
PollingEngine.TryAdd(pollNode);
}
PollingEngine.StartPolling();
// config HTTP API
var hostConfig = new HttpSelfHostConfiguration("http://127.0.0.1:" + config.http.listenerPort);
// Remove the XML formatter
hostConfig.Formatters.Remove(hostConfig.Formatters.XmlFormatter);
hostConfig.Formatters.Add(new RazorFormatter());
hostConfig.Formatters.JsonFormatter.SerializerSettings.TypeNameHandling = Newtonsoft.Json.TypeNameHandling.Auto;
hostConfig.Formatters.JsonFormatter.SerializerSettings.Formatting = Newtonsoft.Json.Formatting.Indented;
hostConfig.Formatters.JsonFormatter.SerializerSettings.Error += (x, y) =>
{
return;
};
hostConfig.Formatters.JsonFormatter.MediaTypeMappings.Add(new QueryStringMapping("type", "json", new MediaTypeHeaderValue("application/json")));
// Attribute routing.
hostConfig.MapHttpAttributeRoutes();
hostConfig.IncludeErrorDetailPolicy = IncludeErrorDetailPolicy.Always;
hostConfig.Filters.Add(new ApiExceptionFilterAttribute());
httpServer = new HttpSelfHostServer(hostConfig);
httpServer.OpenAsync().Wait();
}
public override bool Start()
{
// Setting up pipeline
if (!Start(_eventStream, "syslogserver", true))
{
return(false);
}
// Set up for listening on port
IPAddress localIp = null;
if (!string.IsNullOrEmpty(_adapterName))
{
localIp = GetLocalIp(_adapterName);
}
else
{
localIp = IPAddress.IPv6Any;
}
var endPoint = new IPEndPoint(localIp, _udpport);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
// Setting up syslog parser
var parser = SyslogParser.CreateDefault();
parser.AddValueExtractors(new SyslogKeywordValuesExtractor(), new SyslogPatternBasedValuesExtractor());
// Setting up syslog listener
var listener = new SyslogListener(parser, PortListener);
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
listener.Subscribe(ConvertToDictionary);
listener.Start();
return(true);
}
static void ProcessSyslogRealTime(
string logFileName,
string listenerAdapterName,
int listenerUdpPort,
string queryFile,
string consoleLogOption,
string outputFileName,
string blobConnectionString,
string blobContainerName,
KustoConnectionStringBuilder kscbAdmin,
KustoConnectionStringBuilder kscbIngest,
bool directIngest,
string tableName,
bool resetTable)
{
SyslogListener listener = null;
SyslogFileListener fileListener = null;
bool fileReadMode = false;
BlockingKustoUploader ku = null;
FileOutput fileOutput = null;
ConsoleOutput consoleOutput = null;
var parser = CreateSIEMfxSyslogParser();
var _converter = new SyslogEntryToRecordConverter();
// input
if (string.IsNullOrEmpty(logFileName))
{
// reading from local port
IPAddress localIp = null;
if (!string.IsNullOrEmpty(listenerAdapterName))
{
localIp = GetLocalIp(listenerAdapterName);
}
localIp ??= IPAddress.IPv6Any;
var endPoint = new IPEndPoint(localIp, listenerUdpPort);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
listener = new SyslogListener(parser, PortListener);
var filter = new SyslogFilter();
if (filter != null)
{
listener.Filter = filter.Allow;
}
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
listener.Subscribe(_converter);
listener.Start();
}
else
{
// reading from local file
var fileStream = new FileStream(logFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
fileListener = new SyslogFileListener(parser, fileStream);
fileReadMode = true;
var filter = new SyslogFilter();
if (filter != null)
{
fileListener.Filter = filter.Allow;
}
fileListener.Error += FileListener_Error;
fileListener.EntryReceived += FileListener_EntryReceived;
fileListener.Subscribe(_converter);
fileListener.Start();
}
Console.WriteLine();
Console.WriteLine("Listening to Syslog events. Press any key to terminate");
// output
if (kscbAdmin != null)
{
// output to kusto
ku = CreateUploader(UploadTimespan, blobConnectionString, blobContainerName, kscbAdmin, kscbIngest, directIngest, tableName, resetTable);
Task task = Task.Factory.StartNew(() =>
{
RunUploader(ku, _converter, queryFile);
});
}
else if (!string.IsNullOrEmpty(outputFileName))
{
// output to file
fileOutput = new FileOutput(outputFileName);
RunFileOutput(fileOutput, _converter, queryFile);
}
else
{
// output to console
bool tableFormat = consoleLogOption == "table" ? true : false;
consoleOutput = new ConsoleOutput(tableFormat);
RunConsoleOutput(consoleOutput, _converter, queryFile);
}
string readline = Console.ReadLine();
// clean up
if (!fileReadMode)
{
listener.Stop();
listener.Dispose();
listener = null;
}
else
{
fileListener.Stop();
fileListener.Dispose();
fileListener = null;
}
if (kscbAdmin != null)
{
ku.OnCompleted();
}
else if (!string.IsNullOrEmpty(outputFileName))
{
fileOutput.OnCompleted();
}
else
{
consoleOutput.OnCompleted();
}
}
