static void Main(string[] args)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
List<LogRecord> logRecords = logReader.GetUnreadRecords();
// If we have any records then output kvp format to the console
if(logRecords.Count > 0)
{
foreach(LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
static void Main(string[] args)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
List <LogRecord> logRecords = logReader.GetUnreadRecords();
// If we have any records then output kvp format to the console
if (logRecords.Count > 0)
{
foreach (LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
private void Run()
{
List <LogRecord> records = logReader.GetUnreadRecords();
addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString());
using (System.IO.StreamWriter file = new System.IO.StreamWriter(txtFileName.Text, true))
{
foreach (LogRecord lr in records)
{
file.WriteLine(lr.ToTSV());
}
}
}
//Function to read logs and send them to filebeat
private void shipLogs()
{
//Set up the aaLogReader and get unread records
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
List <LogRecord> logRecords = logReader.GetUnreadRecords();
//If there were any records to ship:
if (logRecords.Count > 0)
{
//Log message
//localEventLog.WriteEntry("Shipping entries, count:" + logRecords.Count().ToString(), EventLogEntryType.Information);
//Use a Process to run the filebeat binary
using (Process fileBeatProcess = new Process())
{
//Set up the process
//TODO: make locations configurable
fileBeatProcess.StartInfo.FileName = this.binaryLocation;
fileBeatProcess.StartInfo.Arguments = "-c \"" + this.configLocation + "\" -e -once";
fileBeatProcess.StartInfo.UseShellExecute = false;
//Important: redirect standard input as this is how we will send the data to Filebeat
fileBeatProcess.StartInfo.RedirectStandardInput = true;
//Start the process
fileBeatProcess.Start();
//Get a StreamWriter object to send data to stdin of the process
StreamWriter fileBeatInputStream = fileBeatProcess.StandardInput;
//Ship each record
foreach (LogRecord record in logRecords)
{
//Ship to stdin as json, which can be decoded by filebeat
fileBeatInputStream.WriteLine(record.ToJSON());
}
//Close the stream; this should cause Filebeat to exit since we used -once
fileBeatInputStream.Close();
fileBeatProcess.WaitForExit();
//Log the end of the file
//localEventLog.WriteEntry("Finished shipping entries", EventLogEntryType.Information);
}
}
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await(Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List <LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
await eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}
static void Main(string[] args)
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
try
{
string answer;
long totalTime;
long totalRecords;
aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject<OptionsStruct>(System.IO.File.ReadAllText("options.json"));
testOptions.IgnoreCacheFileOnFirstRead = true;
#region Filter Section
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" });
#endregion
answer = "y";
totalTime = 0;
totalRecords = 0;
for (int x = 1; x <= 50; x++)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
List<LogRecord> records = new List<LogRecord>();
sw.Reset();
sw.Start();
records = logReader.GetUnreadRecords(100000);
////log.Info("Back");
////sw.Start();
////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back);
//sw.Start();
////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count);
////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10);
////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"));
////records.Add(record);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest);
////records.Add(record);
////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10));
////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10));
sw.Stop();
//log.InfoFormat("Found {0} messages", records.Count);
Console.WriteLine(records.Count + " records");
//log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds);
Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds");
//log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds);
Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds/1000) + " records/second");
totalRecords += records.Count;
totalTime += sw.ElapsedMilliseconds;
//writelogs(records);
//log.Info(JsonConvert.SerializeObject(records));
//sw.Stop();
//log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds);
//log.InfoFormat("Count - {0}", records.Count);
logReader.Dispose();
records = null;
sw = null;
}
Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime/1000) + " records/second");
Console.WriteLine("Complete");
Console.ReadLine();
//while (answer.ToLower() == "y")
//{
// Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)");
// answer = Console.ReadLine();
// if (answer.ToLower() == "y")
// {
// List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false);
// Console.WriteLine("Record count : " + records.Count.ToString());
// foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber))
// {
// //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// log.Info(writeMsg);
// //Console.WriteLine(writeMsg);
// }
// }
//}
}
catch (Exception ex)
{
log.Error(ex);
}
Console.Read();
return;
}
static void Main(string[] args)
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
try
{
string answer;
long totalTime;
long totalRecords;
aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json"));
testOptions.IgnoreCacheFileOnFirstRead = true;
#region Filter Section
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" });
#endregion
answer = "y";
totalTime = 0;
totalRecords = 0;
for (int x = 1; x <= 50; x++)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
List <LogRecord> records = new List <LogRecord>();
sw.Reset();
sw.Start();
records = logReader.GetUnreadRecords(100000);
////log.Info("Back");
////sw.Start();
////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back);
//sw.Start();
////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count);
////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10);
////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"));
////records.Add(record);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest);
////records.Add(record);
////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10));
////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10));
sw.Stop();
//log.InfoFormat("Found {0} messages", records.Count);
Console.WriteLine(records.Count + " records");
//log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds);
Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds");
//log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds);
Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds / 1000) + " records/second");
totalRecords += records.Count;
totalTime += sw.ElapsedMilliseconds;
//writelogs(records);
//log.Info(JsonConvert.SerializeObject(records));
//sw.Stop();
//log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds);
//log.InfoFormat("Count - {0}", records.Count);
logReader.Dispose();
records = null;
sw = null;
}
Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime / 1000) + " records/second");
Console.WriteLine("Complete");
Console.ReadLine();
//while (answer.ToLower() == "y")
//{
// Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)");
// answer = Console.ReadLine();
// if (answer.ToLower() == "y")
// {
// List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false);
// Console.WriteLine("Record count : " + records.Count.ToString());
// foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber))
// {
// //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// log.Info(writeMsg);
// //Console.WriteLine(writeMsg);
// }
// }
//}
}
catch (Exception ex)
{
log.Error(ex);
}
Console.Read();
return;
}
static void Main(string[] args)
{
try
{
ulong messageCount = 0;
aaLogReader.OptionsStruct options = new OptionsStruct();
string aaLGXDirectory = "";
log.Info("Starting Log Collection");
// Read the message count as the single argument to the exe
try
{
messageCount = System.Convert.ToUInt32(args[0]);
}
catch (Exception)
{
//Ignore the error and use default value of 1000
log.InfoFormat("No maximum message count specified. Include a single integer argument when calling the application to set the maximum message count retrieved");
messageCount = 1000;
}
// Final check to make sure we don't have a null messageCount
if (messageCount <= 0)
{
messageCount = 1000;
}
try
{
// Read the aaLGX directory command line switch
if (args.Length >= 2)
{
aaLGXDirectory = args[1] ?? "";
}
if (aaLGXDirectory != "")
{
log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory);
if (Directory.Exists(aaLGXDirectory))
{
string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx");
foreach (string fileName in filesList)
{
log.InfoFormat("Processing file {0}", fileName);
var aaLGXRecords = aaLogReader.aaLgxReader.ReadLogRecords(fileName);
log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName);
foreach (LogRecord record in aaLGXRecords)
{
Console.WriteLine(record.ToKVP());
}
log.InfoFormat("Deleting {0} after reading records.", fileName);
File.Delete(fileName);
}
}
else
{
log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory);
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
//Read the options from a local file
try
{
if (System.IO.File.Exists("options.json"))
{
log.DebugFormat("Reading Options from local options.json");
options = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json"));
}
}
catch
{
log.InfoFormat("No options.json file exists. Using default options");
options = new OptionsStruct();
}
log.InfoFormat("Reading records with maximum message count of {0}", messageCount);
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(options);
List <LogRecord> logRecords = logReader.GetUnreadRecords(maximumMessages: messageCount);
log.InfoFormat("{0} unread log records found", logRecords.Count);
// If we have any records then output kvp format to the console
if (logRecords.Count > 0)
{
foreach (LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await (Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}
private void dostuff()
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
//txtLog.Text = "";
try
{
++totalCount;
//aaLogReader.LogRecord lastRecordRead = new LogRecord();
Stopwatch sw = Stopwatch.StartNew();
//int RecordsToRead = 100000;
//addlog("Timer(ms) " + sw.ElapsedMilliseconds.ToString());
//addlog("Actual Records " + records.Count.ToString());
//addlog("Rate (records/s): " + ((int)(1000.0 * (float)records.Count / (float)sw.ElapsedMilliseconds)).ToString());
//addlog("");
//addlog(JsonConvert.SerializeObject(records, Formatting.Indented));
////lastRecordRead = logReader.ReadStatusCacheFile();
//addlog(JsonConvert.SerializeObject(lastRecordRead, Formatting.Indented));
//addlog("");
//addlog(JsonConvert.SerializeObject(logReader.ReadLogHeader(), Formatting.Indented));
////Debug.Print(JsonConvert.SerializeObject(record,Formatting.Indented));
List <LogRecord> records = logReader.GetUnreadRecords();
addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString());
if (ForwardLogsCheckBox.Checked)
{
// Open socket to send logs to remote host
TcpClient vSocket = new System.Net.Sockets.TcpClient(RemoteHost, RemotePort);
System.Net.Sockets.NetworkStream ServerStream = vSocket.GetStream();
System.IO.StreamWriter swriter = new StreamWriter(ServerStream);
foreach (LogRecord lr in records)
{
swriter.WriteLine(lr.ToKVP());
}
//swriter.Close();
//ServerStream.Close();
vSocket.Close();
}
++successCount;
failureLastTime = false;
}
catch (Exception ex)
{
++failureCount;
if (failureLastTime)
{
++failureConsecCount;
}
else
{
failureConsecCount = 1;
}
failureLastTime = true;
addlog("***********");
addlog(ex.ToString());
addlog("***********");
log.Error(ex);
}
finally
{
if (logReader != null)
{
logReader.CloseCurrentLogFile();
}
}
}
private void dostuff()
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
//txtLog.Text = "";
try
{
++totalCount;
//aaLogReader.LogRecord lastRecordRead = new LogRecord();
Stopwatch sw = Stopwatch.StartNew();
//int RecordsToRead = 100000;
//addlog("Timer(ms) " + sw.ElapsedMilliseconds.ToString());
//addlog("Actual Records " + records.Count.ToString());
//addlog("Rate (records/s): " + ((int)(1000.0 * (float)records.Count / (float)sw.ElapsedMilliseconds)).ToString());
//addlog("");
//addlog(JsonConvert.SerializeObject(records, Formatting.Indented));
////lastRecordRead = logReader.ReadStatusCacheFile();
//addlog(JsonConvert.SerializeObject(lastRecordRead, Formatting.Indented));
//addlog("");
//addlog(JsonConvert.SerializeObject(logReader.ReadLogHeader(), Formatting.Indented));
////Debug.Print(JsonConvert.SerializeObject(record,Formatting.Indented));
List<LogRecord> records = logReader.GetUnreadRecords();
addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString());
if (ForwardLogsCheckBox.Checked)
{
// Open socket to send logs to remote host
TcpClient vSocket = new System.Net.Sockets.TcpClient(RemoteHost, RemotePort);
System.Net.Sockets.NetworkStream ServerStream = vSocket.GetStream();
System.IO.StreamWriter swriter = new StreamWriter(ServerStream);
foreach (LogRecord lr in records)
{
swriter.WriteLine(lr.ToKVP());
}
//swriter.Close();
//ServerStream.Close();
vSocket.Close();
}
++successCount;
failureLastTime = false;
}
catch (Exception ex)
{
++failureCount;
if (failureLastTime) { ++failureConsecCount; }
else { failureConsecCount = 1; }
failureLastTime = true;
addlog("***********");
addlog(ex.ToString());
addlog("***********");
log.Error(ex);
}
finally
{
if (logReader != null)
logReader.CloseCurrentLogFile();
}
}
