/// <summary>
/// Constructor using specificed options
/// </summary>
///<param name="InitializationOptions">InitializationOptions passed as an OptionsStruct object </param>
public aaLogReader(OptionsStruct InitializationOptions)
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
log.Debug("Create aaLogReader");
log.Debug("Options - " + JsonConvert.SerializeObject(InitializationOptions));
this.Options = InitializationOptions;
this.Initialize();
}
/// <summary>
/// Default constructor using default options
/// </summary>
public aaLogReader()
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
log.Debug("Create aaLogReader");
// Initialize with default options
Options = new OptionsStruct();
this.Initialize();
}
public virtual void CollectMessage()
{
// Read SMC Message
aaLogReader.OptionsStruct testOptions = new aaLogReader.OptionsStruct();
testOptions.LogDirectory = "\\\\localhost\\C$\\ProgramData\\ArchestrA\\LogFiles";
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
DateTime dtStartDate = new System.DateTime(2019, 3, 20, DateTime.Now.Hour, DateTime.Now.Minute, DateTime.Now.Second);
DateTime dtEndDate = new System.DateTime(2019, 3, 22, DateTime.Now.Hour, DateTime.Now.Minute, DateTime.Now.Second);
Console.Write("Start time " + dtStartDate + "\n");
Console.Write("End time " + dtEndDate + "\n");
ulong MessageNumber_Start = 0;
ulong MessageNumber_End = 0;
try
{
List <LogRecord> records_start = logReader.GetRecordsByStartTimestampAndCount(dtStartDate, 1);
List <LogRecord> records_end = logReader.GetRecordsByStartTimestampAndCount(dtEndDate, 1);
if (records_start.Count == 0)
{
Console.Write("No log rows found.!" + "\n");
}
else
{
foreach (var LogRecord in records_start)
{
MessageNumber_Start = LogRecord.MessageNumber;
}
}
if (records_end.Count == 0)
{
LogRecord LogLastRecord = new LogRecord();
LogLastRecord = logReader.GetLastRecord();
MessageNumber_End = LogLastRecord.MessageNumber;
}
else
{
foreach (var LogRecord in records_end)
{
MessageNumber_End = LogRecord.MessageNumber;
}
}
int NumberMessageRecords;
if (Convert.ToInt32(MessageNumber_End - MessageNumber_Start) <= 10000000)
{
NumberMessageRecords = Convert.ToInt32(MessageNumber_End - MessageNumber_Start);
}
else
{
NumberMessageRecords = 10000000;
}
List <LogRecord> records = logReader.GetRecordsByStartTimestampAndCount(dtStartDate, NumberMessageRecords);
List <string> groupedMessages = new List <string>();
//setting all the records in a concurrent queue FIFO style
//if we want to expand the features, let's set the entire object.
foreach (var LogRecord in records)
{
SMC_Messages.Enqueue(LogRecord);
}
Console.WriteLine("Finished setting the logrecord in the queue.\n");
Console.WriteLine("The size of the concurrent queue is " + SMC_Messages.Count.ToString());
Console.WriteLine("END.\n");
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
static void Main(string[] args)
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
try
{
string answer;
long totalTime;
long totalRecords;
aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json"));
testOptions.IgnoreCacheFileOnFirstRead = true;
#region Filter Section
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" });
#endregion
answer = "y";
totalTime = 0;
totalRecords = 0;
for (int x = 1; x <= 50; x++)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
List <LogRecord> records = new List <LogRecord>();
sw.Reset();
sw.Start();
records = logReader.GetUnreadRecords(100000);
////log.Info("Back");
////sw.Start();
////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back);
//sw.Start();
////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count);
////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10);
////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"));
////records.Add(record);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest);
////records.Add(record);
////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10));
////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10));
sw.Stop();
//log.InfoFormat("Found {0} messages", records.Count);
Console.WriteLine(records.Count + " records");
//log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds);
Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds");
//log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds);
Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds / 1000) + " records/second");
totalRecords += records.Count;
totalTime += sw.ElapsedMilliseconds;
//writelogs(records);
//log.Info(JsonConvert.SerializeObject(records));
//sw.Stop();
//log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds);
//log.InfoFormat("Count - {0}", records.Count);
logReader.Dispose();
records = null;
sw = null;
}
Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime / 1000) + " records/second");
Console.WriteLine("Complete");
Console.ReadLine();
//while (answer.ToLower() == "y")
//{
// Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)");
// answer = Console.ReadLine();
// if (answer.ToLower() == "y")
// {
// List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false);
// Console.WriteLine("Record count : " + records.Count.ToString());
// foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber))
// {
// //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// log.Info(writeMsg);
// //Console.WriteLine(writeMsg);
// }
// }
//}
}
catch (Exception ex)
{
log.Error(ex);
}
Console.Read();
return;
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await (Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}