static void Main(string[] args)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
List <LogRecord> logRecords = logReader.GetUnreadRecords();
// If we have any records then output kvp format to the console
if (logRecords.Count > 0)
{
foreach (LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
static void Main(string[] args)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
List<LogRecord> logRecords = logReader.GetUnreadRecords();
// If we have any records then output kvp format to the console
if(logRecords.Count > 0)
{
foreach(LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
//Function to read logs and send them to filebeat
private void shipLogs()
{
//Set up the aaLogReader and get unread records
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
List <LogRecord> logRecords = logReader.GetUnreadRecords();
//If there were any records to ship:
if (logRecords.Count > 0)
{
//Log message
//localEventLog.WriteEntry("Shipping entries, count:" + logRecords.Count().ToString(), EventLogEntryType.Information);
//Use a Process to run the filebeat binary
using (Process fileBeatProcess = new Process())
{
//Set up the process
//TODO: make locations configurable
fileBeatProcess.StartInfo.FileName = this.binaryLocation;
fileBeatProcess.StartInfo.Arguments = "-c \"" + this.configLocation + "\" -e -once";
fileBeatProcess.StartInfo.UseShellExecute = false;
//Important: redirect standard input as this is how we will send the data to Filebeat
fileBeatProcess.StartInfo.RedirectStandardInput = true;
//Start the process
fileBeatProcess.Start();
//Get a StreamWriter object to send data to stdin of the process
StreamWriter fileBeatInputStream = fileBeatProcess.StandardInput;
//Ship each record
foreach (LogRecord record in logRecords)
{
//Ship to stdin as json, which can be decoded by filebeat
fileBeatInputStream.WriteLine(record.ToJSON());
}
//Close the stream; this should cause Filebeat to exit since we used -once
fileBeatInputStream.Close();
fileBeatProcess.WaitForExit();
//Log the end of the file
//localEventLog.WriteEntry("Finished shipping entries", EventLogEntryType.Information);
}
}
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await(Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List <LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
await eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}
public virtual void CollectMessage()
{
// Read SMC Message
aaLogReader.OptionsStruct testOptions = new aaLogReader.OptionsStruct();
testOptions.LogDirectory = "\\\\localhost\\C$\\ProgramData\\ArchestrA\\LogFiles";
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
DateTime dtStartDate = new System.DateTime(2019, 3, 20, DateTime.Now.Hour, DateTime.Now.Minute, DateTime.Now.Second);
DateTime dtEndDate = new System.DateTime(2019, 3, 22, DateTime.Now.Hour, DateTime.Now.Minute, DateTime.Now.Second);
Console.Write("Start time " + dtStartDate + "\n");
Console.Write("End time " + dtEndDate + "\n");
ulong MessageNumber_Start = 0;
ulong MessageNumber_End = 0;
try
{
List <LogRecord> records_start = logReader.GetRecordsByStartTimestampAndCount(dtStartDate, 1);
List <LogRecord> records_end = logReader.GetRecordsByStartTimestampAndCount(dtEndDate, 1);
if (records_start.Count == 0)
{
Console.Write("No log rows found.!" + "\n");
}
else
{
foreach (var LogRecord in records_start)
{
MessageNumber_Start = LogRecord.MessageNumber;
}
}
if (records_end.Count == 0)
{
LogRecord LogLastRecord = new LogRecord();
LogLastRecord = logReader.GetLastRecord();
MessageNumber_End = LogLastRecord.MessageNumber;
}
else
{
foreach (var LogRecord in records_end)
{
MessageNumber_End = LogRecord.MessageNumber;
}
}
int NumberMessageRecords;
if (Convert.ToInt32(MessageNumber_End - MessageNumber_Start) <= 10000000)
{
NumberMessageRecords = Convert.ToInt32(MessageNumber_End - MessageNumber_Start);
}
else
{
NumberMessageRecords = 10000000;
}
List <LogRecord> records = logReader.GetRecordsByStartTimestampAndCount(dtStartDate, NumberMessageRecords);
List <string> groupedMessages = new List <string>();
//setting all the records in a concurrent queue FIFO style
//if we want to expand the features, let's set the entire object.
foreach (var LogRecord in records)
{
SMC_Messages.Enqueue(LogRecord);
}
Console.WriteLine("Finished setting the logrecord in the queue.\n");
Console.WriteLine("The size of the concurrent queue is " + SMC_Messages.Count.ToString());
Console.WriteLine("END.\n");
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
public MainForm()
{
InitializeComponent();
logReader = new aaLogReader.aaLogReader();
addlog("Start");
}
static void Main(string[] args)
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
try
{
string answer;
long totalTime;
long totalRecords;
aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject<OptionsStruct>(System.IO.File.ReadAllText("options.json"));
testOptions.IgnoreCacheFileOnFirstRead = true;
#region Filter Section
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" });
#endregion
answer = "y";
totalTime = 0;
totalRecords = 0;
for (int x = 1; x <= 50; x++)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
List<LogRecord> records = new List<LogRecord>();
sw.Reset();
sw.Start();
records = logReader.GetUnreadRecords(100000);
////log.Info("Back");
////sw.Start();
////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back);
//sw.Start();
////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count);
////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10);
////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"));
////records.Add(record);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest);
////records.Add(record);
////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10));
////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10));
sw.Stop();
//log.InfoFormat("Found {0} messages", records.Count);
Console.WriteLine(records.Count + " records");
//log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds);
Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds");
//log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds);
Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds/1000) + " records/second");
totalRecords += records.Count;
totalTime += sw.ElapsedMilliseconds;
//writelogs(records);
//log.Info(JsonConvert.SerializeObject(records));
//sw.Stop();
//log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds);
//log.InfoFormat("Count - {0}", records.Count);
logReader.Dispose();
records = null;
sw = null;
}
Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime/1000) + " records/second");
Console.WriteLine("Complete");
Console.ReadLine();
//while (answer.ToLower() == "y")
//{
// Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)");
// answer = Console.ReadLine();
// if (answer.ToLower() == "y")
// {
// List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false);
// Console.WriteLine("Record count : " + records.Count.ToString());
// foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber))
// {
// //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// log.Info(writeMsg);
// //Console.WriteLine(writeMsg);
// }
// }
//}
}
catch (Exception ex)
{
log.Error(ex);
}
Console.Read();
return;
}
static void Main(string[] args)
{
// Setup logging
log4net.Config.BasicConfigurator.Configure();
try
{
string answer;
long totalTime;
long totalRecords;
aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json"));
testOptions.IgnoreCacheFileOnFirstRead = true;
#region Filter Section
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" });
//testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" });
#endregion
answer = "y";
totalTime = 0;
totalRecords = 0;
for (int x = 1; x <= 50; x++)
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
List <LogRecord> records = new List <LogRecord>();
sw.Reset();
sw.Start();
records = logReader.GetUnreadRecords(100000);
////log.Info("Back");
////sw.Start();
////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back);
//sw.Start();
////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count);
////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10);
////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"));
////records.Add(record);
////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest);
////records.Add(record);
////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10));
////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10));
sw.Stop();
//log.InfoFormat("Found {0} messages", records.Count);
Console.WriteLine(records.Count + " records");
//log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds);
Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds");
//log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds);
Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds / 1000) + " records/second");
totalRecords += records.Count;
totalTime += sw.ElapsedMilliseconds;
//writelogs(records);
//log.Info(JsonConvert.SerializeObject(records));
//sw.Stop();
//log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds);
//log.InfoFormat("Count - {0}", records.Count);
logReader.Dispose();
records = null;
sw = null;
}
Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime / 1000) + " records/second");
Console.WriteLine("Complete");
Console.ReadLine();
//while (answer.ToLower() == "y")
//{
// Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)");
// answer = Console.ReadLine();
// if (answer.ToLower() == "y")
// {
// List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false);
// Console.WriteLine("Record count : " + records.Count.ToString());
// foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber))
// {
// //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message);
// log.Info(writeMsg);
// //Console.WriteLine(writeMsg);
// }
// }
//}
}
catch (Exception ex)
{
log.Error(ex);
}
Console.Read();
return;
}
static void Main(string[] args)
{
try
{
ulong messageCount = 0;
aaLogReader.OptionsStruct options = new OptionsStruct();
string aaLGXDirectory = "";
log.Info("Starting Log Collection");
// Read the message count as the single argument to the exe
try
{
messageCount = System.Convert.ToUInt32(args[0]);
}
catch (Exception)
{
//Ignore the error and use default value of 1000
log.InfoFormat("No maximum message count specified. Include a single integer argument when calling the application to set the maximum message count retrieved");
messageCount = 1000;
}
// Final check to make sure we don't have a null messageCount
if (messageCount <= 0)
{
messageCount = 1000;
}
try
{
// Read the aaLGX directory command line switch
if (args.Length >= 2)
{
aaLGXDirectory = args[1] ?? "";
}
if (aaLGXDirectory != "")
{
log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory);
if (Directory.Exists(aaLGXDirectory))
{
string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx");
foreach (string fileName in filesList)
{
log.InfoFormat("Processing file {0}", fileName);
var aaLGXRecords = aaLogReader.aaLgxReader.ReadLogRecords(fileName);
log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName);
foreach (LogRecord record in aaLGXRecords)
{
Console.WriteLine(record.ToKVP());
}
log.InfoFormat("Deleting {0} after reading records.", fileName);
File.Delete(fileName);
}
}
else
{
log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory);
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
//Read the options from a local file
try
{
if (System.IO.File.Exists("options.json"))
{
log.DebugFormat("Reading Options from local options.json");
options = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json"));
}
}
catch
{
log.InfoFormat("No options.json file exists. Using default options");
options = new OptionsStruct();
}
log.InfoFormat("Reading records with maximum message count of {0}", messageCount);
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(options);
List <LogRecord> logRecords = logReader.GetUnreadRecords(maximumMessages: messageCount);
log.InfoFormat("{0} unread log records found", logRecords.Count);
// If we have any records then output kvp format to the console
if (logRecords.Count > 0)
{
foreach (LogRecord record in logRecords)
{
Console.WriteLine(record.ToKVP());
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await (Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}
private void dostuff()
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
//txtLog.Text = "";
try
{
++totalCount;
//aaLogReader.LogRecord lastRecordRead = new LogRecord();
Stopwatch sw = Stopwatch.StartNew();
//int RecordsToRead = 100000;
//addlog("Timer(ms) " + sw.ElapsedMilliseconds.ToString());
//addlog("Actual Records " + records.Count.ToString());
//addlog("Rate (records/s): " + ((int)(1000.0 * (float)records.Count / (float)sw.ElapsedMilliseconds)).ToString());
//addlog("");
//addlog(JsonConvert.SerializeObject(records, Formatting.Indented));
////lastRecordRead = logReader.ReadStatusCacheFile();
//addlog(JsonConvert.SerializeObject(lastRecordRead, Formatting.Indented));
//addlog("");
//addlog(JsonConvert.SerializeObject(logReader.ReadLogHeader(), Formatting.Indented));
////Debug.Print(JsonConvert.SerializeObject(record,Formatting.Indented));
List <LogRecord> records = logReader.GetUnreadRecords();
addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString());
if (ForwardLogsCheckBox.Checked)
{
// Open socket to send logs to remote host
TcpClient vSocket = new System.Net.Sockets.TcpClient(RemoteHost, RemotePort);
System.Net.Sockets.NetworkStream ServerStream = vSocket.GetStream();
System.IO.StreamWriter swriter = new StreamWriter(ServerStream);
foreach (LogRecord lr in records)
{
swriter.WriteLine(lr.ToKVP());
}
//swriter.Close();
//ServerStream.Close();
vSocket.Close();
}
++successCount;
failureLastTime = false;
}
catch (Exception ex)
{
++failureCount;
if (failureLastTime)
{
++failureConsecCount;
}
else
{
failureConsecCount = 1;
}
failureLastTime = true;
addlog("***********");
addlog(ex.ToString());
addlog("***********");
log.Error(ex);
}
finally
{
if (logReader != null)
{
logReader.CloseCurrentLogFile();
}
}
}
/// <summary>
/// Read data and transmit via HTTP to Splunk
/// </summary>
/// <param name="query"></param>
/// <param name="sqlConnectionObject"></param>
private static void ReadAndTransmitData(aaLogReader.aaLogReader aaLogReader)
{
string kvpValue = "";
List <LogRecord> logRecords = new List <LogRecord>();
List <LogRecord> lgxRecords = new List <LogRecord>();
try
{
//Get the last message number written
var lastMessageNumberWritten = LastMessageNumberWritten;
if (lastMessageNumberWritten == 0)
{
log.DebugFormat("Executing GetUnreadRecords for {0} records", runtimeOptions.MaxRecords);
// Get records starting with last log record and move backwards since we are working with no existing cache file
//AALogReader.Options.IgnoreCacheFileOnFirstRead = true;
//logRecords = AALogReader.GetRecordsByEndTimestampAndCount(System.DateTime.Now.AddMinutes(1), (int)runtimeOptions.MaxRecords);
logRecords = AALogReader.GetUnreadRecords((ulong)runtimeOptions.MaxRecords);
}
else
{
log.DebugFormat("Executing GetRecordsByStartMessageNumberAndCount for message number {0} with a maximum of {1} records", lastMessageNumberWritten + 1, runtimeOptions.MaxRecords);
// Get records based on last cached message number
logRecords = AALogReader.GetRecordsByStartMessageNumberAndCount(lastMessageNumberWritten + 1, (int)runtimeOptions.MaxRecords);
}
log.InfoFormat("{0} records retrieved", logRecords.Count);
if (logRecords.Count > 0)
{
//Build the additional KVP values to Append
var additionalKVPValues = new StringBuilder();
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost);
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceData", RuntimeOptions.SplunkSourceData);
//Get the KVP string for the records
kvpValue = logRecords.ToKVP(additionalKVPValues.ToString());
//Transmit the records
var result = SplunkHTTPClient.TransmitValues(kvpValue);
log.DebugFormat("Transmit Values Result - {0}", result);
//If successful then write the last sequence value to disk
if (result.StatusCode == HttpStatusCode.OK)
{
log.DebugFormat("Writing Cache File");
// Write the last sequence value to the cache value named for the SQLSequence Field. Order the result set by the sequence field then select the first record
WriteCacheFile(logRecords, CacheFilename, RuntimeOptions);
if (ReadTimer.Interval != RuntimeOptions.ReadInterval)
{
//Reset timer interval
ClearTimerBackoff(ReadTimer, RuntimeOptions);
}
}
else
{
// Implement a timer backoff so we don't flood the endpoint
IncrementTimerBackoff(ReadTimer, RuntimeOptions);
log.WarnFormat("HTTP Transmission not OK - {0}", result);
}
}
var aaLGXDirectory = runtimeOptions.AALGXDirectory ?? "";
// Parse AALGX Files
if (aaLGXDirectory != "")
{
log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory);
if (Directory.Exists(aaLGXDirectory))
{
var successFolder = aaLGXDirectory + "\\success";
if (!Directory.Exists(successFolder))
{
Directory.CreateDirectory(successFolder);
}
var errorFolder = aaLGXDirectory + "\\error";
if (!Directory.Exists(errorFolder))
{
Directory.CreateDirectory(errorFolder);
}
string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx");
foreach (string fileName in filesList)
{
log.InfoFormat("Processing file {0}", fileName);
var aaLGXRecords = aaLgxReader.ReadLogRecords(fileName);
log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName);
if (aaLGXRecords.Count() > 0)
{
//Build the additional KVP values to Append
var additionalKVPValues = new StringBuilder();
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost);
additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceData", RuntimeOptions.SplunkSourceData);
//Get the KVP string for the records
kvpValue = aaLGXRecords.ToKVP(additionalKVPValues.ToString());
//Transmit the records
var result = SplunkHTTPClient.TransmitValues(kvpValue);
log.DebugFormat("Transmit Values Result - {0}", result);
//If successful then write the last sequence value to disk
if (result.StatusCode == HttpStatusCode.OK)
{
log.InfoFormat("Successfully transmitted {0} records from file {1}", aaLGXRecords.Count, fileName);
log.InfoFormat("Moving {0} to {1}", fileName, successFolder);
var destinationFilename = Path.Combine(successFolder, Path.GetFileName(fileName));
if (File.Exists(destinationFilename))
{
log.WarnFormat("Deleting file {0} in preparation for move from {1}", destinationFilename, fileName);
try
{
File.Delete(destinationFilename);
}
catch (Exception ex)
{
log.Error(ex);
}
}
try
{
File.Move(fileName, destinationFilename);
}
catch
{
log.WarnFormat("Error moving {0} to {1}", fileName, destinationFilename);
}
if (ReadTimer.Interval != RuntimeOptions.ReadInterval)
{
//Reset timer interval
ClearTimerBackoff(ReadTimer, RuntimeOptions);
}
}
else
{
// Implement a timer backoff so we don't flood the endpoint
IncrementTimerBackoff(ReadTimer, RuntimeOptions);
log.WarnFormat("HTTP Transmission not OK - {0}", result);
log.InfoFormat("Moving {0} to {1}", fileName, errorFolder);
var destinationFilename = Path.Combine(errorFolder, Path.GetFileName(fileName));
if (File.Exists(destinationFilename))
{
log.WarnFormat("Deleting file {0} in preparation for move from {1}", destinationFilename, fileName);
try
{
File.Delete(destinationFilename);
}
catch (Exception ex)
{
log.Error(ex);
}
}
try
{
File.Move(fileName, destinationFilename);
}
catch
{
log.WarnFormat("Error moving {0} to {1}", fileName, destinationFilename);
}
}
}
}
}
else
{
log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory);
}
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
public MainForm()
{
InitializeComponent();
logReader = new aaLogReader.aaLogReader();
addlog("Start");
}
private void dostuff()
{
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader();
//txtLog.Text = "";
try
{
++totalCount;
//aaLogReader.LogRecord lastRecordRead = new LogRecord();
Stopwatch sw = Stopwatch.StartNew();
//int RecordsToRead = 100000;
//addlog("Timer(ms) " + sw.ElapsedMilliseconds.ToString());
//addlog("Actual Records " + records.Count.ToString());
//addlog("Rate (records/s): " + ((int)(1000.0 * (float)records.Count / (float)sw.ElapsedMilliseconds)).ToString());
//addlog("");
//addlog(JsonConvert.SerializeObject(records, Formatting.Indented));
////lastRecordRead = logReader.ReadStatusCacheFile();
//addlog(JsonConvert.SerializeObject(lastRecordRead, Formatting.Indented));
//addlog("");
//addlog(JsonConvert.SerializeObject(logReader.ReadLogHeader(), Formatting.Indented));
////Debug.Print(JsonConvert.SerializeObject(record,Formatting.Indented));
List<LogRecord> records = logReader.GetUnreadRecords();
addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString());
if (ForwardLogsCheckBox.Checked)
{
// Open socket to send logs to remote host
TcpClient vSocket = new System.Net.Sockets.TcpClient(RemoteHost, RemotePort);
System.Net.Sockets.NetworkStream ServerStream = vSocket.GetStream();
System.IO.StreamWriter swriter = new StreamWriter(ServerStream);
foreach (LogRecord lr in records)
{
swriter.WriteLine(lr.ToKVP());
}
//swriter.Close();
//ServerStream.Close();
vSocket.Close();
}
++successCount;
failureLastTime = false;
}
catch (Exception ex)
{
++failureCount;
if (failureLastTime) { ++failureConsecCount; }
else { failureConsecCount = 1; }
failureLastTime = true;
addlog("***********");
addlog(ex.ToString());
addlog("***********");
log.Error(ex);
}
finally
{
if (logReader != null)
logReader.CloseCurrentLogFile();
}
}
