static void Main(string[] args) { aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(); List <LogRecord> logRecords = logReader.GetUnreadRecords(); // If we have any records then output kvp format to the console if (logRecords.Count > 0) { foreach (LogRecord record in logRecords) { Console.WriteLine(record.ToKVP()); } } }
static void Main(string[] args) { aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(); List<LogRecord> logRecords = logReader.GetUnreadRecords(); // If we have any records then output kvp format to the console if(logRecords.Count > 0) { foreach(LogRecord record in logRecords) { Console.WriteLine(record.ToKVP()); } } }
//Function to read logs and send them to filebeat private void shipLogs() { //Set up the aaLogReader and get unread records aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(); List <LogRecord> logRecords = logReader.GetUnreadRecords(); //If there were any records to ship: if (logRecords.Count > 0) { //Log message //localEventLog.WriteEntry("Shipping entries, count:" + logRecords.Count().ToString(), EventLogEntryType.Information); //Use a Process to run the filebeat binary using (Process fileBeatProcess = new Process()) { //Set up the process //TODO: make locations configurable fileBeatProcess.StartInfo.FileName = this.binaryLocation; fileBeatProcess.StartInfo.Arguments = "-c \"" + this.configLocation + "\" -e -once"; fileBeatProcess.StartInfo.UseShellExecute = false; //Important: redirect standard input as this is how we will send the data to Filebeat fileBeatProcess.StartInfo.RedirectStandardInput = true; //Start the process fileBeatProcess.Start(); //Get a StreamWriter object to send data to stdin of the process StreamWriter fileBeatInputStream = fileBeatProcess.StandardInput; //Ship each record foreach (LogRecord record in logRecords) { //Ship to stdin as json, which can be decoded by filebeat fileBeatInputStream.WriteLine(record.ToJSON()); } //Close the stream; this should cause Filebeat to exit since we used -once fileBeatInputStream.Close(); fileBeatProcess.WaitForExit(); //Log the end of the file //localEventLog.WriteEntry("Finished shipping entries", EventLogEntryType.Information); } } }
/// <summary> /// Write events to Splunk from this modular input. /// </summary> /// <remarks> /// This function will be invoked once for each instance of the modular input, though that invocation /// may or may not be in separate processes, depending on how the modular input is configured. It should /// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt> /// (which is thread safe). /// </remarks> /// <param name="inputDefinition">a specification of this instance of the modular input.</param> /// <param name="eventWriter">an object that handles writing events to Splunk.</param> public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { try { string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString(); Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32(); Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32(); //Setup the options input OptionsStruct localOptionsStruct = new OptionsStruct(); localOptionsStruct.LogDirectory = logfilepath; // Initialize the log reader aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct); // Write an entry to the Splunk system log indicating we have initialized await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString()); while (true) { await(Task.Delay(cycletime)); //Simple call to get all unread records, limiting the return count to max message count List <LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount); // Loop through each lastRecordRead and send to Splunk foreach (LogRecord record in logRecords) { await eventWriter.QueueEventForWriting(new Event { Stanza = inputDefinition.Name, Data = record.ToKVP() }); } } } catch (Exception ex) { // Eat error message await eventWriter.LogAsync(Severity.Error, ex.ToString()); } }
public virtual void CollectMessage() { // Read SMC Message aaLogReader.OptionsStruct testOptions = new aaLogReader.OptionsStruct(); testOptions.LogDirectory = "\\\\localhost\\C$\\ProgramData\\ArchestrA\\LogFiles"; aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions); DateTime dtStartDate = new System.DateTime(2019, 3, 20, DateTime.Now.Hour, DateTime.Now.Minute, DateTime.Now.Second); DateTime dtEndDate = new System.DateTime(2019, 3, 22, DateTime.Now.Hour, DateTime.Now.Minute, DateTime.Now.Second); Console.Write("Start time " + dtStartDate + "\n"); Console.Write("End time " + dtEndDate + "\n"); ulong MessageNumber_Start = 0; ulong MessageNumber_End = 0; try { List <LogRecord> records_start = logReader.GetRecordsByStartTimestampAndCount(dtStartDate, 1); List <LogRecord> records_end = logReader.GetRecordsByStartTimestampAndCount(dtEndDate, 1); if (records_start.Count == 0) { Console.Write("No log rows found.!" + "\n"); } else { foreach (var LogRecord in records_start) { MessageNumber_Start = LogRecord.MessageNumber; } } if (records_end.Count == 0) { LogRecord LogLastRecord = new LogRecord(); LogLastRecord = logReader.GetLastRecord(); MessageNumber_End = LogLastRecord.MessageNumber; } else { foreach (var LogRecord in records_end) { MessageNumber_End = LogRecord.MessageNumber; } } int NumberMessageRecords; if (Convert.ToInt32(MessageNumber_End - MessageNumber_Start) <= 10000000) { NumberMessageRecords = Convert.ToInt32(MessageNumber_End - MessageNumber_Start); } else { NumberMessageRecords = 10000000; } List <LogRecord> records = logReader.GetRecordsByStartTimestampAndCount(dtStartDate, NumberMessageRecords); List <string> groupedMessages = new List <string>(); //setting all the records in a concurrent queue FIFO style //if we want to expand the features, let's set the entire object. foreach (var LogRecord in records) { SMC_Messages.Enqueue(LogRecord); } Console.WriteLine("Finished setting the logrecord in the queue.\n"); Console.WriteLine("The size of the concurrent queue is " + SMC_Messages.Count.ToString()); Console.WriteLine("END.\n"); Console.ReadKey(); } catch (Exception ex) { Console.WriteLine(ex.Message); } }
public MainForm() { InitializeComponent(); logReader = new aaLogReader.aaLogReader(); addlog("Start"); }
static void Main(string[] args) { // Setup logging log4net.Config.BasicConfigurator.Configure(); try { string answer; long totalTime; long totalRecords; aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject<OptionsStruct>(System.IO.File.ReadAllText("options.json")); testOptions.IgnoreCacheFileOnFirstRead = true; #region Filter Section //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" }); #endregion answer = "y"; totalTime = 0; totalRecords = 0; for (int x = 1; x <= 50; x++) { aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions); System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); List<LogRecord> records = new List<LogRecord>(); sw.Reset(); sw.Start(); records = logReader.GetUnreadRecords(100000); ////log.Info("Back"); ////sw.Start(); ////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back); //sw.Start(); ////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count); ////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10); ////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30); ////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33")); ////records.Add(record); ////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest); ////records.Add(record); ////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10)); ////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10)); sw.Stop(); //log.InfoFormat("Found {0} messages", records.Count); Console.WriteLine(records.Count + " records"); //log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds); Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds"); //log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds); Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds/1000) + " records/second"); totalRecords += records.Count; totalTime += sw.ElapsedMilliseconds; //writelogs(records); //log.Info(JsonConvert.SerializeObject(records)); //sw.Stop(); //log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds); //log.InfoFormat("Count - {0}", records.Count); logReader.Dispose(); records = null; sw = null; } Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime/1000) + " records/second"); Console.WriteLine("Complete"); Console.ReadLine(); //while (answer.ToLower() == "y") //{ // Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)"); // answer = Console.ReadLine(); // if (answer.ToLower() == "y") // { // List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false); // Console.WriteLine("Record count : " + records.Count.ToString()); // foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber)) // { // //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message); // string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message); // log.Info(writeMsg); // //Console.WriteLine(writeMsg); // } // } //} } catch (Exception ex) { log.Error(ex); } Console.Read(); return; }
static void Main(string[] args) { // Setup logging log4net.Config.BasicConfigurator.Configure(); try { string answer; long totalTime; long totalRecords; aaLogReader.OptionsStruct testOptions = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json")); testOptions.IgnoreCacheFileOnFirstRead = true; #region Filter Section //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Warning 40|Message 41" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMin", Filter = "6826080" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "MessageNumberMax", Filter = "6826085" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMin", Filter = "2015-06-19 01:45:00" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "DateTimeMax", Filter = "2015-06-19 01:45:05" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ProcessID", Filter = "7260" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "ThreadID", Filter = "7264" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "Message", Filter = "Started" }); //testOptions.LogRecordPostFilters.Add(new LogRecordFilterStruct() { Field = "HostFQDN", Filter = "865" }); #endregion answer = "y"; totalTime = 0; totalRecords = 0; for (int x = 1; x <= 50; x++) { aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(testOptions); System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); List <LogRecord> records = new List <LogRecord>(); sw.Reset(); sw.Start(); records = logReader.GetUnreadRecords(100000); ////log.Info("Back"); ////sw.Start(); ////records = logReader.GetRecordsByStartMessageNumberAndCount(startmsg, count, SearchDirection.Back); //sw.Start(); ////records = logReader.GetRecordsByStartandEndMessageNumber(startmsg, endmsg, count); ////records = logReader.GetRecordsByEndMessageNumberAndCount(18064517, 10); ////records = logReader.GetRecordsByStartMessageNumberAndCount(8064512, 30); ////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33")); ////records.Add(record); ////record = logReader.GetRecordByTimestamp(DateTime.Parse("2015-06-27 13:42:33"),EarliestOrLatest.Latest); ////records.Add(record); ////writelogs(logReader.GetRecordsByEndTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"),10)); ////writelogs(logReader.GetRecordsByStartTimestampAndCount(DateTime.Parse("2015-06-27 13:42:33"), 10)); sw.Stop(); //log.InfoFormat("Found {0} messages", records.Count); Console.WriteLine(records.Count + " records"); //log.InfoFormat("Time - {0} millseconds", sw.ElapsedMilliseconds); Console.WriteLine(sw.ElapsedMilliseconds + " milliseconds"); //log.InfoFormat("Rate - {0} records/second", records.Count / sw.Elapsed.TotalSeconds); Console.WriteLine("Rate - " + records.Count / ((float)sw.ElapsedMilliseconds / 1000) + " records/second"); totalRecords += records.Count; totalTime += sw.ElapsedMilliseconds; //writelogs(records); //log.Info(JsonConvert.SerializeObject(records)); //sw.Stop(); //log.InfoFormat("Time - {0}", sw.ElapsedMilliseconds); //log.InfoFormat("Count - {0}", records.Count); logReader.Dispose(); records = null; sw = null; } Console.WriteLine("Average Rate - " + totalRecords / ((float)totalTime / 1000) + " records/second"); Console.WriteLine("Complete"); Console.ReadLine(); //while (answer.ToLower() == "y") //{ // Console.WriteLine("Read Unread Records (Y=Yes, Any Other Key=Exit)"); // answer = Console.ReadLine(); // if (answer.ToLower() == "y") // { // List<LogRecord> recordslocal = logReader.GetUnreadRecords(1000,"",false); // Console.WriteLine("Record count : " + records.Count.ToString()); // foreach (LogRecord lr in recordslocal.OrderBy(x => x.MessageNumber)) // { // //string writeMsg = (lr.MessageNumber.ToString() + '\t' + lr.EventFileTime.ToString() + '\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message); // string writeMsg = (lr.MessageNumber.ToString() +'\t' + lr.EventDateTime.ToString("yyyy-MM-dd hh:mm:ss.fff tt") + '\t' + lr.LogFlag + '\t' + lr.Message); // log.Info(writeMsg); // //Console.WriteLine(writeMsg); // } // } //} } catch (Exception ex) { log.Error(ex); } Console.Read(); return; }
static void Main(string[] args) { try { ulong messageCount = 0; aaLogReader.OptionsStruct options = new OptionsStruct(); string aaLGXDirectory = ""; log.Info("Starting Log Collection"); // Read the message count as the single argument to the exe try { messageCount = System.Convert.ToUInt32(args[0]); } catch (Exception) { //Ignore the error and use default value of 1000 log.InfoFormat("No maximum message count specified. Include a single integer argument when calling the application to set the maximum message count retrieved"); messageCount = 1000; } // Final check to make sure we don't have a null messageCount if (messageCount <= 0) { messageCount = 1000; } try { // Read the aaLGX directory command line switch if (args.Length >= 2) { aaLGXDirectory = args[1] ?? ""; } if (aaLGXDirectory != "") { log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory); if (Directory.Exists(aaLGXDirectory)) { string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx"); foreach (string fileName in filesList) { log.InfoFormat("Processing file {0}", fileName); var aaLGXRecords = aaLogReader.aaLgxReader.ReadLogRecords(fileName); log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName); foreach (LogRecord record in aaLGXRecords) { Console.WriteLine(record.ToKVP()); } log.InfoFormat("Deleting {0} after reading records.", fileName); File.Delete(fileName); } } else { log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory); } } } catch (Exception ex) { log.Error(ex); } //Read the options from a local file try { if (System.IO.File.Exists("options.json")) { log.DebugFormat("Reading Options from local options.json"); options = JsonConvert.DeserializeObject <OptionsStruct>(System.IO.File.ReadAllText("options.json")); } } catch { log.InfoFormat("No options.json file exists. Using default options"); options = new OptionsStruct(); } log.InfoFormat("Reading records with maximum message count of {0}", messageCount); aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(options); List <LogRecord> logRecords = logReader.GetUnreadRecords(maximumMessages: messageCount); log.InfoFormat("{0} unread log records found", logRecords.Count); // If we have any records then output kvp format to the console if (logRecords.Count > 0) { foreach (LogRecord record in logRecords) { Console.WriteLine(record.ToKVP()); } } } catch (Exception ex) { log.Error(ex); } }
/// <summary> /// Write events to Splunk from this modular input. /// </summary> /// <remarks> /// This function will be invoked once for each instance of the modular input, though that invocation /// may or may not be in separate processes, depending on how the modular input is configured. It should /// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt> /// (which is thread safe). /// </remarks> /// <param name="inputDefinition">a specification of this instance of the modular input.</param> /// <param name="eventWriter">an object that handles writing events to Splunk.</param> public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { try { string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString(); Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32(); Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32(); //Setup the options input OptionsStruct localOptionsStruct = new OptionsStruct(); localOptionsStruct.LogDirectory = logfilepath; // Initialize the log reader aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct); // Write an entry to the Splunk system log indicating we have initialized await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString()); while (true) { await (Task.Delay(cycletime)); //Simple call to get all unread records, limiting the return count to max message count List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount); // Loop through each lastRecordRead and send to Splunk foreach (LogRecord record in logRecords) { await eventWriter.QueueEventForWriting(new Event { Stanza = inputDefinition.Name, Data = record.ToKVP() }); } } } catch (Exception ex) { // Eat error message eventWriter.LogAsync(Severity.Error, ex.ToString()); } }
private void dostuff() { aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(); //txtLog.Text = ""; try { ++totalCount; //aaLogReader.LogRecord lastRecordRead = new LogRecord(); Stopwatch sw = Stopwatch.StartNew(); //int RecordsToRead = 100000; //addlog("Timer(ms) " + sw.ElapsedMilliseconds.ToString()); //addlog("Actual Records " + records.Count.ToString()); //addlog("Rate (records/s): " + ((int)(1000.0 * (float)records.Count / (float)sw.ElapsedMilliseconds)).ToString()); //addlog(""); //addlog(JsonConvert.SerializeObject(records, Formatting.Indented)); ////lastRecordRead = logReader.ReadStatusCacheFile(); //addlog(JsonConvert.SerializeObject(lastRecordRead, Formatting.Indented)); //addlog(""); //addlog(JsonConvert.SerializeObject(logReader.ReadLogHeader(), Formatting.Indented)); ////Debug.Print(JsonConvert.SerializeObject(record,Formatting.Indented)); List <LogRecord> records = logReader.GetUnreadRecords(); addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString()); if (ForwardLogsCheckBox.Checked) { // Open socket to send logs to remote host TcpClient vSocket = new System.Net.Sockets.TcpClient(RemoteHost, RemotePort); System.Net.Sockets.NetworkStream ServerStream = vSocket.GetStream(); System.IO.StreamWriter swriter = new StreamWriter(ServerStream); foreach (LogRecord lr in records) { swriter.WriteLine(lr.ToKVP()); } //swriter.Close(); //ServerStream.Close(); vSocket.Close(); } ++successCount; failureLastTime = false; } catch (Exception ex) { ++failureCount; if (failureLastTime) { ++failureConsecCount; } else { failureConsecCount = 1; } failureLastTime = true; addlog("***********"); addlog(ex.ToString()); addlog("***********"); log.Error(ex); } finally { if (logReader != null) { logReader.CloseCurrentLogFile(); } } }
/// <summary> /// Read data and transmit via HTTP to Splunk /// </summary> /// <param name="query"></param> /// <param name="sqlConnectionObject"></param> private static void ReadAndTransmitData(aaLogReader.aaLogReader aaLogReader) { string kvpValue = ""; List <LogRecord> logRecords = new List <LogRecord>(); List <LogRecord> lgxRecords = new List <LogRecord>(); try { //Get the last message number written var lastMessageNumberWritten = LastMessageNumberWritten; if (lastMessageNumberWritten == 0) { log.DebugFormat("Executing GetUnreadRecords for {0} records", runtimeOptions.MaxRecords); // Get records starting with last log record and move backwards since we are working with no existing cache file //AALogReader.Options.IgnoreCacheFileOnFirstRead = true; //logRecords = AALogReader.GetRecordsByEndTimestampAndCount(System.DateTime.Now.AddMinutes(1), (int)runtimeOptions.MaxRecords); logRecords = AALogReader.GetUnreadRecords((ulong)runtimeOptions.MaxRecords); } else { log.DebugFormat("Executing GetRecordsByStartMessageNumberAndCount for message number {0} with a maximum of {1} records", lastMessageNumberWritten + 1, runtimeOptions.MaxRecords); // Get records based on last cached message number logRecords = AALogReader.GetRecordsByStartMessageNumberAndCount(lastMessageNumberWritten + 1, (int)runtimeOptions.MaxRecords); } log.InfoFormat("{0} records retrieved", logRecords.Count); if (logRecords.Count > 0) { //Build the additional KVP values to Append var additionalKVPValues = new StringBuilder(); additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost); additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceData", RuntimeOptions.SplunkSourceData); //Get the KVP string for the records kvpValue = logRecords.ToKVP(additionalKVPValues.ToString()); //Transmit the records var result = SplunkHTTPClient.TransmitValues(kvpValue); log.DebugFormat("Transmit Values Result - {0}", result); //If successful then write the last sequence value to disk if (result.StatusCode == HttpStatusCode.OK) { log.DebugFormat("Writing Cache File"); // Write the last sequence value to the cache value named for the SQLSequence Field. Order the result set by the sequence field then select the first record WriteCacheFile(logRecords, CacheFilename, RuntimeOptions); if (ReadTimer.Interval != RuntimeOptions.ReadInterval) { //Reset timer interval ClearTimerBackoff(ReadTimer, RuntimeOptions); } } else { // Implement a timer backoff so we don't flood the endpoint IncrementTimerBackoff(ReadTimer, RuntimeOptions); log.WarnFormat("HTTP Transmission not OK - {0}", result); } } var aaLGXDirectory = runtimeOptions.AALGXDirectory ?? ""; // Parse AALGX Files if (aaLGXDirectory != "") { log.InfoFormat("Reading aaLGX files from {0}", aaLGXDirectory); if (Directory.Exists(aaLGXDirectory)) { var successFolder = aaLGXDirectory + "\\success"; if (!Directory.Exists(successFolder)) { Directory.CreateDirectory(successFolder); } var errorFolder = aaLGXDirectory + "\\error"; if (!Directory.Exists(errorFolder)) { Directory.CreateDirectory(errorFolder); } string[] filesList = Directory.GetFiles(aaLGXDirectory, "*.aalgx"); foreach (string fileName in filesList) { log.InfoFormat("Processing file {0}", fileName); var aaLGXRecords = aaLgxReader.ReadLogRecords(fileName); log.InfoFormat("Found {0} records in {1}", aaLGXRecords.Count(), fileName); if (aaLGXRecords.Count() > 0) { //Build the additional KVP values to Append var additionalKVPValues = new StringBuilder(); additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceHost", RuntimeOptions.SplunkSourceHost); additionalKVPValues.AppendFormat("{0}=\"{1}\", ", "SourceData", RuntimeOptions.SplunkSourceData); //Get the KVP string for the records kvpValue = aaLGXRecords.ToKVP(additionalKVPValues.ToString()); //Transmit the records var result = SplunkHTTPClient.TransmitValues(kvpValue); log.DebugFormat("Transmit Values Result - {0}", result); //If successful then write the last sequence value to disk if (result.StatusCode == HttpStatusCode.OK) { log.InfoFormat("Successfully transmitted {0} records from file {1}", aaLGXRecords.Count, fileName); log.InfoFormat("Moving {0} to {1}", fileName, successFolder); var destinationFilename = Path.Combine(successFolder, Path.GetFileName(fileName)); if (File.Exists(destinationFilename)) { log.WarnFormat("Deleting file {0} in preparation for move from {1}", destinationFilename, fileName); try { File.Delete(destinationFilename); } catch (Exception ex) { log.Error(ex); } } try { File.Move(fileName, destinationFilename); } catch { log.WarnFormat("Error moving {0} to {1}", fileName, destinationFilename); } if (ReadTimer.Interval != RuntimeOptions.ReadInterval) { //Reset timer interval ClearTimerBackoff(ReadTimer, RuntimeOptions); } } else { // Implement a timer backoff so we don't flood the endpoint IncrementTimerBackoff(ReadTimer, RuntimeOptions); log.WarnFormat("HTTP Transmission not OK - {0}", result); log.InfoFormat("Moving {0} to {1}", fileName, errorFolder); var destinationFilename = Path.Combine(errorFolder, Path.GetFileName(fileName)); if (File.Exists(destinationFilename)) { log.WarnFormat("Deleting file {0} in preparation for move from {1}", destinationFilename, fileName); try { File.Delete(destinationFilename); } catch (Exception ex) { log.Error(ex); } } try { File.Move(fileName, destinationFilename); } catch { log.WarnFormat("Error moving {0} to {1}", fileName, destinationFilename); } } } } } else { log.WarnFormat("aaLGX Directory {0} does not exist.", aaLGXDirectory); } } } catch (Exception ex) { log.Error(ex); } }
private void dostuff() { aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(); //txtLog.Text = ""; try { ++totalCount; //aaLogReader.LogRecord lastRecordRead = new LogRecord(); Stopwatch sw = Stopwatch.StartNew(); //int RecordsToRead = 100000; //addlog("Timer(ms) " + sw.ElapsedMilliseconds.ToString()); //addlog("Actual Records " + records.Count.ToString()); //addlog("Rate (records/s): " + ((int)(1000.0 * (float)records.Count / (float)sw.ElapsedMilliseconds)).ToString()); //addlog(""); //addlog(JsonConvert.SerializeObject(records, Formatting.Indented)); ////lastRecordRead = logReader.ReadStatusCacheFile(); //addlog(JsonConvert.SerializeObject(lastRecordRead, Formatting.Indented)); //addlog(""); //addlog(JsonConvert.SerializeObject(logReader.ReadLogHeader(), Formatting.Indented)); ////Debug.Print(JsonConvert.SerializeObject(record,Formatting.Indented)); List<LogRecord> records = logReader.GetUnreadRecords(); addlog(System.DateTime.Now.ToString() + " " + records.Count.ToString()); if (ForwardLogsCheckBox.Checked) { // Open socket to send logs to remote host TcpClient vSocket = new System.Net.Sockets.TcpClient(RemoteHost, RemotePort); System.Net.Sockets.NetworkStream ServerStream = vSocket.GetStream(); System.IO.StreamWriter swriter = new StreamWriter(ServerStream); foreach (LogRecord lr in records) { swriter.WriteLine(lr.ToKVP()); } //swriter.Close(); //ServerStream.Close(); vSocket.Close(); } ++successCount; failureLastTime = false; } catch (Exception ex) { ++failureCount; if (failureLastTime) { ++failureConsecCount; } else { failureConsecCount = 1; } failureLastTime = true; addlog("***********"); addlog(ex.ToString()); addlog("***********"); log.Error(ex); } finally { if (logReader != null) logReader.CloseCurrentLogFile(); } }