/// <summary> /// Handle trapping of ctrl+break and ctrl+c signals. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void TrapControlSignals(object sender, ConsoleCancelEventArgs e) { if (e.SpecialKey == ConsoleSpecialKey.ControlBreak || e.SpecialKey == ConsoleSpecialKey.ControlC) { if (writer != null) { writer.LogAsync(Severity.Info, "Splunk signaled shutdown via control signal.").Wait(); } FireShutdownEvent(); e.Cancel = true; return; } e.Cancel = false; }
/// <summary> /// Performs the action specified by the <paramref name="args"/> parameter. /// </summary> /// <param name="args"> /// Command-line arguments provided by Splunk when it invokes the /// modular input program. Implementers should pass the arguments to /// the main method of the program as the value of this parameter. /// </param> /// <returns> /// <param name="stdin"> /// Reader to use for the stdin stream /// </param> /// <param name="stdout"> /// Writer to use for the stdout stream /// </param> /// <param name="stderr"> /// Writer to use for the stderr stream /// </param> /// <param name="progress"> /// Reports back progress as events are written to the <see cref="EventWriter"/> /// </param> /// <param name="attachPoints"> /// Defines the <see cref="DebuggerAttachPoints"/> for this input /// </param> /// <param name="timeout"> /// Number of seconds to wait for a debugger to attach before continuing processing. /// </param> /// A value which should be used as the exit code from the modular /// input program. A value of <c>0</c> indicates success. A non-zero /// value indicates failure. /// </returns> /// <remarks> /// If the <paramref name="args"/> are not in the supported set of values, /// the method will do nothing and return a non-zero value. Any /// exceptions and internal progress messages encountered during /// execution are written to the splunkd log file. /// </remarks> public async Task <int> RunAsync(string[] args, TextReader stdin = null, TextWriter stdout = null, TextWriter stderr = null, IProgress <EventWrittenProgressReport> progress = null, DebuggerAttachPoints attachPoints = DebuggerAttachPoints.None, uint timeout = 0 ) { EventWriter writer = null; string name = this.GetType().FullName; try { /// Console default is OEM text encoding, which is not handled by Splunk, //// resulting in loss of chars such as O with an umlaut (\u0150) //// Splunk's default is UTF8. if (stdin == null) { stdin = Console.In; Console.InputEncoding = Encoding.UTF8; } if (stdout == null) { stdout = Console.Out; Console.OutputEncoding = Encoding.UTF8; } if (stderr == null) { stderr = Console.Error; Console.OutputEncoding = Encoding.UTF8; } if (progress == null) { progress = new Progress <EventWrittenProgressReport>(); } writer = new EventWriter(stdout, stderr, progress); // Check if the developer has specified they want to attach a debugger bool wait = ShouldWaitForDebuggerToAttach(args, attachPoints); // If a debugger is going to attach if (wait) { WaitForAttach(timeout); } if (args.Length == 0) { try { List <Task> instances = new List <Task>(); InputDefinitionCollection inputDefinitions = (InputDefinitionCollection) new XmlSerializer(typeof(InputDefinitionCollection)). Deserialize(stdin); foreach (InputDefinition inputDefinition in inputDefinitions) { var inputTask = this.StreamEventsAsync(inputDefinition, writer); instances.Add(inputTask); var inputName = inputDefinition.Name; inputTask.ContinueWith(t => { if (inputTask.Exception != null) { var message = RemoveNewLines(inputTask.Exception.InnerException.ToString()); writer.LogAsync(Severity.Fatal, string.Format("Exception during streaming: name={0} | {1}", inputName, message)) .Wait(); } }); } try { await Task.WhenAll(instances.ToArray()); } catch { } await writer.CompleteAsync(); } catch (Exception e) { var message = RemoveNewLines(e.ToString()); writer.LogAsync(Severity.Fatal, string.Format("Exception during streaming: name={0} | {1}", name, message)) .Wait(); return(-1); } return(0); } if (args[0].ToLower().Equals("--scheme")) { Scheme scheme = null; try { scheme = this.Scheme; if (scheme != null) { StringWriter stringWriter = new StringWriter(); new XmlSerializer(typeof(Scheme)).Serialize(stringWriter, scheme); stdout.WriteLine(stringWriter.ToString()); return(0); } throw new NullReferenceException("Scheme was null; could not serialize."); } catch (Exception e) { var message = RemoveNewLines(e.ToString()); writer.LogAsync(Severity.Fatal, string.Format("Exception getting scheme: name={0} | {1}", name, message)) .Wait(); return(-1); } finally { writer.CompleteAsync().Wait(); } } if (args[0].ToLower().Equals("--validate-arguments")) { string errorMessage = null; string inputDoc = null; try { inputDoc = await stdin.ReadToEndAsync(); inputDoc = RemoveNewLines(inputDoc); writer.LogAsync(Severity.Info, inputDoc).Wait(); Validation validation = (Validation) new XmlSerializer(typeof(Validation)). Deserialize(new StringReader(inputDoc)); name = validation.Name; bool validationSuccessful = true; Scheme scheme = this.Scheme; foreach (Argument arg in scheme.Arguments) { if (arg.ValidationDelegate != null) { if (!arg.ValidationDelegate(validation.Parameters[arg.Name], out errorMessage)) { validationSuccessful = false; break; } } } if (validationSuccessful && this.Validate(validation, out errorMessage)) { return(0); // Validation succeeded } } catch (Exception e) { var message = RemoveNewLines(e.ToString()); errorMessage = e.Message; writer.LogAsync(Severity.Fatal, string.Format("Exception during validation: name={0} | {1}", name, message)) .Wait(); } finally { writer.CompleteAsync().Wait(); } if (errorMessage != null) { using (var xmlWriter = XmlWriter.Create(stdout, new XmlWriterSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment })) { await xmlWriter.WriteStartElementAsync(prefix : null, localName : "error", ns : null); await xmlWriter.WriteElementStringAsync(prefix : null, localName : "message", ns : null, value : errorMessage); await xmlWriter.WriteEndElementAsync(); } } return(-1); } } catch (Exception e) { if (writer != null) { var message = RemoveNewLines(e.ToString()); writer.LogAsync(Severity.Error, string.Format("Exception during execution: name={0} | {1}", name, message)).Wait(); } } finally { if (writer != null) { writer.CompleteAsync().Wait(); } } await writer.LogAsync(Severity.Error, string.Format("Exception during execution: Invalid arguments: name={0}", name)); return(-1); }
/// <summary> /// Pulls down commit data from GitHub and creates events for each commit, which are then streamed to Splunk. /// </summary> /// <remarks> /// This function will be invoked once for each instance of the modular input, though that invocation /// may or may not be in separate processes, depending on how the modular input is configured. It should /// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt> /// (which is thread safe). /// </remarks> /// <param name="inputDefinition">The definition for this instance of the GitHub input, representing a GitHub repository.</param> /// <param name="eventWriter">An object that handles writing events to Splunk.</param> public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { var owner = ((SingleValueParameter)inputDefinition.Parameters["Owner"]).ToString(); var repository = ((SingleValueParameter)inputDefinition.Parameters["Repository"]).ToString(); var checkpointFilePath = Path.Combine(inputDefinition.CheckpointDirectory, owner + " " + repository + ".txt"); var productHeader = new ProductHeaderValue("splunk-sdk-csharp-github-commits"); ObservableGitHubClient client; if (!inputDefinition.Parameters.ContainsKey("Token") || String.IsNullOrWhiteSpace(((SingleValueParameter)inputDefinition.Parameters["Token"]).ToString())) { client = new ObservableGitHubClient(productHeader); } else { client = new ObservableGitHubClient(productHeader, new InMemoryCredentialStore(new Credentials(((SingleValueParameter)inputDefinition.Parameters["Token"]).ToString()))); } var shaKeys = new HashSet<string>(); var fileReader = new StreamReader(File.Open(checkpointFilePath, System.IO.FileMode.OpenOrCreate)); string line; while (!String.IsNullOrWhiteSpace(line = await fileReader.ReadLineAsync())) { shaKeys.Add(line); } fileReader.Close(); bool done = false; var fileWriter = new StreamWriter(checkpointFilePath); // Use Rx to stream an event for each commit as they come in client.Repository.Commits.GetAll(owner, repository).Subscribe( async githubCommit => { if (!shaKeys.Contains(githubCommit.Sha)) { await StreamCommit(githubCommit, eventWriter, owner, repository); await fileWriter.WriteLineAsync(githubCommit.Sha); // Write to the checkpoint file shaKeys.Add(githubCommit.Sha); await eventWriter.LogAsync(Severity.Info, repository + " indexed a Github commit with sha: " + githubCommit.Sha); } }, async e => { //error handing goes here await eventWriter.LogAsync(Severity.Error, e.GetType() + " - " + e.StackTrace); }, () => { //completion handling goes here fileWriter.Close(); done = true; } ); // Wait for Rx subscribe to finish above while (!done) { await Task.Delay(100); } }
/// <summary> /// Performs the action specified by the <paramref name="args"/> parameter. /// </summary> /// <param name="args"> /// Command-line arguments provided by Splunk when it invokes the /// modular input program. Implementers should pass the arguments to /// the main method of the program as the value of this parameter. /// </param> /// <returns> /// <param name="stdin"> /// Reader to use for the stdin stream /// </param> /// <param name="stdout"> /// Writer to use for the stdout stream /// </param> /// <param name="stderr"> /// Writer to use for the stderr stream /// </param> /// <param name="progress"> /// Reports back progress as events are written to the <see cref="EventWriter"/> /// </param> /// <param name="attachPoints"> /// Defines the <see cref="DebuggerAttachPoints"/> for this input /// </param> /// <param name="timeout"> /// Number of seconds to wait for a debugger to attach before continuing processing. /// </param> /// A value which should be used as the exit code from the modular /// input program. A value of <c>0</c> indicates success. A non-zero /// value indicates failure. /// </returns> /// <remarks> /// If the <paramref name="args"/> are not in the supported set of values, /// the method will do nothing and return a non-zero value. Any /// exceptions and internal progress messages encountered during /// execution are written to the splunkd log file. /// </remarks> public async Task <int> RunAsync(string[] args, TextReader stdin = null, TextWriter stdout = null, TextWriter stderr = null, IProgress <EventWrittenProgressReport> progress = null, DebuggerAttachPoints attachPoints = DebuggerAttachPoints.None, uint timeout = 0 ) { EventWriter writer = null; string name = this.GetType().FullName; try { /// Console default is OEM text encoding, which is not handled by Splunk, //// resulting in loss of chars such as O with an umlaut (\u0150) //// Splunk's default is UTF8. if (stdin == null) { stdin = Console.In; Console.InputEncoding = Encoding.UTF8; } if (stdout == null) { stdout = Console.Out; Console.OutputEncoding = Encoding.UTF8; } if (stderr == null) { stderr = Console.Error; Console.OutputEncoding = Encoding.UTF8; } if (progress == null) { progress = new Progress <EventWrittenProgressReport>(); } writer = new EventWriter(stdout, stderr, progress); // Check if the developer has specified they want to attach a debugger bool wait = ShouldWaitForDebuggerToAttach(args, attachPoints); // If a debugger is going to attach if (wait) { WaitForAttach(timeout); } if (args.Length == 0) { try { var serializer = new XmlSerializer(typeof(InputDefinitionCollection)); var inputDefinitions = (InputDefinitionCollection)serializer.Deserialize(stdin); var instances = new List <Task>(); foreach (InputDefinition inputDefinition in inputDefinitions) { var inputTask = this.StreamEventsAsync(inputDefinition, writer); instances.Add(inputTask); # pragma warning disable 4014 inputTask.ContinueWith(t => { if (inputTask.Exception != null) { var message = RemoveNewLines(inputTask.Exception.InnerException.ToString()); writer.LogAsync(Severity.Fatal, string.Format("Exception during streaming: name={0} | {1}", inputDefinition.Name, message)).Wait(); } }); # pragma warning restore 4014 } try { await Task.WhenAll(instances.ToArray()); } catch { } await writer.CompleteAsync(); } catch (Exception e) { var message = RemoveNewLines(e.ToString()); writer.LogAsync(Severity.Fatal, string.Format("Exception during streaming: name={0} | {1}", name, message)).Wait(); return(-1); } return(0); }
/// <summary> /// Write events to Splunk from this modular input. /// </summary> /// <remarks> /// This function will be invoked once for each instance of the modular input, though that invocation /// may or may not be in separate processes, depending on how the modular input is configured. It should /// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt> /// (which is thread safe). /// </remarks> /// <param name="inputDefinition">a specification of this instance of the modular input.</param> /// <param name="eventWriter">an object that handles writing events to Splunk.</param> public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { try { string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString(); Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32(); Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32(); //Setup the options input OptionsStruct localOptionsStruct = new OptionsStruct(); localOptionsStruct.LogDirectory = logfilepath; // Initialize the log reader aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct); // Write an entry to the Splunk system log indicating we have initialized await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString()); while (true) { await (Task.Delay(cycletime)); //Simple call to get all unread records, limiting the return count to max message count List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount); // Loop through each lastRecordRead and send to Splunk foreach (LogRecord record in logRecords) { await eventWriter.QueueEventForWriting(new Event { Stanza = inputDefinition.Name, Data = record.ToKVP() }); } } } catch (Exception ex) { // Eat error message eventWriter.LogAsync(Severity.Error, ex.ToString()); } }
/// <summary> /// Performs the action specified by the <paramref name="args"/> parameter. /// </summary> /// <param name="args"> /// Command-line arguments provided by Splunk when it invokes the /// modular input program. Implementers should pass the arguments to /// the main method of the program as the value of this parameter. /// </param> /// <returns> /// A value which should be used as the exit code from the modular /// input program. A value of <c>0</c> indicates success. A non-zero /// value indicates failure. /// </returns> /// <remarks> /// If the <paramref name="args"/> are not in the supported set of values, /// the method will do nothing and return a non-zero value. Any /// exceptions and internal progress messages encountered during /// execution are written to the splunkd log file. /// </remarks> public async Task <int> RunAsync(string[] args, TextReader stdin = null, TextWriter stdout = null, TextWriter stderr = null, IProgress <EventWrittenProgressReport> progress = null) { if (progress == null) { progress = new Progress <EventWrittenProgressReport>(); } /// Console default is OEM text encoding, which is not handled by Splunk, //// resulting in loss of chars such as O with an umlaut (\u0150) //// Splunk's default is UTF8. if (stdin == null) { stdin = Console.In; Console.InputEncoding = Encoding.UTF8; } if (stdout == null) { stdout = Console.Out; Console.OutputEncoding = Encoding.UTF8; } if (stderr == null) { stderr = Console.Error; Console.OutputEncoding = Encoding.UTF8; } EventWriter writer = new EventWriter(stdout, stderr, progress); try { if (args.Length == 0) { List <Task> instances = new List <Task>(); InputDefinitionCollection inputDefinitions = (InputDefinitionCollection) new XmlSerializer(typeof(InputDefinitionCollection)). Deserialize(stdin); foreach (InputDefinition inputDefinition in inputDefinitions) { instances.Add(this.StreamEventsAsync(inputDefinition, writer)); } await Task.WhenAll(instances.ToArray()); return(0); } else if (args[0].ToLower().Equals("--scheme")) { Scheme scheme = this.Scheme; if (scheme != null) { StringWriter stringWriter = new StringWriter(); new XmlSerializer(typeof(Scheme)).Serialize(stringWriter, scheme); stdout.WriteLine(stringWriter.ToString()); return(0); } else { throw new NullReferenceException("Scheme was null; could not serialize."); } } else if (args[0].ToLower().Equals("--validate-arguments")) { string errorMessage = null; Validation validation = (Validation) new XmlSerializer(typeof(Validation)). Deserialize(stdin); try { bool validationSuccessful = true; Scheme scheme = this.Scheme; foreach (Argument arg in scheme.Arguments) { if (arg.ValidationDelegate != null) { if (!arg.ValidationDelegate(validation.Parameters[arg.Name], out errorMessage)) { validationSuccessful = false; break; } } } if (validationSuccessful && this.Validate(validation, out errorMessage)) { return(0); // Validation succeeded } } catch (Exception) { } using (var xmlWriter = XmlWriter.Create(stdout, new XmlWriterSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment })) { await xmlWriter.WriteStartElementAsync(prefix : null, localName : "error", ns : null); await xmlWriter.WriteElementStringAsync(prefix : null, localName : "message", ns : null, value : errorMessage); await xmlWriter.WriteEndElementAsync(); } return(-1); } else { await writer.LogAsync("ERROR", "Invalid arguments to modular input."); return(-1); } } catch (Exception e) { var full = e.ToString().Replace(Environment.NewLine, " | "); writer.LogAsync(string.Format("Unhandled exception: {0}", full), "FATAL").Wait(); return(-1); } finally { writer.CompleteAsync().Wait(); } }