public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { double min = ((SingleValueParameter)inputDefinition.Parameters["min"]).ToDouble(); double max = ((SingleValueParameter)inputDefinition.Parameters["max"]).ToDouble(); while (true) { await Task.Delay(100); await eventWriter.QueueEventForWriting(new Event { Stanza = inputDefinition.Name, Data = "number=" + (rnd.NextDouble() * (max - min) + min) }); } }
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { int interval = 1000; try { // if user didn't give value for polling_interval, type conversion to SingleValueParameter will throw interval = int.Parse(((SingleValueParameter)inputDefinition.Parameters["polling_interval"]).ToString()); } catch (Exception) { } const string Seperator = @"://"; int index = inputDefinition.Name.IndexOf(Seperator) + Seperator.Length; string varName = inputDefinition.Name.Substring(index); string lastVarValue = null; while (true) { await Task.Delay(interval); string varValue = Environment.GetEnvironmentVariable(varName, EnvironmentVariableTarget.Machine); // Event data can't be null for real events. varValue = varValue ?? "(not exist)"; // Splunk does not record lines with only white spaces. varValue = string.IsNullOrWhiteSpace(varValue) ? "(white space)" : varValue; if (varValue != lastVarValue) { await eventWriter.QueueEventForWriting(new Event { Stanza = varName, Data = string.Format("{0}, interval={1}, inputDefinition.Name={2} , varName={3}", varValue, interval, inputDefinition.Name, varName) }); lastVarValue = varValue; } } }
/// <summary> /// Perform test verifications and stream events. /// </summary> /// <param name="inputDefinition">Input definition</param> public override void StreamEvents(InputDefinition inputDefinition) { // Verify every part of the input definition is received // parsed, and later recontructed correctly. var reconstructed = Serialize(inputDefinition); AssertEqualWithExpectedFile(InputDefinitionFilePath, reconstructed); // Test the dictionary for single value parameter. var stanza = inputDefinition.Stanzas["foobar://bbb"]; var parameterName = "param2"; // Test full parameter dictionary. var parameterValue = stanza.Parameters[parameterName]; var singleValue = (SingleValueParameter.Value) parameterValue; Assert.AreEqual("value22", singleValue); // Test single value parameter dictionary. var stringValue = stanza.SingleValueParameters[parameterName]; Assert.AreEqual("value22", stringValue); // Test the dictionary for multi value parameter. stanza = inputDefinition.Stanzas["foobar://bbb"]; parameterValue = stanza.Parameters["multiValue2"]; var multiValue = (MultiValueParameter.Value) parameterValue; var elementInMultiValue = multiValue[1]; Assert.AreEqual("value4", elementInMultiValue); // Stanza property can't be used since there are // more than one. stanza = null; try { stanza = inputDefinition.Stanza; } catch (InvalidOperationException e) { Assert.IsTrue(e.Message.Contains( "Use Stanzas property instead")); } Assert.IsNull(stanza); // Write events through EventStreamWriter. WriteEvents(); }
/// <summary> /// Streams the events. /// </summary> /// <param name="inputDefinition">The input definition.</param> public override void StreamEvents(InputDefinition inputDefinition) { string tailFilters = inputDefinition.Stanza.SingleValueParameters.GetValueOrDefault("tailfilterproperty",""); string filter = inputDefinition.Stanza.SingleValueParameters.GetValueOrDefault("filter", ""); string address = inputDefinition.Stanza.SingleValueParameters["address"]; string store = string.Empty; string last = string.Empty; bool tail = !string.IsNullOrEmpty(tailFilters); if (tail) { store = Path.Combine(inputDefinition.CheckpointDirectory, "lastvalue.txt"); last = File.Exists(store) ? File.ReadAllText(store) // .Split('\n') : inputDefinition.Stanza.SingleValueParameters.GetValueOrDefault("defaulttailfilter",""); //.Split('\n'); if (!string.IsNullOrEmpty(filter)) { filter = string.Format(filter, last); } else { address = string.Format(address, last); } } IEnumerable<IDictionary<string, object>> source = new Splunk.ODataSource() { Credentials = System.Net.CredentialCache.DefaultNetworkCredentials, Address = address, Resource = inputDefinition.Stanza.SingleValueParameters.GetValueOrDefault("resource",""), Filter = filter, //OrderBy = inputDefinition.Stanza.MultiValueParameterXmlElements, //Select = inputDefinition.Stanza.MultiValueParameterXmlElements, }; string includeEmptyString; bool includeEmpty = false; if(inputDefinition.Stanza.SingleValueParameters.TryGetValue("includeEmpty", out includeEmptyString)) { bool.TryParse(includeEmptyString, out includeEmpty); } using (var writer = new EventStreamWriter()) { foreach (IDictionary<string, object> item in source) { writer.Write(new EventElement { Data = item.ToString(includeEmpty: includeEmpty), Stanza = inputDefinition.Stanza.Name, }); if (tail) { last = ((IDictionary) item).SelectRecursive(tailFilters.Split(',')).ToStringInvariant(); } } if (tail) { using (var stream = File.Open(store, FileMode.Create, FileAccess.Write)) { using (var storeWriter = new StreamWriter(stream)) { storeWriter.WriteLine(last); } } } } }
/// <summary> /// Pulls down commit data from GitHub and creates events for each commit, which are then streamed to Splunk. /// </summary> /// <remarks> /// This function will be invoked once for each instance of the modular input, though that invocation /// may or may not be in separate processes, depending on how the modular input is configured. It should /// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt> /// (which is thread safe). /// </remarks> /// <param name="inputDefinition">The definition for this instance of the GitHub input, representing a GitHub repository.</param> /// <param name="eventWriter">An object that handles writing events to Splunk.</param> public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { var owner = ((SingleValueParameter)inputDefinition.Parameters["Owner"]).ToString(); var repository = ((SingleValueParameter)inputDefinition.Parameters["Repository"]).ToString(); var checkpointFilePath = Path.Combine(inputDefinition.CheckpointDirectory, owner + " " + repository + ".txt"); var productHeader = new ProductHeaderValue("splunk-sdk-csharp-github-commits"); ObservableGitHubClient client; if (!inputDefinition.Parameters.ContainsKey("Token") || String.IsNullOrWhiteSpace(((SingleValueParameter)inputDefinition.Parameters["Token"]).ToString())) { client = new ObservableGitHubClient(productHeader); } else { client = new ObservableGitHubClient(productHeader, new InMemoryCredentialStore(new Credentials(((SingleValueParameter)inputDefinition.Parameters["Token"]).ToString()))); } var shaKeys = new HashSet<string>(); var fileReader = new StreamReader(File.Open(checkpointFilePath, System.IO.FileMode.OpenOrCreate)); string line; while (!String.IsNullOrWhiteSpace(line = await fileReader.ReadLineAsync())) { shaKeys.Add(line); } fileReader.Close(); bool done = false; var fileWriter = new StreamWriter(checkpointFilePath); // Use Rx to stream an event for each commit as they come in client.Repository.Commits.GetAll(owner, repository).Subscribe( async githubCommit => { if (!shaKeys.Contains(githubCommit.Sha)) { await StreamCommit(githubCommit, eventWriter, owner, repository); await fileWriter.WriteLineAsync(githubCommit.Sha); // Write to the checkpoint file shaKeys.Add(githubCommit.Sha); await eventWriter.LogAsync(Severity.Info, repository + " indexed a Github commit with sha: " + githubCommit.Sha); } }, async e => { //error handing goes here await eventWriter.LogAsync(Severity.Error, e.GetType() + " - " + e.StackTrace); }, () => { //completion handling goes here fileWriter.Close(); done = true; } ); // Wait for Rx subscribe to finish above while (!done) { await Task.Delay(100); } }
/// <summary> /// Stream events into stdout /// </summary> /// <param name="inputDefinition">Input definition from Splunk</param> public override void StreamEvents(InputDefinition inputDefinition) { string lastVarValue = null; using (var writer = new EventStreamWriter()) { var stanza = inputDefinition.Stanza; // Gets input name. It is also the env var name. const string Seperator = @"://"; var indexInputName = stanza.Name.IndexOf(Seperator) + Seperator.Length; var varName = stanza.Name.Substring(indexInputName); SystemLogger.Write( string.Format( "Name of the var to monitor is : {0}", varName)); var interval = 1000; string intervalParam; if (stanza.SingleValueParameters.TryGetValue( PollingInterval, out intervalParam)) { interval = int.Parse(intervalParam); } SystemLogger.Write( string.Format( "Polling interval is : {0}", interval)); while (true) { var varValue = Environment.GetEnvironmentVariable( varName, EnvironmentVariableTarget.Machine); // Event data can't be null for real events. varValue = varValue ?? "(not exist)"; // Splunk does not record lines with only white spaces. varValue = string.IsNullOrWhiteSpace(varValue) ? "(white space)" : varValue; if (varValue != lastVarValue) { writer.Write( new EventElement { Source = varName, Data = varValue, }); lastVarValue = varValue; } Thread.Sleep(interval); } } }
/// <summary> /// Write events to Splunk from this modular input. /// </summary> /// <remarks> /// This function will be invoked once for each instance of the modular input, though that invocation /// may or may not be in separate processes, depending on how the modular input is configured. It should /// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt> /// (which is thread safe). /// </remarks> /// <param name="inputDefinition">a specification of this instance of the modular input.</param> /// <param name="eventWriter">an object that handles writing events to Splunk.</param> public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter) { try { string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString(); Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32(); Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32(); //Setup the options input OptionsStruct localOptionsStruct = new OptionsStruct(); localOptionsStruct.LogDirectory = logfilepath; // Initialize the log reader aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct); // Write an entry to the Splunk system log indicating we have initialized await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString()); while (true) { await (Task.Delay(cycletime)); //Simple call to get all unread records, limiting the return count to max message count List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount); // Loop through each lastRecordRead and send to Splunk foreach (LogRecord record in logRecords) { await eventWriter.QueueEventForWriting(new Event { Stanza = inputDefinition.Name, Data = record.ToKVP() }); } } } catch (Exception ex) { // Eat error message eventWriter.LogAsync(Severity.Error, ex.ToString()); } }
/// <summary> /// Streams events to Splunk through the provided EventWriter. /// </summary> /// <param name="inputDefinition"> /// Input definition from Splunk for this input. /// </param> /// <param name="eventWriter"> /// An object encapsulating writing events and log messages to Splunk. /// </param> public abstract Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter);