public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
double min = ((SingleValueParameter)inputDefinition.Parameters["min"]).ToDouble();
double max = ((SingleValueParameter)inputDefinition.Parameters["max"]).ToDouble();
while (true)
{
await Task.Delay(100);
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = "number=" + (rnd.NextDouble() * (max - min) + min)
});
}
}
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
int interval = 1000;
try
{
// if user didn't give value for polling_interval, type conversion to SingleValueParameter will throw
interval = int.Parse(((SingleValueParameter)inputDefinition.Parameters["polling_interval"]).ToString());
}
catch (Exception)
{
}
const string Seperator = @"://";
int index = inputDefinition.Name.IndexOf(Seperator) + Seperator.Length;
string varName = inputDefinition.Name.Substring(index);
string lastVarValue = null;
while (true)
{
await Task.Delay(interval);
string varValue = Environment.GetEnvironmentVariable(varName, EnvironmentVariableTarget.Machine);
// Event data can't be null for real events.
varValue = varValue ?? "(not exist)";
// Splunk does not record lines with only white spaces.
varValue = string.IsNullOrWhiteSpace(varValue) ? "(white space)" : varValue;
if (varValue != lastVarValue)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = varName,
Data = string.Format("{0}, interval={1}, inputDefinition.Name={2} , varName={3}",
varValue, interval, inputDefinition.Name, varName)
});
lastVarValue = varValue;
}
}
}
/// <summary>
/// Pulls down commit data from GitHub and creates events for each commit, which are then streamed to Splunk.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">The definition for this instance of the GitHub input, representing a GitHub repository.</param>
/// <param name="eventWriter">An object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
var owner = ((SingleValueParameter)inputDefinition.Parameters["Owner"]).ToString();
var repository = ((SingleValueParameter)inputDefinition.Parameters["Repository"]).ToString();
var checkpointFilePath = Path.Combine(inputDefinition.CheckpointDirectory, owner + " " + repository + ".txt");
var productHeader = new ProductHeaderValue("splunk-sdk-csharp-github-commits");
ObservableGitHubClient client;
if (!inputDefinition.Parameters.ContainsKey("Token") || String.IsNullOrWhiteSpace(((SingleValueParameter)inputDefinition.Parameters["Token"]).ToString()))
{
client = new ObservableGitHubClient(productHeader);
}
else
{
client = new ObservableGitHubClient(productHeader, new InMemoryCredentialStore(new Credentials(((SingleValueParameter)inputDefinition.Parameters["Token"]).ToString())));
}
var shaKeys = new HashSet<string>();
var fileReader = new StreamReader(File.Open(checkpointFilePath, System.IO.FileMode.OpenOrCreate));
string line;
while (!String.IsNullOrWhiteSpace(line = await fileReader.ReadLineAsync()))
{
shaKeys.Add(line);
}
fileReader.Close();
bool done = false;
var fileWriter = new StreamWriter(checkpointFilePath);
// Use Rx to stream an event for each commit as they come in
client.Repository.Commits.GetAll(owner, repository).Subscribe(
async githubCommit =>
{
if (!shaKeys.Contains(githubCommit.Sha))
{
await StreamCommit(githubCommit, eventWriter, owner, repository);
await fileWriter.WriteLineAsync(githubCommit.Sha); // Write to the checkpoint file
shaKeys.Add(githubCommit.Sha);
await eventWriter.LogAsync(Severity.Info, repository + " indexed a Github commit with sha: " + githubCommit.Sha);
}
},
async e =>
{
//error handing goes here
await eventWriter.LogAsync(Severity.Error, e.GetType() + " - " + e.StackTrace);
},
() =>
{
//completion handling goes here
fileWriter.Close();
done = true;
}
);
// Wait for Rx subscribe to finish above
while (!done)
{
await Task.Delay(100);
}
}
/// <summary>
/// Handles creating an Event object from a GitHub commit which will be streamed into Splunk.
/// </summary>
/// <param name="githubCommit">The individual GithubCommit object which holds the data for a commit.</param>
/// <param name="eventWriter">The EventWriter for streaming events to Splunk.</param>
/// <param name="owner">The GitHub repository owner's name.</param>
/// <param name="repositoryName">The GitHub repository's name.</param>
public async Task StreamCommit(GitHubCommit githubCommit, EventWriter eventWriter, string owner, string repositoryName)
{
string authorName = githubCommit.Commit.Author.Name;
DateTime date = githubCommit.Commit.Author.Date;
// Replace any newlines with a space
string commitMessage = Regex.Replace(githubCommit.Commit.Message, "\\n|\\r", " ");
dynamic json = new JObject();
json.sha = githubCommit.Sha;
json.api_url = githubCommit.Url;
json.url = "http://github.com/" + owner + "/" + repositoryName + "/commit/" + githubCommit.Sha;
json.message = commitMessage;
json.author = authorName;
json.date = date.ToString();
var commitEvent = new Event();
commitEvent.Stanza = repositoryName;
commitEvent.SourceType = "github_commits";
commitEvent.Time = date;
commitEvent.Data = json.ToString(Formatting.None);
await eventWriter.QueueEventForWriting(commitEvent);
}
/// <summary>
/// Performs the action specified by the <paramref name="args"/> parameter.
/// </summary>
/// <param name="args">
/// Command-line arguments provided by Splunk when it invokes the
/// modular input program. Implementers should pass the arguments to
/// the main method of the program as the value of this parameter.
/// </param>
/// <returns>
/// <param name="stdin">
/// Reader to use for the stdin stream
/// </param>
/// <param name="stdout">
/// Writer to use for the stdout stream
/// </param>
/// <param name="stderr">
/// Writer to use for the stderr stream
/// </param>
/// <param name="progress">
/// Reports back progress as events are written to the <see cref="EventWriter"/>
/// </param>
/// <param name="attachPoints">
/// Defines the <see cref="DebuggerAttachPoints"/> for this input
/// </param>
/// <param name="timeout">
/// Number of seconds to wait for a debugger to attach before continuing processing.
/// </param>
/// A value which should be used as the exit code from the modular
/// input program. A value of <c>0</c> indicates success. A non-zero
/// value indicates failure.
/// </returns>
/// <remarks>
/// If the <paramref name="args"/> are not in the supported set of values,
/// the method will do nothing and return a non-zero value. Any
/// exceptions and internal progress messages encountered during
/// execution are written to the splunkd log file.
/// </remarks>
public async Task <int> RunAsync(string[] args,
TextReader stdin = null,
TextWriter stdout = null,
TextWriter stderr = null,
IProgress <EventWrittenProgressReport> progress = null,
DebuggerAttachPoints attachPoints = DebuggerAttachPoints.None,
uint timeout = 0
)
{
EventWriter writer = null;
string name = this.GetType().FullName;
try
{
/// Console default is OEM text encoding, which is not handled by Splunk,
//// resulting in loss of chars such as O with an umlaut (\u0150)
//// Splunk's default is UTF8.
if (stdin == null)
{
stdin = Console.In;
Console.InputEncoding = Encoding.UTF8;
}
if (stdout == null)
{
stdout = Console.Out;
Console.OutputEncoding = Encoding.UTF8;
}
if (stderr == null)
{
stderr = Console.Error;
Console.OutputEncoding = Encoding.UTF8;
}
if (progress == null)
{
progress = new Progress <EventWrittenProgressReport>();
}
writer = new EventWriter(stdout, stderr, progress);
// Check if the developer has specified they want to attach a debugger
bool wait = ShouldWaitForDebuggerToAttach(args, attachPoints);
// If a debugger is going to attach
if (wait)
{
WaitForAttach(timeout);
}
if (args.Length == 0)
{
try
{
var serializer = new XmlSerializer(typeof(InputDefinitionCollection));
var inputDefinitions = (InputDefinitionCollection)serializer.Deserialize(stdin);
var instances = new List <Task>();
foreach (InputDefinition inputDefinition in inputDefinitions)
{
var inputTask = this.StreamEventsAsync(inputDefinition, writer);
instances.Add(inputTask);
# pragma warning disable 4014
inputTask.ContinueWith(t =>
{
if (inputTask.Exception != null)
{
var message = RemoveNewLines(inputTask.Exception.InnerException.ToString());
writer.LogAsync(Severity.Fatal, string.Format("Exception during streaming: name={0} | {1}", inputDefinition.Name, message)).Wait();
}
});
# pragma warning restore 4014
}
try
{
await Task.WhenAll(instances.ToArray());
}
catch
{ }
await writer.CompleteAsync();
}
catch (Exception e)
{
var message = RemoveNewLines(e.ToString());
writer.LogAsync(Severity.Fatal, string.Format("Exception during streaming: name={0} | {1}",
name, message)).Wait();
return(-1);
}
return(0);
}
/// <summary>
/// Write events to Splunk from this modular input.
/// </summary>
/// <remarks>
/// This function will be invoked once for each instance of the modular input, though that invocation
/// may or may not be in separate processes, depending on how the modular input is configured. It should
/// extract the arguments it needs from <tt>inputDefinition</tt>, then write events to <tt>eventWriter</tt>
/// (which is thread safe).
/// </remarks>
/// <param name="inputDefinition">a specification of this instance of the modular input.</param>
/// <param name="eventWriter">an object that handles writing events to Splunk.</param>
public override async Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter)
{
try
{
string logfilepath = ((SingleValueParameter)(inputDefinition.Parameters["logfilepath"])).ToString();
Int32 maxmessagecount = ((SingleValueParameter)(inputDefinition.Parameters["maxmessagecount"])).ToInt32();
Int32 cycletime = ((SingleValueParameter)(inputDefinition.Parameters["cycletime"])).ToInt32();
//Setup the options input
OptionsStruct localOptionsStruct = new OptionsStruct();
localOptionsStruct.LogDirectory = logfilepath;
// Initialize the log reader
aaLogReader.aaLogReader logReader = new aaLogReader.aaLogReader(localOptionsStruct);
// Write an entry to the Splunk system log indicating we have initialized
await eventWriter.LogAsync(Severity.Info, "Initialized Log reader for path " + logfilepath + " and message count " + maxmessagecount.ToString());
while (true)
{
await (Task.Delay(cycletime));
//Simple call to get all unread records, limiting the return count to max message count
List<LogRecord> logRecords = logReader.GetUnreadRecords((ulong)maxmessagecount);
// Loop through each lastRecordRead and send to Splunk
foreach (LogRecord record in logRecords)
{
await eventWriter.QueueEventForWriting(new Event
{
Stanza = inputDefinition.Name,
Data = record.ToKVP()
});
}
}
}
catch (Exception ex)
{
// Eat error message
eventWriter.LogAsync(Severity.Error, ex.ToString());
}
}
/// <summary>
/// Performs the action specified by the <paramref name="args"/> parameter.
/// </summary>
/// <param name="args">
/// Command-line arguments provided by Splunk when it invokes the
/// modular input program. Implementers should pass the arguments to
/// the main method of the program as the value of this parameter.
/// </param>
/// <returns>
/// A value which should be used as the exit code from the modular
/// input program. A value of <c>0</c> indicates success. A non-zero
/// value indicates failure.
/// </returns>
/// <remarks>
/// If the <paramref name="args"/> are not in the supported set of values,
/// the method will do nothing and return a non-zero value. Any
/// exceptions and internal progress messages encountered during
/// execution are written to the splunkd log file.
/// </remarks>
public async Task <int> RunAsync(string[] args,
TextReader stdin = null,
TextWriter stdout = null,
TextWriter stderr = null,
IProgress <EventWrittenProgressReport> progress = null)
{
if (progress == null)
{
progress = new Progress <EventWrittenProgressReport>();
}
/// Console default is OEM text encoding, which is not handled by Splunk,
//// resulting in loss of chars such as O with an umlaut (\u0150)
//// Splunk's default is UTF8.
if (stdin == null)
{
stdin = Console.In;
Console.InputEncoding = Encoding.UTF8;
}
if (stdout == null)
{
stdout = Console.Out;
Console.OutputEncoding = Encoding.UTF8;
}
if (stderr == null)
{
stderr = Console.Error;
Console.OutputEncoding = Encoding.UTF8;
}
EventWriter writer = new EventWriter(stdout, stderr, progress);
try
{
if (args.Length == 0)
{
List <Task> instances = new List <Task>();
InputDefinitionCollection inputDefinitions =
(InputDefinitionCollection) new XmlSerializer(typeof(InputDefinitionCollection)).
Deserialize(stdin);
foreach (InputDefinition inputDefinition in inputDefinitions)
{
instances.Add(this.StreamEventsAsync(inputDefinition, writer));
}
await Task.WhenAll(instances.ToArray());
return(0);
}
else if (args[0].ToLower().Equals("--scheme"))
{
Scheme scheme = this.Scheme;
if (scheme != null)
{
StringWriter stringWriter = new StringWriter();
new XmlSerializer(typeof(Scheme)).Serialize(stringWriter, scheme);
stdout.WriteLine(stringWriter.ToString());
return(0);
}
else
{
throw new NullReferenceException("Scheme was null; could not serialize.");
}
}
else if (args[0].ToLower().Equals("--validate-arguments"))
{
string errorMessage = null;
Validation validation = (Validation) new XmlSerializer(typeof(Validation)).
Deserialize(stdin);
try
{
bool validationSuccessful = true;
Scheme scheme = this.Scheme;
foreach (Argument arg in scheme.Arguments)
{
if (arg.ValidationDelegate != null)
{
if (!arg.ValidationDelegate(validation.Parameters[arg.Name], out errorMessage))
{
validationSuccessful = false;
break;
}
}
}
if (validationSuccessful && this.Validate(validation, out errorMessage))
{
return(0); // Validation succeeded
}
}
catch (Exception) { }
using (var xmlWriter = XmlWriter.Create(stdout, new XmlWriterSettings
{
Async = true,
ConformanceLevel = ConformanceLevel.Fragment
}))
{
await xmlWriter.WriteStartElementAsync(prefix : null, localName : "error", ns : null);
await xmlWriter.WriteElementStringAsync(prefix : null, localName : "message", ns : null, value : errorMessage);
await xmlWriter.WriteEndElementAsync();
}
return(-1);
}
else
{
await writer.LogAsync("ERROR", "Invalid arguments to modular input.");
return(-1);
}
}
catch (Exception e)
{
var full = e.ToString().Replace(Environment.NewLine, " | ");
writer.LogAsync(string.Format("Unhandled exception: {0}", full), "FATAL").Wait();
return(-1);
}
finally
{
writer.CompleteAsync().Wait();
}
}
/// <summary> /// Streams events to Splunk through the provided EventWriter. /// </summary> /// <param name="inputDefinition"> /// Input definition from Splunk for this input. /// </param> /// <param name="eventWriter"> /// An object encapsulating writing events and log messages to Splunk. /// </param> public abstract Task StreamEventsAsync(InputDefinition inputDefinition, EventWriter eventWriter);
