private (int sourceLoaded, int sourcesFailed) LoadEventSources()
{
var sourcesSection = _config.GetSection("Sources");
var sourceSections = sourcesSection.GetChildren();
int sourcesLoaded = 0;
int sourcesFailed = 0;
foreach (var sourceSection in sourceSections)
{
string id = _config.GetChildConfig(sourceSection.Path, ConfigConstants.ID);
string sourceType = _config.GetChildConfig(sourceSection.Path, ConfigConstants.SOURCE_TYPE);
var factory = _sourceFactoryCatalog.GetFactory(sourceType);
if (factory != null)
{
try
{
ISource source = (ISource)factory.CreateInstance(sourceType, CreatePlugInContext(sourceSection));
source.Id = id;
_sources[id] = source;
sourcesLoaded++;
}
catch (Exception ex)
{
sourcesFailed++;
_logger?.LogError("Unable to load event source {0} exception {1}", id, ex.ToMinimized());
}
}
else
{
_logger?.LogError("Source Type {0} is not recognized.", sourceType);
}
}
return(sourcesLoaded, sourcesFailed);
}
private void LoadCredentialProviders()
{
var credentialsSection = _config.GetSection("Credentials");
var credentialStarted = 0;
var credentialFailed = 0;
if (credentialsSection is null || !credentialsSection.GetChildren().Any())
{
//this config file contains no credentials section
return;
}
var credentialSections = credentialsSection.GetChildren();
foreach (var credentialSection in credentialSections)
{
var id = credentialSection[ConfigConstants.ID];
if (_credentialProviders.ContainsKey(id))
{
credentialFailed++;
continue;
}
var credentialType = credentialSection[ConfigConstants.CREDENTIAL_TYPE];
var factory = _credentialProviderFactoryCatalog.GetFactory(credentialType);
if (factory != null)
{
try
{
var credentialProvider = factory.CreateInstance(credentialType, CreatePlugInContext(credentialSection));
credentialProvider.Id = id;
_credentialProviders[id] = credentialProvider;
credentialStarted++;
_logger.LogDebug($"Created cred provider {credentialType} Id {id}");
}
catch (Exception ex)
{
_logger?.LogError(ex, "Unable to load credential {0}", id);
credentialFailed++;
}
}
else
{
_logger?.LogError("Credential Type {0} is not recognized.", credentialType);
credentialFailed++;
}
_metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>()
{
{ MetricsConstants.SINKS_STARTED, new MetricValue(credentialStarted) },
{ MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(credentialFailed) }
});
}
}
private void LoadEventSinks()
{
var sinksSection = _config.GetSection("Sinks");
var sinkSections = sinksSection.GetChildren();
int sinksStarted = 0;
int sinksFailed = 0;
foreach (var sinkSection in sinkSections)
{
string id = _config.GetChildConfig(sinkSection.Path, ConfigConstants.ID);
string sinkType = _config.GetChildConfig(sinkSection.Path, ConfigConstants.SINK_TYPE);
var factory = _sinkFactoryCatalog.GetFactory(sinkType);
if (factory != null)
{
IEventSink sink = null;
try
{
if (string.IsNullOrWhiteSpace(id))
{
throw new Exception("Sink id is required.");
}
sink = factory.CreateInstance(sinkType, CreatePlugInContext(sinkSection));
sink.Start();
_sinks[sink.Id] = sink;
sinksStarted++;
}
catch (Exception ex)
{
sinksFailed++;
_logger.LogError(0, ex, $"Unable to load event sink {id} exception");
// If the sink fails to start due to "Rate limit exceeded" error,
// add it to the sinks that need to be retarted
if (ex is RateExceededException)
{
_sinksToRestart[sink.Id] = sink;
}
}
}
else
{
sinksFailed++;
_logger.LogError("Sink Type {0} is not recognized.", sinkType);
}
}
_metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>()
{
{ MetricsConstants.SINKS_STARTED, new MetricValue(sinksStarted) },
{ MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(sinksFailed) }
});
}
private async Task LoadEventSinks(CancellationToken stopToken)
{
var sinksSection = _config.GetSection("Sinks");
var sinkSections = sinksSection.GetChildren();
var sinksStarted = 0;
var sinksFailed = 0;
foreach (var sinkSection in sinkSections)
{
var id = _config.GetChildConfig(sinkSection.Path, ConfigConstants.ID);
var sinkType = _config.GetChildConfig(sinkSection.Path, ConfigConstants.SINK_TYPE);
var factory = _sinkFactoryCatalog.GetFactory(sinkType);
if (factory != null)
{
IEventSink sink = null;
try
{
if (string.IsNullOrWhiteSpace(id))
{
throw new Exception("Sink id is required.");
}
sink = factory.CreateInstance(sinkType, CreatePlugInContext(sinkSection));
await sink.StartAsync(stopToken);
_sinks[sink.Id] = sink;
sinksStarted++;
}
catch (Exception ex)
{
sinksFailed++;
_logger.LogError(ex, $"Unable to load event sink {id}");
}
}
else
{
sinksFailed++;
_logger.LogError("Sink Type {0} is not recognized.", sinkType);
}
}
_metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>()
{
{ MetricsConstants.SINKS_STARTED, new MetricValue(sinksStarted) },
{ MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(sinksFailed) }
});
}
private void LoadCredentialProviders()
{
var credentialsSection = _config.GetSection("Credentials");
var credentialSections = credentialsSection.GetChildren();
int credentialStarted = 0;
int credentialFailed = 0;
foreach (var credentialSection in credentialSections)
{
string id = _config.GetChildConfig(credentialSection.Path, ConfigConstants.ID);
string credentialType = _config.GetChildConfig(credentialSection.Path, ConfigConstants.CREDENTIAL_TYPE);
var factory = _credentialProviderFactoryCatalog.GetFactory(credentialType);
if (factory != null)
{
try
{
ICredentialProvider credentialProvider = factory.CreateInstance(credentialType, CreatePlugInContext(credentialSection));
credentialProvider.Id = id;
_credentialProviders[id] = credentialProvider;
credentialStarted++;
}
catch (Exception ex)
{
credentialFailed++;
_logger?.LogError($"Unable to load event sink {id} exception {ex.ToMinimized()}");
}
}
else
{
credentialFailed++;
_logger?.LogError("Credential Type {0} is not recognized.", credentialType);
}
}
_metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>()
{
{ MetricsConstants.SINKS_STARTED, new MetricValue(credentialStarted) },
{ MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(credentialFailed) }
});
}
private (int sourceLoaded, int sourcesFailed) LoadEventSources()
{
//metrics source
_sources[KINESISTAP_METRICS_SOURCE] = _metrics;
var sourcesSection = _config.GetSection("Sources");
var sourceSections = sourcesSection.GetChildren();
var sourcesLoaded = 0;
var sourcesFailed = 0;
foreach (var sourceSection in sourceSections)
{
var id = _config.GetChildConfig(sourceSection.Path, ConfigConstants.ID);
var sourceType = _config.GetChildConfig(sourceSection.Path, ConfigConstants.SOURCE_TYPE);
var factory = _sourceFactoryCatalog.GetFactory(sourceType);
if (factory != null)
{
try
{
var source = factory.CreateInstance(sourceType, CreatePlugInContext(sourceSection));
source.Id = id;
_sources[id] = source;
sourcesLoaded++;
}
catch (Exception ex)
{
sourcesFailed++;
_logger.LogError(ex, "Unable to load event source {0}", id);
}
}
else
{
_logger.LogError("Source Type {0} is not recognized.", sourceType);
}
}
return(sourcesLoaded, sourcesFailed);
}
/// <summary>
/// Get the factory from catalog
/// </summary>
/// <param name="entry">The name of the factory to get</param>
/// <returns>A factory</returns>
public IFactory <T> GetFactory(string entry)
{
return(_factoryCatalog.GetFactory(entry));
}
/// <summary>
/// Create an instance o the DirectorySource
/// </summary>
/// <param name="entry">Name of the source</param>
/// <param name="context">Plug-in Context</param>
/// <returns></returns>
public virtual ISource CreateInstance(string entry, IPlugInContext context)
{
IConfiguration config = context.Configuration;
ILogger logger = context.Logger;
switch (entry.ToLower())
{
case "directorysource":
string recordParser = (config["RecordParser"] ?? string.Empty).ToLower();
string timetampFormat = config["TimestampFormat"];
string timestampField = config["TimestampField"];
DateTimeKind timeZoneKind = Utility.ParseTimeZoneKind(config["TimeZoneKind"]);
string removeUnmatchedConfig = config["RemoveUnmatched"];
bool removeUnmatched = false;
if (!string.IsNullOrWhiteSpace(removeUnmatchedConfig))
{
removeUnmatched = bool.Parse(removeUnmatchedConfig);
}
string extractionPattern = config["ExtractionPattern"];
string extractionRegexOptions = config["ExtractionRegexOptions"];
switch (recordParser)
{
case "singleline":
return(CreateEventSource(context,
new SingeLineRecordParser()));
case "regex":
string pattern = config["Pattern"];
return(CreateEventSource(context,
new RegexRecordParser(pattern,
timetampFormat,
logger,
extractionPattern,
extractionRegexOptions,
timeZoneKind,
new RegexRecordParserOptions {
RemoveUnmatchedRecord = removeUnmatched
})));
case "timestamp":
return(CreateEventSource(context,
new TimeStampRecordParser(timetampFormat, logger, extractionPattern, extractionRegexOptions, timeZoneKind,
new RegexRecordParserOptions {
RemoveUnmatchedRecord = removeUnmatched
})));
case "syslog":
return(CreateEventSource(context,
new SysLogParser(logger, timeZoneKind,
new RegexRecordParserOptions {
RemoveUnmatchedRecord = removeUnmatched
})));
case "delimited":
return(CreateEventSourceWithDelimitedLogParser(context, timetampFormat, timeZoneKind));
case "singlelinejson":
return(CreateEventSource(context,
new SingleLineJsonParser(timestampField, timetampFormat)));
default:
IFactoryCatalog <IRecordParser> parserFactories =
context?.ContextData?[PluginContext.PARSER_FACTORIES] as IFactoryCatalog <IRecordParser>;
var parserFactory = parserFactories.GetFactory(recordParser);
if (parserFactory == null)
{
throw new ArgumentException($"Unknown parser {recordParser}");
}
else
{
return(CreateEventSource(context,
parserFactory.CreateInstance(recordParser, context)));
}
}
case "w3svclogsource":
return(CreateEventSource(
context,
new W3SVCLogParser(context)));
default:
throw new ArgumentException($"Source {entry} not recognized.");
}
}