private (int sourceLoaded, int sourcesFailed) LoadEventSources() { var sourcesSection = _config.GetSection("Sources"); var sourceSections = sourcesSection.GetChildren(); int sourcesLoaded = 0; int sourcesFailed = 0; foreach (var sourceSection in sourceSections) { string id = _config.GetChildConfig(sourceSection.Path, ConfigConstants.ID); string sourceType = _config.GetChildConfig(sourceSection.Path, ConfigConstants.SOURCE_TYPE); var factory = _sourceFactoryCatalog.GetFactory(sourceType); if (factory != null) { try { ISource source = (ISource)factory.CreateInstance(sourceType, CreatePlugInContext(sourceSection)); source.Id = id; _sources[id] = source; sourcesLoaded++; } catch (Exception ex) { sourcesFailed++; _logger?.LogError("Unable to load event source {0} exception {1}", id, ex.ToMinimized()); } } else { _logger?.LogError("Source Type {0} is not recognized.", sourceType); } } return(sourcesLoaded, sourcesFailed); }
private void LoadCredentialProviders() { var credentialsSection = _config.GetSection("Credentials"); var credentialStarted = 0; var credentialFailed = 0; if (credentialsSection is null || !credentialsSection.GetChildren().Any()) { //this config file contains no credentials section return; } var credentialSections = credentialsSection.GetChildren(); foreach (var credentialSection in credentialSections) { var id = credentialSection[ConfigConstants.ID]; if (_credentialProviders.ContainsKey(id)) { credentialFailed++; continue; } var credentialType = credentialSection[ConfigConstants.CREDENTIAL_TYPE]; var factory = _credentialProviderFactoryCatalog.GetFactory(credentialType); if (factory != null) { try { var credentialProvider = factory.CreateInstance(credentialType, CreatePlugInContext(credentialSection)); credentialProvider.Id = id; _credentialProviders[id] = credentialProvider; credentialStarted++; _logger.LogDebug($"Created cred provider {credentialType} Id {id}"); } catch (Exception ex) { _logger?.LogError(ex, "Unable to load credential {0}", id); credentialFailed++; } } else { _logger?.LogError("Credential Type {0} is not recognized.", credentialType); credentialFailed++; } _metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>() { { MetricsConstants.SINKS_STARTED, new MetricValue(credentialStarted) }, { MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(credentialFailed) } }); } }
private void LoadEventSinks() { var sinksSection = _config.GetSection("Sinks"); var sinkSections = sinksSection.GetChildren(); int sinksStarted = 0; int sinksFailed = 0; foreach (var sinkSection in sinkSections) { string id = _config.GetChildConfig(sinkSection.Path, ConfigConstants.ID); string sinkType = _config.GetChildConfig(sinkSection.Path, ConfigConstants.SINK_TYPE); var factory = _sinkFactoryCatalog.GetFactory(sinkType); if (factory != null) { IEventSink sink = null; try { if (string.IsNullOrWhiteSpace(id)) { throw new Exception("Sink id is required."); } sink = factory.CreateInstance(sinkType, CreatePlugInContext(sinkSection)); sink.Start(); _sinks[sink.Id] = sink; sinksStarted++; } catch (Exception ex) { sinksFailed++; _logger.LogError(0, ex, $"Unable to load event sink {id} exception"); // If the sink fails to start due to "Rate limit exceeded" error, // add it to the sinks that need to be retarted if (ex is RateExceededException) { _sinksToRestart[sink.Id] = sink; } } } else { sinksFailed++; _logger.LogError("Sink Type {0} is not recognized.", sinkType); } } _metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>() { { MetricsConstants.SINKS_STARTED, new MetricValue(sinksStarted) }, { MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(sinksFailed) } }); }
private async Task LoadEventSinks(CancellationToken stopToken) { var sinksSection = _config.GetSection("Sinks"); var sinkSections = sinksSection.GetChildren(); var sinksStarted = 0; var sinksFailed = 0; foreach (var sinkSection in sinkSections) { var id = _config.GetChildConfig(sinkSection.Path, ConfigConstants.ID); var sinkType = _config.GetChildConfig(sinkSection.Path, ConfigConstants.SINK_TYPE); var factory = _sinkFactoryCatalog.GetFactory(sinkType); if (factory != null) { IEventSink sink = null; try { if (string.IsNullOrWhiteSpace(id)) { throw new Exception("Sink id is required."); } sink = factory.CreateInstance(sinkType, CreatePlugInContext(sinkSection)); await sink.StartAsync(stopToken); _sinks[sink.Id] = sink; sinksStarted++; } catch (Exception ex) { sinksFailed++; _logger.LogError(ex, $"Unable to load event sink {id}"); } } else { sinksFailed++; _logger.LogError("Sink Type {0} is not recognized.", sinkType); } } _metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>() { { MetricsConstants.SINKS_STARTED, new MetricValue(sinksStarted) }, { MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(sinksFailed) } }); }
private void LoadCredentialProviders() { var credentialsSection = _config.GetSection("Credentials"); var credentialSections = credentialsSection.GetChildren(); int credentialStarted = 0; int credentialFailed = 0; foreach (var credentialSection in credentialSections) { string id = _config.GetChildConfig(credentialSection.Path, ConfigConstants.ID); string credentialType = _config.GetChildConfig(credentialSection.Path, ConfigConstants.CREDENTIAL_TYPE); var factory = _credentialProviderFactoryCatalog.GetFactory(credentialType); if (factory != null) { try { ICredentialProvider credentialProvider = factory.CreateInstance(credentialType, CreatePlugInContext(credentialSection)); credentialProvider.Id = id; _credentialProviders[id] = credentialProvider; credentialStarted++; } catch (Exception ex) { credentialFailed++; _logger?.LogError($"Unable to load event sink {id} exception {ex.ToMinimized()}"); } } else { credentialFailed++; _logger?.LogError("Credential Type {0} is not recognized.", credentialType); } } _metrics.PublishCounters(string.Empty, MetricsConstants.CATEGORY_PROGRAM, CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>() { { MetricsConstants.SINKS_STARTED, new MetricValue(credentialStarted) }, { MetricsConstants.SINKS_FAILED_TO_START, new MetricValue(credentialFailed) } }); }
private (int sourceLoaded, int sourcesFailed) LoadEventSources() { //metrics source _sources[KINESISTAP_METRICS_SOURCE] = _metrics; var sourcesSection = _config.GetSection("Sources"); var sourceSections = sourcesSection.GetChildren(); var sourcesLoaded = 0; var sourcesFailed = 0; foreach (var sourceSection in sourceSections) { var id = _config.GetChildConfig(sourceSection.Path, ConfigConstants.ID); var sourceType = _config.GetChildConfig(sourceSection.Path, ConfigConstants.SOURCE_TYPE); var factory = _sourceFactoryCatalog.GetFactory(sourceType); if (factory != null) { try { var source = factory.CreateInstance(sourceType, CreatePlugInContext(sourceSection)); source.Id = id; _sources[id] = source; sourcesLoaded++; } catch (Exception ex) { sourcesFailed++; _logger.LogError(ex, "Unable to load event source {0}", id); } } else { _logger.LogError("Source Type {0} is not recognized.", sourceType); } } return(sourcesLoaded, sourcesFailed); }
/// <summary> /// Get the factory from catalog /// </summary> /// <param name="entry">The name of the factory to get</param> /// <returns>A factory</returns> public IFactory <T> GetFactory(string entry) { return(_factoryCatalog.GetFactory(entry)); }
/// <summary> /// Create an instance o the DirectorySource /// </summary> /// <param name="entry">Name of the source</param> /// <param name="context">Plug-in Context</param> /// <returns></returns> public virtual ISource CreateInstance(string entry, IPlugInContext context) { IConfiguration config = context.Configuration; ILogger logger = context.Logger; switch (entry.ToLower()) { case "directorysource": string recordParser = (config["RecordParser"] ?? string.Empty).ToLower(); string timetampFormat = config["TimestampFormat"]; string timestampField = config["TimestampField"]; DateTimeKind timeZoneKind = Utility.ParseTimeZoneKind(config["TimeZoneKind"]); string removeUnmatchedConfig = config["RemoveUnmatched"]; bool removeUnmatched = false; if (!string.IsNullOrWhiteSpace(removeUnmatchedConfig)) { removeUnmatched = bool.Parse(removeUnmatchedConfig); } string extractionPattern = config["ExtractionPattern"]; string extractionRegexOptions = config["ExtractionRegexOptions"]; switch (recordParser) { case "singleline": return(CreateEventSource(context, new SingeLineRecordParser())); case "regex": string pattern = config["Pattern"]; return(CreateEventSource(context, new RegexRecordParser(pattern, timetampFormat, logger, extractionPattern, extractionRegexOptions, timeZoneKind, new RegexRecordParserOptions { RemoveUnmatchedRecord = removeUnmatched }))); case "timestamp": return(CreateEventSource(context, new TimeStampRecordParser(timetampFormat, logger, extractionPattern, extractionRegexOptions, timeZoneKind, new RegexRecordParserOptions { RemoveUnmatchedRecord = removeUnmatched }))); case "syslog": return(CreateEventSource(context, new SysLogParser(logger, timeZoneKind, new RegexRecordParserOptions { RemoveUnmatchedRecord = removeUnmatched }))); case "delimited": return(CreateEventSourceWithDelimitedLogParser(context, timetampFormat, timeZoneKind)); case "singlelinejson": return(CreateEventSource(context, new SingleLineJsonParser(timestampField, timetampFormat))); default: IFactoryCatalog <IRecordParser> parserFactories = context?.ContextData?[PluginContext.PARSER_FACTORIES] as IFactoryCatalog <IRecordParser>; var parserFactory = parserFactories.GetFactory(recordParser); if (parserFactory == null) { throw new ArgumentException($"Unknown parser {recordParser}"); } else { return(CreateEventSource(context, parserFactory.CreateInstance(recordParser, context))); } } case "w3svclogsource": return(CreateEventSource( context, new W3SVCLogParser(context))); default: throw new ArgumentException($"Source {entry} not recognized."); } }