/// <summary> /// Performs configuration /// </summary> /// <param name="configuration">Previous configuration</param> /// <param name="config">Akka configuration</param> /// <returns>Updated configuration</returns> public LoggerConfiguration Configure(LoggerConfiguration configuration, Config config) { var minimumLevel = config.GetString("ClusterKit.Log.ElasticSearch.minimumLevel", "none")?.Trim(); LogEventLevel level; if (!Enum.TryParse(minimumLevel, true, out level)) { return configuration; } var nodes = config.GetStringList("ClusterKit.Log.ElasticSearch.nodes"); var indexFormat = config.GetString("ClusterKit.Log.ElasticSearch.indexFormat", "logstash-{0:yyyy.MM.dd}"); Log.Information( "{Type}: \n\tMinimum level: {MinimumLevel}\n\tIndex format: {IndexFormat}\n\tNodes:\n\t\t{NodeList}\n", this.GetType().FullName, minimumLevel, indexFormat, string.Join("\n\t\t", nodes)); SelfLog.Enable(Console.WriteLine); var options = new ElasticsearchSinkOptions(nodes.Select(s => new Uri(s))) { MinimumLogEventLevel = level, AutoRegisterTemplate = true, IndexFormat = indexFormat }; return configuration.WriteTo.Elasticsearch(options); }
private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) { throw new ArgumentException("options.IndexFormat"); } if (string.IsNullOrWhiteSpace(options.TypeName)) { throw new ArgumentException("options.TypeName"); } if (string.IsNullOrWhiteSpace(options.TemplateName)) { throw new ArgumentException("options.TemplateName"); } _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); _options = options; Func <ConnectionConfiguration, IElasticsearchSerializer> serializerFactory = null; if (options.Serializer != null) { serializerFactory = s => options.Serializer; } var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, serializerFactory) .RequestTimeout(options.ConnectionTimeout); if (options.ModifyConnectionSettings != null) { configuration = options.ModifyConnectionSettings(configuration); } configuration.ThrowExceptions(); _client = new ElasticLowLevelClient(configuration); _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, renderMessage: true, closingDelimiter: string.Empty, serializer: options.Serializer, inlineFields: options.InlineFields ); _durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, renderMessage: true, closingDelimiter: Environment.NewLine, serializer: options.Serializer, inlineFields: options.InlineFields ); _registerTemplateOnStartup = options.AutoRegisterTemplate; TemplateRegistrationSuccess = !_registerTemplateOnStartup; }
/// <summary> /// Overload to allow basic configuration through AppSettings. /// </summary> /// <param name="loggerSinkConfiguration">Options for the sink.</param> /// <param name="nodeUris">A comma or semi column separated list of URIs for Elasticsearch nodes.</param> /// <param name="indexFormat"><see cref="ElasticsearchSinkOptions.IndexFormat"/></param> /// <param name="templateName"><see cref="ElasticsearchSinkOptions.TemplateName"/></param> /// <returns>LoggerConfiguration object</returns> /// <exception cref="ArgumentNullException"><paramref name="nodeUris"/> is <see langword="null" />.</exception> public static LoggerConfiguration Elasticsearch( this LoggerSinkConfiguration loggerSinkConfiguration, string nodeUris, string indexFormat = null, string templateName = null) { if (string.IsNullOrEmpty(nodeUris)) throw new ArgumentNullException("nodeUris", "No Elasticsearch node(s) specified."); IEnumerable<Uri> nodes = nodeUris .Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries) .Select(uriString => new Uri(uriString)); var options = new ElasticsearchSinkOptions(nodes); if (!string.IsNullOrWhiteSpace(indexFormat)) { options.IndexFormat = indexFormat; } if (!string.IsNullOrWhiteSpace(templateName)) { options.AutoRegisterTemplate = true; options.TemplateName = templateName; } return Elasticsearch(loggerSinkConfiguration, options); }
/// <summary> /// Creates a new ElasticsearchSink instance with the provided options /// </summary> /// <param name="options">Options configuring how the sink behaves, may NOT be null</param> public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period, options.QueueSizeLimit) { _state = ElasticsearchSinkState.Create(options); _state.DiscoverClusterVersion(); _state.RegisterTemplateIfNeeded(); }
public static ITextFormatter CreateDefaultFormatter(ElasticsearchSinkOptions options) { return(new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, closingDelimiter: string.Empty, serializer: options.Serializer, inlineFields: options.InlineFields )); }
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) { if (options == null) { throw new ArgumentNullException("options"); } return new ElasticsearchSinkState(options); }
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) { if (options == null) { throw new ArgumentNullException(nameof(options)); } return(new ElasticsearchSinkState(options)); }
public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOptions options) { return(new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, closingDelimiter: Environment.NewLine, serializer: options.Serializer, inlineFields: options.InlineFields )); }
public OnErrorInterceptor() { var uri = ConfigurationManager.AppSettings["ElasticNode"]; if(!string.IsNullOrEmpty(uri)) { var options = new ElasticsearchSinkOptions(new Uri(uri)); Log.Logger = new LoggerConfiguration().WriteTo.Elasticsearch(options).CreateLogger(); } }
public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOptions options) { return(new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, closingDelimiter: Environment.NewLine, serializer: options.Serializer != null ? new SerializerAdapter(options.Serializer) : null, inlineFields: options.InlineFields, formatStackTraceAsArray: options.FormatStackTraceAsArray )); }
private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) { throw new ArgumentException("options.IndexFormat"); } if (string.IsNullOrWhiteSpace(options.TemplateName)) { throw new ArgumentException("options.TemplateName"); } // Strip type argument if ESv7 since multiple types are not supported anymore if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7) { options.TypeName = "_doc"; } else { if (string.IsNullOrWhiteSpace(options.TypeName)) { throw new ArgumentException("options.TypeName"); } } _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant()); _bufferedIndexDecider = options.BufferIndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant()); _options = options; var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer) .RequestTimeout(options.ConnectionTimeout); if (options.ModifyConnectionSettings != null) { configuration = options.ModifyConnectionSettings(configuration); } configuration.ThrowExceptions(); _client = new ElasticLowLevelClient(configuration); _formatter = options.CustomFormatter ?? CreateDefaultFormatter(options); _durableFormatter = options.CustomDurableFormatter ?? CreateDefaultDurableFormatter(options); _registerTemplateOnStartup = options.AutoRegisterTemplate; TemplateRegistrationSuccess = !_registerTemplateOnStartup; }
private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) { throw new ArgumentException("options.IndexFormat"); } if (string.IsNullOrWhiteSpace(options.TypeName)) { throw new ArgumentException("options.TypeName"); } if (string.IsNullOrWhiteSpace(options.TemplateName)) { throw new ArgumentException("options.TemplateName"); } this._templateName = options.TemplateName; this._templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); _typeName = options.TypeName; _options = options; var configuration = new ConnectionConfiguration(options.ConnectionPool) .SetTimeout(options.ConnectionTimeout) .SetMaximumAsyncConnections(20); if (options.ModifyConnectionSetttings != null) { configuration = options.ModifyConnectionSetttings(configuration); } _client = new ElasticsearchClient(configuration, connection: options.Connection, serializer: options.Serializer); _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, renderMessage: true, closingDelimiter: string.Empty, serializer: options.Serializer, inlineFields: options.InlineFields ); _durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, renderMessage: true, closingDelimiter: Environment.NewLine, serializer: options.Serializer, inlineFields: options.InlineFields ); this._registerTemplateOnStartup = options.AutoRegisterTemplate; }
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) { if (options == null) { throw new ArgumentNullException("options"); } var state = new ElasticsearchSinkState(options); if (state.Options.AutoRegisterTemplate) { state.RegisterTemplateIfNeeded(); } return(state); }
private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat"); if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName"); this._templateName = options.TemplateName; this._templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); _typeName = options.TypeName; _options = options; Func<ConnectionConfiguration, IElasticsearchSerializer> serializerFactory = null; if (options.Serializer != null) { serializerFactory = (s) => options.Serializer; } var configuration = new ConnectionConfiguration( options.ConnectionPool, options.Connection, serializerFactory) .RequestTimeout(TimeSpan.FromMilliseconds(options.ConnectionTimeout)); if (options.ModifyConnectionSettings != null) configuration = options.ModifyConnectionSettings(configuration); _client = new ElasticsearchClient(configuration); _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, renderMessage: true, closingDelimiter: string.Empty, serializer: options.Serializer, inlineFields: options.InlineFields ); _durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, renderMessage: true, closingDelimiter: Environment.NewLine, serializer: options.Serializer, inlineFields: options.InlineFields ); _registerTemplateOnStartup = options.AutoRegisterTemplate; }
/// <summary> /// Adds a sink that writes log events as documents to an Elasticsearch index. /// This works great with the Kibana web interface when using the default settings. /// /// By passing in the BufferBaseFilename, you make this into a durable sink. /// Meaning it will log to disk first and tries to deliver to the Elasticsearch server in the background. /// </summary> /// <remarks> /// Make sure to have a sensible mapping in your Elasticsearch indexes. /// You can automatically create one by specifying this in the options. /// </remarks> /// <param name="loggerSinkConfiguration">Options for the sink.</param> /// <param name="options">Provides options specific to the Elasticsearch sink</param> /// <returns>LoggerConfiguration object</returns> public static LoggerConfiguration Elasticsearch( this LoggerSinkConfiguration loggerSinkConfiguration, ElasticsearchSinkOptions options = null) { //TODO make sure we do not kill appdata injection //TODO handle bulk errors and write to self log, what does logstash do in this case? //TODO NEST trace logging ID's to corrolate requests to eachother options = options ?? new ElasticsearchSinkOptions(new[] { new Uri(DefaultNodeUri) }); var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename) ? (ILogEventSink)new ElasticsearchSink(options) : new DurableElasticsearchSink(options); return loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum); }
public DurableElasticsearchSink(ElasticsearchSinkOptions options) { _state = ElasticsearchSinkState.Create(options); if (string.IsNullOrWhiteSpace(options.BufferBaseFilename)) { throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!"); } _sink = new RollingFileSink( options.BufferBaseFilename + FileNameSuffix, _state.DurableFormatter, options.BufferFileSizeLimitBytes, null); _shipper = new ElasticsearchLogShipper(_state); }
public DurableElasticsearchSink(ElasticsearchSinkOptions options) { _state = ElasticsearchSinkState.Create(options); if (string.IsNullOrWhiteSpace(options.BufferBaseFilename)) { throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!"); } _sink = new RollingFileSink( options.BufferBaseFilename + FileNameSuffix, _state.DurableFormatter, options.BufferFileSizeLimitBytes, null); _shipper = new ElasticsearchLogShipper(_state); }
private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) { throw new ArgumentException("options.IndexFormat"); } if (string.IsNullOrWhiteSpace(options.TypeName)) { throw new ArgumentException("options.TypeName"); } if (string.IsNullOrWhiteSpace(options.TemplateName)) { throw new ArgumentException("options.TemplateName"); } _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); _options = options; var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer) .RequestTimeout(options.ConnectionTimeout); if (options.ModifyConnectionSettings != null) { configuration = options.ModifyConnectionSettings(configuration); } configuration.ThrowExceptions(); _client = new ElasticLowLevelClient(configuration); _formatter = options.CustomFormatter ?? CreateDefaultFormatter(options); _durableFormatter = options.CustomDurableFormatter ?? CreateDefaultDurableFormatter(options); _registerTemplateOnStartup = options.AutoRegisterTemplate; TemplateRegistrationSuccess = !_registerTemplateOnStartup; }
public static object GetTemplate(ElasticsearchSinkOptions options, string discoveredVersion, Dictionary <string, string> settings, string templateMatchString, AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2) { switch (version) { case AutoRegisterTemplateVersion.ESv5: return(GetTemplateESv5(settings, templateMatchString)); case AutoRegisterTemplateVersion.ESv2: return(GetTemplateESv2(settings, templateMatchString)); case AutoRegisterTemplateVersion.ESv6: return(GetTemplateESv6(options, discoveredVersion, settings, templateMatchString)); case AutoRegisterTemplateVersion.ESv7: return(GetTemplateESv7(options, discoveredVersion, settings, templateMatchString)); default: throw new ArgumentOutOfRangeException(nameof(version), version, null); } }
/// <summary> /// Creates a new ElasticsearchSink instance with the provided options /// </summary> /// <param name="options">Options configuring how the sink behaves, may NOT be null</param> public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period) { _state = ElasticsearchSinkState.Create(options); _state.RegisterTemplateIfNeeded(); }
private static object GetTemplateESv7(ElasticsearchSinkOptions options, string discoveredVersion, Dictionary <string, string> settings, string templateMatchString) { object mappings = new { dynamic_templates = new List <Object> { //when you use serilog as an adaptor for third party frameworks //where you have no control over the log message they typically //contain {0} ad infinitum, we force numeric property names to //contain strings by default. { new { numerics_in_fields = new { path_match = @"fields\.[\d+]$", match_pattern = "regex", mapping = new { type = "text", index = true, norms = false } } } }, { new { string_fields = new { match = "*", match_mapping_type = "string", mapping = new { type = "text", index = true, norms = false, fields = new { raw = new { type = "keyword", index = true, ignore_above = 256 } } } } } } }, properties = new Dictionary <string, object> { { "message", new { type = "text", index = true } }, { "exceptions", new { type = "nested", properties = new Dictionary <string, object> { { "Depth", new { type = "integer" } }, { "RemoteStackIndex", new { type = "integer" } }, { "HResult", new { type = "integer" } }, { "StackTraceString", new { type = "text", index = true } }, { "RemoteStackTraceString", new { type = "text", index = true } }, { "ExceptionMessage", new { type = "object", properties = new Dictionary <string, object> { { "MemberType", new { type = "integer" } }, } } } } } } } }; mappings = discoveredVersion?.StartsWith("6.") ?? false ? new { _doc = mappings } : mappings; return(new { index_patterns = new[] { templateMatchString }, settings = settings, mappings = mappings }); }
/// <summary> /// Creates a new ElasticsearchSink instance with the provided options /// </summary> /// <param name="options">Options configuring how the sink behaves, may NOT be null</param> public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period) { _state = ElasticsearchSinkState.Create(options); }
/// <summary> /// Creates a new ElasticsearchSink instance with the provided options /// </summary> /// <param name="options">Options configuring how the sink behaves, may NOT be null</param> public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period) { _state = ElasticsearchSinkState.Create(options); }
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) { if (options == null) throw new ArgumentNullException("options"); var state = new ElasticsearchSinkState(options); if (state.Options.AutoRegisterTemplate) state.RegisterTemplateIfNeeded(); return state; }
/// <summary> /// Creates a new ElasticsearchSink instance with the provided options /// </summary> /// <param name="options">Options configuring how the sink behaves, may NOT be null</param> public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period) { _state = ElasticsearchSinkState.Create(options); _state.RegisterTemplateIfNeeded(); }