/// <summary>
/// Performs configuration
/// </summary>
/// <param name="configuration">Previous configuration</param>
/// <param name="config">Akka configuration</param>
/// <returns>Updated configuration</returns>
public LoggerConfiguration Configure(LoggerConfiguration configuration, Config config)
{
var minimumLevel = config.GetString("ClusterKit.Log.ElasticSearch.minimumLevel", "none")?.Trim();
LogEventLevel level;
if (!Enum.TryParse(minimumLevel, true, out level))
{
return configuration;
}
var nodes = config.GetStringList("ClusterKit.Log.ElasticSearch.nodes");
var indexFormat = config.GetString("ClusterKit.Log.ElasticSearch.indexFormat", "logstash-{0:yyyy.MM.dd}");
Log.Information(
"{Type}: \n\tMinimum level: {MinimumLevel}\n\tIndex format: {IndexFormat}\n\tNodes:\n\t\t{NodeList}\n",
this.GetType().FullName,
minimumLevel,
indexFormat,
string.Join("\n\t\t", nodes));
SelfLog.Enable(Console.WriteLine);
var options = new ElasticsearchSinkOptions(nodes.Select(s => new Uri(s)))
{
MinimumLogEventLevel = level,
AutoRegisterTemplate = true,
IndexFormat = indexFormat
};
return configuration.WriteTo.Elasticsearch(options);
}
private ElasticsearchSinkState(ElasticsearchSinkOptions options)
{
if (string.IsNullOrWhiteSpace(options.IndexFormat))
{
throw new ArgumentException("options.IndexFormat");
}
if (string.IsNullOrWhiteSpace(options.TypeName))
{
throw new ArgumentException("options.TypeName");
}
if (string.IsNullOrWhiteSpace(options.TemplateName))
{
throw new ArgumentException("options.TemplateName");
}
_templateName = options.TemplateName;
_templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
_indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset));
_options = options;
Func <ConnectionConfiguration, IElasticsearchSerializer> serializerFactory = null;
if (options.Serializer != null)
{
serializerFactory = s => options.Serializer;
}
var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, serializerFactory)
.RequestTimeout(options.ConnectionTimeout);
if (options.ModifyConnectionSettings != null)
{
configuration = options.ModifyConnectionSettings(configuration);
}
configuration.ThrowExceptions();
_client = new ElasticLowLevelClient(configuration);
_formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
renderMessage: true,
closingDelimiter: string.Empty,
serializer: options.Serializer,
inlineFields: options.InlineFields
);
_durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
renderMessage: true,
closingDelimiter: Environment.NewLine,
serializer: options.Serializer,
inlineFields: options.InlineFields
);
_registerTemplateOnStartup = options.AutoRegisterTemplate;
TemplateRegistrationSuccess = !_registerTemplateOnStartup;
}
/// <summary>
/// Overload to allow basic configuration through AppSettings.
/// </summary>
/// <param name="loggerSinkConfiguration">Options for the sink.</param>
/// <param name="nodeUris">A comma or semi column separated list of URIs for Elasticsearch nodes.</param>
/// <param name="indexFormat"><see cref="ElasticsearchSinkOptions.IndexFormat"/></param>
/// <param name="templateName"><see cref="ElasticsearchSinkOptions.TemplateName"/></param>
/// <returns>LoggerConfiguration object</returns>
/// <exception cref="ArgumentNullException"><paramref name="nodeUris"/> is <see langword="null" />.</exception>
public static LoggerConfiguration Elasticsearch(
this LoggerSinkConfiguration loggerSinkConfiguration,
string nodeUris,
string indexFormat = null,
string templateName = null)
{
if (string.IsNullOrEmpty(nodeUris))
throw new ArgumentNullException("nodeUris", "No Elasticsearch node(s) specified.");
IEnumerable<Uri> nodes = nodeUris
.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
.Select(uriString => new Uri(uriString));
var options = new ElasticsearchSinkOptions(nodes);
if (!string.IsNullOrWhiteSpace(indexFormat))
{
options.IndexFormat = indexFormat;
}
if (!string.IsNullOrWhiteSpace(templateName))
{
options.AutoRegisterTemplate = true;
options.TemplateName = templateName;
}
return Elasticsearch(loggerSinkConfiguration, options);
}
/// <summary>
/// Creates a new ElasticsearchSink instance with the provided options
/// </summary>
/// <param name="options">Options configuring how the sink behaves, may NOT be null</param>
public ElasticsearchSink(ElasticsearchSinkOptions options)
: base(options.BatchPostingLimit, options.Period, options.QueueSizeLimit)
{
_state = ElasticsearchSinkState.Create(options);
_state.DiscoverClusterVersion();
_state.RegisterTemplateIfNeeded();
}
public static ITextFormatter CreateDefaultFormatter(ElasticsearchSinkOptions options)
{
return(new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
closingDelimiter: string.Empty,
serializer: options.Serializer,
inlineFields: options.InlineFields
));
}
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
return new ElasticsearchSinkState(options);
}
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
return(new ElasticsearchSinkState(options));
}
public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOptions options)
{
return(new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
closingDelimiter: Environment.NewLine,
serializer: options.Serializer,
inlineFields: options.InlineFields
));
}
public OnErrorInterceptor()
{
var uri = ConfigurationManager.AppSettings["ElasticNode"];
if(!string.IsNullOrEmpty(uri))
{
var options = new ElasticsearchSinkOptions(new Uri(uri));
Log.Logger = new LoggerConfiguration().WriteTo.Elasticsearch(options).CreateLogger();
}
}
public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOptions options)
{
return(new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
closingDelimiter: Environment.NewLine,
serializer: options.Serializer != null ? new SerializerAdapter(options.Serializer) : null,
inlineFields: options.InlineFields,
formatStackTraceAsArray: options.FormatStackTraceAsArray
));
}
private ElasticsearchSinkState(ElasticsearchSinkOptions options)
{
if (string.IsNullOrWhiteSpace(options.IndexFormat))
{
throw new ArgumentException("options.IndexFormat");
}
if (string.IsNullOrWhiteSpace(options.TemplateName))
{
throw new ArgumentException("options.TemplateName");
}
// Strip type argument if ESv7 since multiple types are not supported anymore
if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7)
{
options.TypeName = "_doc";
}
else
{
if (string.IsNullOrWhiteSpace(options.TypeName))
{
throw new ArgumentException("options.TypeName");
}
}
_templateName = options.TemplateName;
_templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
_indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant());
_bufferedIndexDecider = options.BufferIndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant());
_options = options;
var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer)
.RequestTimeout(options.ConnectionTimeout);
if (options.ModifyConnectionSettings != null)
{
configuration = options.ModifyConnectionSettings(configuration);
}
configuration.ThrowExceptions();
_client = new ElasticLowLevelClient(configuration);
_formatter = options.CustomFormatter ?? CreateDefaultFormatter(options);
_durableFormatter = options.CustomDurableFormatter ?? CreateDefaultDurableFormatter(options);
_registerTemplateOnStartup = options.AutoRegisterTemplate;
TemplateRegistrationSuccess = !_registerTemplateOnStartup;
}
private ElasticsearchSinkState(ElasticsearchSinkOptions options)
{
if (string.IsNullOrWhiteSpace(options.IndexFormat))
{
throw new ArgumentException("options.IndexFormat");
}
if (string.IsNullOrWhiteSpace(options.TypeName))
{
throw new ArgumentException("options.TypeName");
}
if (string.IsNullOrWhiteSpace(options.TemplateName))
{
throw new ArgumentException("options.TemplateName");
}
this._templateName = options.TemplateName;
this._templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
_indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset));
_typeName = options.TypeName;
_options = options;
var configuration = new ConnectionConfiguration(options.ConnectionPool)
.SetTimeout(options.ConnectionTimeout)
.SetMaximumAsyncConnections(20);
if (options.ModifyConnectionSetttings != null)
{
configuration = options.ModifyConnectionSetttings(configuration);
}
_client = new ElasticsearchClient(configuration, connection: options.Connection, serializer: options.Serializer);
_formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
renderMessage: true,
closingDelimiter: string.Empty,
serializer: options.Serializer,
inlineFields: options.InlineFields
);
_durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
renderMessage: true,
closingDelimiter: Environment.NewLine,
serializer: options.Serializer,
inlineFields: options.InlineFields
);
this._registerTemplateOnStartup = options.AutoRegisterTemplate;
}
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
var state = new ElasticsearchSinkState(options);
if (state.Options.AutoRegisterTemplate)
{
state.RegisterTemplateIfNeeded();
}
return(state);
}
private ElasticsearchSinkState(ElasticsearchSinkOptions options)
{
if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat");
if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName");
if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName");
this._templateName = options.TemplateName;
this._templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
_indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset));
_typeName = options.TypeName;
_options = options;
Func<ConnectionConfiguration, IElasticsearchSerializer> serializerFactory = null;
if (options.Serializer != null)
{
serializerFactory = (s) => options.Serializer;
}
var configuration = new ConnectionConfiguration(
options.ConnectionPool,
options.Connection,
serializerFactory)
.RequestTimeout(TimeSpan.FromMilliseconds(options.ConnectionTimeout));
if (options.ModifyConnectionSettings != null)
configuration = options.ModifyConnectionSettings(configuration);
_client = new ElasticsearchClient(configuration);
_formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
renderMessage: true,
closingDelimiter: string.Empty,
serializer: options.Serializer,
inlineFields: options.InlineFields
);
_durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter(
formatProvider: options.FormatProvider,
renderMessage: true,
closingDelimiter: Environment.NewLine,
serializer: options.Serializer,
inlineFields: options.InlineFields
);
_registerTemplateOnStartup = options.AutoRegisterTemplate;
}
/// <summary>
/// Adds a sink that writes log events as documents to an Elasticsearch index.
/// This works great with the Kibana web interface when using the default settings.
///
/// By passing in the BufferBaseFilename, you make this into a durable sink.
/// Meaning it will log to disk first and tries to deliver to the Elasticsearch server in the background.
/// </summary>
/// <remarks>
/// Make sure to have a sensible mapping in your Elasticsearch indexes.
/// You can automatically create one by specifying this in the options.
/// </remarks>
/// <param name="loggerSinkConfiguration">Options for the sink.</param>
/// <param name="options">Provides options specific to the Elasticsearch sink</param>
/// <returns>LoggerConfiguration object</returns>
public static LoggerConfiguration Elasticsearch(
this LoggerSinkConfiguration loggerSinkConfiguration,
ElasticsearchSinkOptions options = null)
{
//TODO make sure we do not kill appdata injection
//TODO handle bulk errors and write to self log, what does logstash do in this case?
//TODO NEST trace logging ID's to corrolate requests to eachother
options = options ?? new ElasticsearchSinkOptions(new[] { new Uri(DefaultNodeUri) });
var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename)
? (ILogEventSink)new ElasticsearchSink(options)
: new DurableElasticsearchSink(options);
return loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum);
}
public DurableElasticsearchSink(ElasticsearchSinkOptions options)
{
_state = ElasticsearchSinkState.Create(options);
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!");
}
_sink = new RollingFileSink(
options.BufferBaseFilename + FileNameSuffix,
_state.DurableFormatter,
options.BufferFileSizeLimitBytes,
null);
_shipper = new ElasticsearchLogShipper(_state);
}
public DurableElasticsearchSink(ElasticsearchSinkOptions options)
{
_state = ElasticsearchSinkState.Create(options);
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!");
}
_sink = new RollingFileSink(
options.BufferBaseFilename + FileNameSuffix,
_state.DurableFormatter,
options.BufferFileSizeLimitBytes,
null);
_shipper = new ElasticsearchLogShipper(_state);
}
private ElasticsearchSinkState(ElasticsearchSinkOptions options)
{
if (string.IsNullOrWhiteSpace(options.IndexFormat))
{
throw new ArgumentException("options.IndexFormat");
}
if (string.IsNullOrWhiteSpace(options.TypeName))
{
throw new ArgumentException("options.TypeName");
}
if (string.IsNullOrWhiteSpace(options.TemplateName))
{
throw new ArgumentException("options.TemplateName");
}
_templateName = options.TemplateName;
_templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
_indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset));
_options = options;
var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer)
.RequestTimeout(options.ConnectionTimeout);
if (options.ModifyConnectionSettings != null)
{
configuration = options.ModifyConnectionSettings(configuration);
}
configuration.ThrowExceptions();
_client = new ElasticLowLevelClient(configuration);
_formatter = options.CustomFormatter ?? CreateDefaultFormatter(options);
_durableFormatter = options.CustomDurableFormatter ?? CreateDefaultDurableFormatter(options);
_registerTemplateOnStartup = options.AutoRegisterTemplate;
TemplateRegistrationSuccess = !_registerTemplateOnStartup;
}
public static object GetTemplate(ElasticsearchSinkOptions options,
string discoveredVersion,
Dictionary <string, string> settings,
string templateMatchString,
AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2)
{
switch (version)
{
case AutoRegisterTemplateVersion.ESv5:
return(GetTemplateESv5(settings, templateMatchString));
case AutoRegisterTemplateVersion.ESv2:
return(GetTemplateESv2(settings, templateMatchString));
case AutoRegisterTemplateVersion.ESv6:
return(GetTemplateESv6(options, discoveredVersion, settings, templateMatchString));
case AutoRegisterTemplateVersion.ESv7:
return(GetTemplateESv7(options, discoveredVersion, settings, templateMatchString));
default:
throw new ArgumentOutOfRangeException(nameof(version), version, null);
}
}
/// <summary>
/// Creates a new ElasticsearchSink instance with the provided options
/// </summary>
/// <param name="options">Options configuring how the sink behaves, may NOT be null</param>
public ElasticsearchSink(ElasticsearchSinkOptions options)
: base(options.BatchPostingLimit, options.Period)
{
_state = ElasticsearchSinkState.Create(options);
_state.RegisterTemplateIfNeeded();
}
private static object GetTemplateESv7(ElasticsearchSinkOptions options, string discoveredVersion,
Dictionary <string, string> settings,
string templateMatchString)
{
object mappings = new
{
dynamic_templates = new List <Object>
{
//when you use serilog as an adaptor for third party frameworks
//where you have no control over the log message they typically
//contain {0} ad infinitum, we force numeric property names to
//contain strings by default.
{
new
{
numerics_in_fields = new
{
path_match = @"fields\.[\d+]$",
match_pattern = "regex",
mapping = new
{
type = "text",
index = true,
norms = false
}
}
}
},
{
new
{
string_fields = new
{
match = "*",
match_mapping_type = "string",
mapping = new
{
type = "text",
index = true,
norms = false,
fields = new
{
raw = new
{
type = "keyword",
index = true,
ignore_above = 256
}
}
}
}
}
}
},
properties = new Dictionary <string, object>
{
{ "message", new { type = "text", index = true } },
{
"exceptions", new
{
type = "nested",
properties = new Dictionary <string, object>
{
{ "Depth", new { type = "integer" } },
{ "RemoteStackIndex", new { type = "integer" } },
{ "HResult", new { type = "integer" } },
{ "StackTraceString", new { type = "text", index = true } },
{ "RemoteStackTraceString", new { type = "text", index = true } },
{
"ExceptionMessage", new
{
type = "object",
properties = new Dictionary <string, object>
{
{ "MemberType", new { type = "integer" } },
}
}
}
}
}
}
}
};
mappings = discoveredVersion?.StartsWith("6.") ?? false ? new { _doc = mappings } : mappings;
return(new
{
index_patterns = new[] { templateMatchString },
settings = settings,
mappings = mappings
});
}
/// <summary>
/// Creates a new ElasticsearchSink instance with the provided options
/// </summary>
/// <param name="options">Options configuring how the sink behaves, may NOT be null</param>
public ElasticsearchSink(ElasticsearchSinkOptions options)
: base(options.BatchPostingLimit, options.Period)
{
_state = ElasticsearchSinkState.Create(options);
}
/// <summary>
/// Creates a new ElasticsearchSink instance with the provided options
/// </summary>
/// <param name="options">Options configuring how the sink behaves, may NOT be null</param>
public ElasticsearchSink(ElasticsearchSinkOptions options)
: base(options.BatchPostingLimit, options.Period)
{
_state = ElasticsearchSinkState.Create(options);
}
public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options)
{
if (options == null) throw new ArgumentNullException("options");
var state = new ElasticsearchSinkState(options);
if (state.Options.AutoRegisterTemplate)
state.RegisterTemplateIfNeeded();
return state;
}
/// <summary>
/// Creates a new ElasticsearchSink instance with the provided options
/// </summary>
/// <param name="options">Options configuring how the sink behaves, may NOT be null</param>
public ElasticsearchSink(ElasticsearchSinkOptions options)
: base(options.BatchPostingLimit, options.Period)
{
_state = ElasticsearchSinkState.Create(options);
_state.RegisterTemplateIfNeeded();
}