public RabbitMqConfigurator SetLogging(string elasticEndpoint, string indexFormat, string elasticPassword, string logLevel)
{
var sinkOptions = new ElasticsearchSinkOptions(new Uri(elasticEndpoint))
{
AutoRegisterTemplate = true,
IndexFormat = indexFormat,
AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6
};
if (!string.IsNullOrEmpty(elasticPassword))
{
sinkOptions.ModifyConnectionSettings = (c) => c.BasicAuthentication("elastic", elasticPassword);
}
var loggerConfig = new LoggerConfiguration();
if (SetLogLevel.ContainsKey(logLevel))
{
SetLogLevel[logLevel](loggerConfig);
}
else
{
loggerConfig.MinimumLevel.Error();
}
loggerConfig.WriteTo.Elasticsearch(sinkOptions);
Logger = loggerConfig.CreateLogger();
return(this);
}
public static IServiceCollection AddLogger(this IServiceCollection services,
IConfiguration configuration)
{
var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
var podName = Environment.GetEnvironmentVariable("POD_NAME") ?? "Local";
var o = new LoggingOptions();
configuration.Bind(nameof(LoggingOptions), o);
var elasticsearchSinkOptions = new ElasticsearchSinkOptions(new Uri(o.ElasticsearchUrl))
{
AutoRegisterTemplate = true,
IndexFormat = $"logstash-{DateTime.Now:yyyy.MM.dd}",
ModifyConnectionSettings = x => x.BasicAuthentication("elastic", "changeme")
};
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(configuration)
.Enrich.WithProperty("environment", environment)
.Enrich.WithProperty("system", o.SystemName)
.Enrich.WithProperty("pod", podName)
.Enrich.WithExceptionDetails()
.Enrich.FromLogContext()
.WriteTo.Elasticsearch(elasticsearchSinkOptions)
.WriteTo.RollingFile($"C:/logs/{o.SystemName}/{o.SystemName}-{environment}-{{Date}}.txt")
.WriteTo.Console()
.CreateLogger();
Log.Logger.Information("Successfully configured logger");
return(services);
}
protected ElasticsearchSinkTestsBase()
{
Serilog.Debugging.SelfLog.Out = Console.Out;
_serializer = new ElasticsearchJsonNetSerializer();
_connection = A.Fake <IConnection>();
_options = new ElasticsearchSinkOptions(new Uri("http://*****:*****@"{ ""ok"": true }"));
_seenHttpPuts.Add(Tuple.Create(uri, Encoding.UTF8.GetString(postData)));
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "PUT", "/", postData, fixedRespone));
});
A.CallTo(() => _connection.PostSync(A <Uri> ._, A <byte[]> ._, A <IRequestConfiguration> ._))
.ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) =>
{
var fixedRespone = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""ok"": true }"));
_seenHttpPosts.Add(Encoding.UTF8.GetString(postData));
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "POST", "/", postData, fixedRespone));
});
A.CallTo(() => _connection.HeadSync(A <Uri> ._, A <IRequestConfiguration> ._))
.ReturnsLazily((Uri uri, IRequestConfiguration requestConfiguration) =>
{
_seenHttpHeads.Add(_templateExistsReturnCode);
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), _templateExistsReturnCode, "HEAD", "/", null));
});
}
public static ConreignApi Create(ConreignApiConfiguration apiConfiguration)
{
if (apiConfiguration == null)
{
throw new ArgumentNullException(nameof(apiConfiguration));
}
var loggerConfiguration = new LoggerConfiguration();
if (!string.IsNullOrEmpty(apiConfiguration.ElasticSearchUri))
{
var elasticOptions = new ElasticsearchSinkOptions(new Uri(apiConfiguration.ElasticSearchUri))
{
AutoRegisterTemplate = true,
BufferBaseFilename = "logs/elastic-buffer"
};
loggerConfiguration.WriteTo.Elasticsearch(elasticOptions);
}
var logger = loggerConfiguration
.WriteTo.LiterateConsole()
.WriteTo.RollingFile(Path.Combine(ApplicationPath.CurrentDirectory, "logs", "conreign-api-{Date}.log"))
.MinimumLevel.Is(apiConfiguration.MinimumLogLevel)
.Enrich.FromLogContext()
.CreateLogger()
.ForContext(new []
{
new PropertyEnricher("ApplicationId", "Conreign.Api"),
new PropertyEnricher("ClusterId", apiConfiguration.ClusterId),
new PropertyEnricher("InstanceId", apiConfiguration.InstanceId)
});
return(new ConreignApi(apiConfiguration, logger));
}
public static ElasticsearchSinkOptions CreateElasticsearchSinkOptions(ElasticSearchLogConfiguration elasticSearchLogConfiguration, ElasticSearchLogSecretConfiguration elasticSearchLogSecretConfiguration)
{
if (elasticSearchLogConfiguration == null || elasticSearchLogSecretConfiguration == null)
{
return(null);
}
var singleNodeConnectionPool = new SingleNodeConnectionPool(elasticSearchLogConfiguration.Uri);
var awsHttpConnection = new AwsHttpConnection(elasticSearchLogConfiguration.Region, new StaticCredentialsProvider(
new AwsCredentials
{
AccessKey = elasticSearchLogSecretConfiguration.AccessKey,
SecretKey = elasticSearchLogSecretConfiguration.SecretKey
}));
var options = new ElasticsearchSinkOptions(elasticSearchLogConfiguration.Uri)
{
IndexFormat = elasticSearchLogConfiguration.IndexFormat,
InlineFields = elasticSearchLogConfiguration.InlineFields,
MinimumLogEventLevel = elasticSearchLogConfiguration.MinimumLogLevel,
ModifyConnectionSettings = conn => new ConnectionConfiguration(singleNodeConnectionPool, awsHttpConnection)
};
return(options);
}
public static ElasticsearchSinkOptions Create(string indexPrefix, string templateName, Action <ElasticsearchSinkOptions> alterOptions = null)
{
// make sure we run through `ipv4.fiddler` if `fiddler` or `mitmproxy` is running
// NOTE with the latter you need to add `ipv4.fiddler` as an alias to 127.0.0.1 in your HOSTS file manually
var pool = new SingleNodeConnectionPool(new Uri($"http://{ProxyDetection.LocalOrProxyHost}:9200"));
var options = new ElasticsearchSinkOptions(pool)
{
IndexFormat = indexPrefix + "{0:yyyy.MM.dd}",
TemplateName = templateName,
};
alterOptions?.Invoke(options);
// here we make sure we set a proxy on the elasticsearch connection
// when we detect `mitmproxy` running. This is a cli tool similar to fiddler
// on *nix systems which aids debugging what goes over the wire
var provided = options.ModifyConnectionSettings;
options.ModifyConnectionSettings = configuration =>
{
if (ProxyDetection.RunningMitmProxy)
{
configuration.Proxy(ProxyDetection.MitmProxyAddress, null, (string)null);
}
configuration = provided?.Invoke(configuration) ?? configuration;
return(configuration);
};
return(options);
}
/// <summary>
/// Use this instead of mocks to test against your instance.
/// </summary>
/// <returns></returns>
private ILogEventSink GetRealSink()
{
var options = new ElasticsearchSinkOptions(new Uri("http://your.elasticsearch.instance"));
options.TypeName = "idsrvevent";
return(new ElasticsearchSink(options));
}
/// <summary>
/// Overload to allow basic configuration through AppSettings.
/// </summary>
/// <param name="loggerSinkConfiguration">Options for the sink.</param>
/// <param name="nodeUris">A comma or semi column separated list of URIs for Elasticsearch nodes.</param>
/// <param name="indexFormat"><see cref="ElasticsearchSinkOptions.IndexFormat"/></param>
/// <param name="templateName"><see cref="ElasticsearchSinkOptions.TemplateName"/></param>
/// <returns>LoggerConfiguration object</returns>
/// <exception cref="ArgumentNullException"><paramref name="nodeUris"/> is <see langword="null" />.</exception>
public static LoggerConfiguration Elasticsearch(
this LoggerSinkConfiguration loggerSinkConfiguration,
string nodeUris,
string indexFormat = null,
string templateName = null)
{
if (string.IsNullOrEmpty(nodeUris))
{
throw new ArgumentNullException("nodeUris", "No Elasticsearch node(s) specified.");
}
IEnumerable <Uri> nodes = nodeUris
.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
.Select(uriString => new Uri(uriString));
var options = new ElasticsearchSinkOptions(nodes);
if (!string.IsNullOrWhiteSpace(indexFormat))
{
options.IndexFormat = indexFormat;
}
if (!string.IsNullOrWhiteSpace(templateName))
{
options.AutoRegisterTemplate = true;
options.TemplateName = templateName;
}
return(Elasticsearch(loggerSinkConfiguration, options));
}
private static void ConfigureElasticSearchLogging(
LoggerConfiguration loggerConfiguration,
IHostingEnvironment hostingEnvironment,
IConfiguration configuration)
{
var shouldLogToElasticSearch = configuration.GetValue <bool?>("Logging:Output:ElasticSearch:Enabled") ?? !hostingEnvironment.IsDevelopment();
if (shouldLogToElasticSearch)
{
var nodeUrls = configuration.GetValue <string[]>("Logging:Output:ElasticSearch:Nodes") ?? new [] { "http://localhost:9200" };
var logDirectory = configuration.GetValue <string>("Logging:Output:File:Directory") ?? $"..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}logs";
var fileSizeLimit = configuration.GetValue <int?>("Logging:Output:File:FileSizeLimitBytes") ?? 100000000;
var options = new ElasticsearchSinkOptions(nodeUrls.Select(x => new Uri(x, UriKind.Absolute)))
{
EmitEventFailure = EmitEventFailureHandling.WriteToFailureSink,
#pragma warning disable 618
FailureSink = new FileSink(
#pragma warning restore 618
$"{logDirectory}.failure.JOS.Multiple.HostedServices..txt",
new MessageTemplateTextFormatter(OutputFormat, CultureInfo.InvariantCulture),
fileSizeLimitBytes: fileSizeLimit),
QueueSizeLimit = 1000000
};
loggerConfiguration.WriteTo.Elasticsearch(options);
}
}
public static ElasticsearchSinkOptions ConfigurationElasticSink(IConfigurationRoot configuration, string environment)
{
var elasticSink = new ElasticsearchSinkOptions(new Uri(configuration["ElasticConfiguration:Uri"]));
elasticSink.AutoRegisterTemplate = true;
elasticSink.IndexFormat = $"{Assembly.GetExecutingAssembly().GetName().Name.ToLower().Replace(".", "-")}-{environment?.ToLower().Replace(".", "-")}-{DateTime.Now:yyyy-MM-dd}";
return(elasticSink);
}
public static LoggerConfiguration AppendElasticsearchLogger(this LoggerConfiguration logger, string uri, string indexName)
{
var options = new ElasticsearchSinkOptions(new Uri(uri))
{
IndexFormat = $"{indexName}-{0:yyyy.MM}",
};
return(logger
.WriteTo.Elasticsearch(options));
}
/// <summary>
/// Adds a sink that writes log events as documents to an Elasticsearch index.
/// This works great with the Kibana web interface when using the default settings.
///
/// By passing in the BufferBaseFilename, you make this into a durable sink.
/// Meaning it will log to disk first and tries to deliver to the Elasticsearch server in the background.
/// </summary>
/// <remarks>
/// Make sure to have a sensible mapping in your Elasticsearch indexes.
/// You can automatically create one by specifying this in the options.
/// </remarks>
/// <param name="loggerSinkConfiguration">Options for the sink.</param>
/// <param name="options">Provides options specific to the Elasticsearch sink</param>
/// <returns>LoggerConfiguration object</returns>
public static LoggerConfiguration Elasticsearch(
this LoggerSinkConfiguration loggerSinkConfiguration,
ElasticsearchSinkOptions options = null)
{
options = options ?? new ElasticsearchSinkOptions(new[] { new Uri(DefaultNodeUri) });
var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename)
? (ILogEventSink) new ElasticsearchSink(options)
: new DurableElasticsearchSink(options);
return(loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum));
}
public static LoggerConfiguration ApplyEdamosConfiguration(this LoggerConfiguration conf, IConfiguration configuration)
{
//TODO: use real ELK Uri
ElasticsearchSinkOptions sinkOptions =
new ElasticsearchSinkOptions(new Uri(DebugConstants.ElasticSearch.LoggingUri));
sinkOptions.CustomFormatter = new ElasticsearchJsonFormatter(renderMessage: false);
sinkOptions.CustomDurableFormatter = new ElasticsearchJsonFormatter(renderMessage: false);
configuration.Bind(nameof(ElasticsearchSinkOptions), sinkOptions);
conf.WriteTo.Elasticsearch(sinkOptions);
conf.Enrich.FromLogContext();
conf.Enrich.WithProperty("app", AppDomain.CurrentDomain.FriendlyName);
conf.Enrich.WithProperty("machine", Environment.MachineName);
conf.Enrich.WithProperty("startup", DateTime.UtcNow.Ticks);
conf.Enrich.With <EventNameEnricher>();
conf.MinimumLevel.Override("Microsoft.EntityFrameworkCore", LogEventLevel.Warning);
conf.MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning);
conf.MinimumLevel.Override("Microsoft.AspNetCore.DataProtection", LogEventLevel.Information);
#if DEBUG
conf.WriteTo.Console(outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3} {RequestId,-30}] {Source}:{EId}:{EName} {NewLine}{Exception}");
conf.MinimumLevel.Override("Microsoft.EntityFrameworkCore", LogEventLevel.Information);
conf.MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Information);
#endif
//conf.WriteTo.Logger(lc =>
//{
// lc.NewFilterBuilder()
// .AddSource("Microsoft.EntityFrameworkCore") // don't log "Microsoft.EntityFrameworkCore" source by default
// .RemoveLevelsFromSource("Microsoft.EntityFrameworkCore", // keep important events from "Microsoft.EntityFrameworkCore"
// LogEventLevel.Warning,
// LogEventLevel.Error,
// LogEventLevel.Fatal)
// .AddSource("Microsoft.AspNetCore") // don't log "Microsoft.AspNetCore" source by default
// .RemoveLevelsFromSource("Microsoft.AspNetCore", // keep important events from "Microsoft.AspNetCore"
// LogEventLevel.Warning,
// LogEventLevel.Error,
// LogEventLevel.Fatal)
// .RemoveLevelsFromSource("Microsoft.AspNetCore.DataProtection", // keep important events from "Microsoft.AspNetCore.DataProtection"
// LogEventLevel.Information,
// LogEventLevel.Warning,
// LogEventLevel.Error,
// LogEventLevel.Fatal)
// .BuildExclude();
//});
return(conf);
}
public void WhenCreatingOptions_NumberOfShardsInjected_NumberOfShardsAreSet()
{
var options = new ElasticsearchSinkOptions(new Uri("http://localhost:9100"))
{
AutoRegisterTemplate = true,
NumberOfShards = 2,
NumberOfReplicas = 0
};
options.NumberOfReplicas.Should().Be(0);
options.NumberOfShards.Should().Be(2);
}
private ElasticsearchSinkOptions ConfigureElasticSink()
{
var uri = new Uri(_configuration["ElasticConfiguration:Uri"]);
var options = new ElasticsearchSinkOptions(uri);
options.AutoRegisterTemplate = true;
options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7;
options.IndexFormat = GetLogIndentifier();
options.EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
EmitEventFailureHandling.WriteToFailureSink |
EmitEventFailureHandling.RaiseCallback;
return(options);
}
private ILogger ConfigureLogger(LoggingLevelSwitch levelSwitch, IAppConfiguration appConfig)
{
var sinkOptions = new ElasticsearchSinkOptions(CreateElasticSearchUri(appConfig.ElasticSearchSettings))
{
IndexFormat = "logstash-fabricdatabus-{0:yyyy.MM.dd}"
};
return(new LoggerConfiguration()
.MinimumLevel.ControlledBy(levelSwitch)
.Enrich.FromLogContext()
.WriteTo
.Elasticsearch(sinkOptions).CreateLogger());
}
public ILogger Build()
{
var elasticSearchOptions = new ElasticsearchSinkOptions(new Uri(_configuration["elasticSearchEndpoint"]));
return(new LoggerConfiguration()
.MinimumLevel.Information()
.Enrich.FromLogContext()
.Enrich.WithProperty("Facility", "HotelsSample")
.Enrich.WithProperty("InstanceId", Amazon.Util.EC2InstanceMetadata.InstanceId)
.WriteTo.Elasticsearch(elasticSearchOptions)
.WriteTo.Console()
.WriteTo.Udp(IPAddress.Loopback, 7071, outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level}] {Message}{NewLine}{Exception}{NewLine}{Properties}")
.CreateLogger());
}
public Startup(IConfiguration _configuration)
{
_Configuration = _configuration;
var loggerConfig = new LoggerConfiguration().ReadFrom.Configuration(_configuration);
if (!string.IsNullOrWhiteSpace(_configuration["ElasticSearchUri"]))
{
var esOptions = new ElasticsearchSinkOptions(new Uri(_configuration["ElasticSearchUri"]));
esOptions.AutoRegisterTemplate = true;
esOptions.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6;
loggerConfig = loggerConfig.WriteTo.Elasticsearch(esOptions);
}
Log.Logger = loggerConfig.CreateLogger();
}
private static ElasticsearchSinkOptions ConfigureElasticSink(IConfigurationRoot configuration, string environment)
{
var indexFormat = $"{Assembly.GetExecutingAssembly().GetName().Name.ToLower().Replace(".", "-")}-{environment?.ToLower().Replace(".", "-")}-{DateTime.UtcNow:yyyy-MM}";
var options = new ElasticsearchSinkOptions(new Uri(configuration["ElasticConfiguration:Uri"]))
{
AutoRegisterTemplate = true,
IndexFormat = indexFormat,
ModifyConnectionSettings = x => x.BasicAuthentication(configuration["ElasticConfiguration:Username"], configuration["ElasticConfiguration:Password"]),
FailureCallback = e => Console.WriteLine("ELASTICSEARCH FAIL - Unable to submit event " + e.MessageTemplate),
EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog | EmitEventFailureHandling.RaiseCallback
};
return(options);
}
/// <summary>
/// Configure the elastic search logs
/// </summary>
/// <param name="config"></param>
private void ConfigureElasticSearchSink(LoggerConfiguration config)
{
const string appName = "lazygit";
var indexFormat = $"{appName}-{{0:yyyy-MM}}";
var sinkOptions = new ElasticsearchSinkOptions(new Uri(_nodeUri))
{
MinimumLogEventLevel = LogEventLevel.Debug,
IndexFormat = indexFormat
};
config.WriteTo.Elasticsearch(sinkOptions);
}
/// <summary>
/// Добавить настройку для записи в Elasticsearch
/// </summary>
/// <param name="elasticEndpoint"> Адрес Elasticsearch</param>
/// <param name="minLevel"> минимальный уровень </param>
/// <param name="indexBaseName">База имени индекса.
/// Полное имя формируется как: [базовое имя]-[текущая дата в формате yyyy.MM.dd].
/// Дефолтное имя: "logstash-{0:yyyy.MM.dd}"
/// </param>
public SerilogerConfigurator AddElasticSearchTarget(Uri elasticEndpoint, LogEventLevel minLevel, string indexBaseName = null)
{
var option = new ElasticsearchSinkOptions(elasticEndpoint);
option.MinimumLogEventLevel = minLevel;
if (!string.IsNullOrWhiteSpace(indexBaseName))
{
option.IndexFormat = indexBaseName + "-{0:yyyy.MM}";
}
_elasticsearchOptions.Add(option);
return(this);
}
static Logger()
{
var options = new ElasticsearchSinkOptions(new Uri("http://localhost:9200"))
{
AutoRegisterTemplate = true,
};
PerfLogger = new LoggerConfiguration().WriteTo.Elasticsearch(options).CreateLogger();
ErrorLogger = new LoggerConfiguration().WriteTo.Elasticsearch(options).CreateLogger();
MatchLogger = new LoggerConfiguration().WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9200"))
{
AutoRegisterTemplate = true,
IndexFormat = "matchstatsh-{0:yyyy.MM.dd}"
}).CreateLogger();
}
internal static ElasticsearchSinkOptions GetElasticOptions(IEncompassConfig config)
{
var elasticOptions = new ElasticsearchSinkOptions(new Uri(config.GetValue(Logger.ELASTICSEARCH_URL, "NOT PROVIDED")));
var indexFormat = config.GetValue(Logger.ELASTICSEARCH_INDEX_NAME, "sextant-serilog-");
if (!indexFormat.Contains("{"))
{
indexFormat += "{0:yyyy.MM.dd}";
}
elasticOptions.IndexFormat = indexFormat;
elasticOptions.MinimumLogEventLevel = GetMinLevel(config, "ELASTICSEARCH");
elasticOptions.EmitEventFailure = EmitEventFailureHandling.ThrowException;
return(elasticOptions);
}
public static void ConfigureSerilog(this IWebHostBuilder webHostBuilder)
{
webHostBuilder.UseSerilog((hostingContext, loggerConfiguration) =>
{
var elasticConfiguration = hostingContext.Configuration.GetSection("ElasticConfiguration").Get <ElasticConfiguration>();
var elasticSearchSinkOptions = new ElasticsearchSinkOptions(new Uri(elasticConfiguration.Host))
{
ModifyConnectionSettings = x => x.BasicAuthentication(elasticConfiguration.Username, elasticConfiguration.Password),
AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
};
loggerConfiguration
.WriteTo.Elasticsearch(elasticSearchSinkOptions)
.WriteTo.Console();
});
}
public SettingsFromConfigFile()
{
Environment = ConfigurationManager.AppSettings["Environment"];
SuiteName = ConfigurationManager.AppSettings["SuiteName"];
ComponentName = ConfigurationManager.AppSettings["ComponentName"];
LogFileDirectory = ConfigurationManager.AppSettings["LogFileDirectory"];
EsLoggingUrl = ConfigurationManager.AppSettings["EsLoggingUrl"];
ReleaseNumber = ConfigurationManager.AppSettings["ReleaseNumber"];
var loggingLevel = ConfigurationManager.AppSettings["LoggingLevel"];
if (string.IsNullOrEmpty(loggingLevel))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(loggingLevel));
}
LoggingLevel = (LogEventLevel)Enum.Parse(typeof(LogEventLevel), loggingLevel);
if (string.IsNullOrEmpty(Environment))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(Environment));
}
if (string.IsNullOrEmpty(SuiteName))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(SuiteName));
}
if (string.IsNullOrEmpty(ComponentName))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(ComponentName));
}
if (string.IsNullOrEmpty(LogFileDirectory))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(LogFileDirectory));
}
if (string.IsNullOrEmpty(EsLoggingUrl))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(EsLoggingUrl));
}
if (string.IsNullOrEmpty(ReleaseNumber))
{
throw new ArgumentException("AppSetting can't be null/empty", nameof(ReleaseNumber));
}
FileName = $"{LogFileDirectory}{SuiteName}.{ComponentName}.log";
EsLoggingUri = new Uri(EsLoggingUrl);
ElasticsearchSinkOptions = new ElasticsearchSinkOptions(EsLoggingUri);
EasyNetQConfig = ConfigurationManager.AppSettings["EasyNetQConfig"];
}
/// <summary>
/// Adds a sink that writes log events as documents to an Elasticsearch index.
/// This works great with the Kibana web interface when using the default settings.
///
/// By passing in the BufferBaseFilename, you make this into a durable sink.
/// Meaning it will log to disk first and tries to deliver to the Elasticsearch server in the background.
/// </summary>
/// <remarks>
/// Make sure to have a sensible mapping in your Elasticsearch indexes.
/// You can automatically create one by specifying this in the options.
/// </remarks>
/// <param name="loggerSinkConfiguration">Options for the sink.</param>
/// <param name="options">Provides options specific to the Elasticsearch sink</param>
/// <returns>LoggerConfiguration object</returns>
public static LoggerConfiguration Elasticsearch(
this LoggerSinkConfiguration loggerSinkConfiguration,
ElasticsearchSinkOptions options = null)
{
//TODO make sure we do not kill appdata injection
//TODO handle bulk errors and write to self log, what does logstash do in this case?
//TODO NEST trace logging ID's to corrolate requests to eachother
options = options ?? new ElasticsearchSinkOptions(new[] { new Uri(DefaultNodeUri) });
var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename)
? (ILogEventSink) new ElasticsearchSink(options)
: new DurableElasticsearchSink(options);
return(loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum));
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.UseSerilog((context, configuration) =>
{
var elasticSearchSink = new ElasticsearchSinkOptions(new Uri(context.Configuration.GetValue <string>("ElasticsearchUri")));
elasticSearchSink.IndexFormat = $"LocationImageApi-{DateTime.UtcNow:yyyy-MM}";
configuration.Enrich.FromLogContext()
.WriteTo.Console()
.WriteTo.Elasticsearch(elasticSearchSink)
.ReadFrom.Configuration(context.Configuration);
})
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseStartup <Startup>();
});
public DurableElasticsearchSink(ElasticsearchSinkOptions options)
{
_state = ElasticsearchSinkState.Create(options);
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!");
}
_sink = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.File(_state.DurableFormatter,
options.BufferBaseFilename + FileNameSuffix,
rollingInterval: RollingInterval.Day,
fileSizeLimitBytes: options.BufferFileSizeLimitBytes,
rollOnFileSizeLimit: true,
retainedFileCountLimit: options.BufferFileCountLimit,
levelSwitch: _state.Options.LevelSwitch,
encoding: Encoding.UTF8)
.CreateLogger();
var elasticSearchLogClient = new ElasticsearchLogClient(
elasticLowLevelClient: _state.Client,
cleanPayload: _state.Options.BufferCleanPayload,
elasticOpType: _state.Options.BatchAction);
var payloadReader = new ElasticsearchPayloadReader(
pipelineName: _state.Options.PipelineName,
typeName: _state.Options.TypeName,
serialize: _state.Serialize,
getIndexForEvent: _state.GetBufferedIndexForEvent,
elasticOpType: _state.Options.BatchAction);
_shipper = new ElasticsearchLogShipper(
bufferBaseFilename: _state.Options.BufferBaseFilename,
batchPostingLimit: _state.Options.BatchPostingLimit,
period: _state.Options.BufferLogShippingInterval ?? TimeSpan.FromSeconds(5),
eventBodyLimitBytes: _state.Options.SingleEventSizePostingLimit,
levelControlSwitch: _state.Options.LevelSwitch,
logClient: elasticSearchLogClient,
payloadReader: payloadReader,
retainedInvalidPayloadsLimitBytes: _state.Options.BufferRetainedInvalidPayloadsLimitBytes,
bufferSizeLimitBytes: _state.Options.BufferFileSizeLimitBytes,
registerTemplateIfNeeded: _state.RegisterTemplateIfNeeded);
}
/// <summary>
/// Sets up the asp.net core application to log to a Elasticsearch instance.
/// Refernce: https://github.com/serilog/serilog-sinks-elasticsearch/wiki/basic-setup
/// Note: The following configuration variables are required:
/// ELASTICSEARCH_HOST => The url of the Elasticsearch endpoint (ie: http(s)://{Elasticsearch Host}/9200)
/// Note: The following configuration values should be set for Elasticsearch basic authentication
/// ELASTICSEARCH_USERNAME
/// ELASTICSEARCH_PASSWORD
/// Note: The following configuration values should be set for Elasticsearch api key authentication
/// ELASTICSEARCH_USERNAME => This should be the "id" of your token
/// ELASTICSEARCH_APIKEY => This should be the "api_key" of your token
/// Note: IF your Elasticsearch instance is using an invalid SSL cert
/// ELASTICSEARCH_DISABLE_SSL_VALIDATION => set this value to "true"
/// </summary>
/// <param name="startup"></param>
/// <param name="app"></param>
public static ILogger SetupElasticsearch(this CoreWebStartup startup, IApplicationBuilder app)
{
_ = (app?.UseSerilog());
return(SeriLogExtensions.GetLogger(() =>
{
var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
var host = startup.Configuration.GetValue <string>("ELASTICSEARCH_HOST", "http://localhost:9200");
var username = startup.Configuration.GetValue <string>("ELASTICSEARCH_USERNAME");
var password = startup.Configuration.GetValue <string>("ELASTICSEARCH_PASSWORD");
var apiKey = startup.Configuration.GetValue <string>("ELASTICSEARCH_APIKEY");
var elastiOptions = new ElasticsearchSinkOptions(new Uri(host))
{
IndexFormat = $"{startup.StartupAssembly.GetName().Name.ToLower().Replace(".", "-")}-{environment?.ToLower().Replace(".", "-")}-{DateTime.UtcNow:yy-MM}",
AutoRegisterTemplate = true,
AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
};
var modifyConfigSettings = new Func <Func <ConnectionConfiguration>, ConnectionConfiguration>((authFunc) =>
{
var config = authFunc();
if (startup.Configuration.GetValue <bool>("ELASTICSEARCH_DISABLE_SSL_VALIDATION"))
{
_ = config.ServerCertificateValidationCallback((obj, cert, chain, policyErrors) => true);
}
return config;
});
if (!string.IsNullOrWhiteSpace(password))
{
elastiOptions.ModifyConnectionSettings = config => modifyConfigSettings(() => config.BasicAuthentication(username, password));
}
else if (!string.IsNullOrWhiteSpace(apiKey))
{
elastiOptions.ModifyConnectionSettings = config => modifyConfigSettings(() => config.ApiKeyAuthentication(username, apiKey));
}
return new LoggerConfiguration()
{
}
.MinimumLevel.Information()
.Enrich.FromLogContext()
.Enrich.WithMachineName()
.WriteTo.Console()
.WriteTo.Elasticsearch(elastiOptions)
.Enrich.WithProperty("Environment", environment)
.CreateLogger();
}));
}
private static ElasticsearchSinkOptions GetElasticsearchSinkOptions()
{
var elasticsearchUri = new Uri("http://192.168.0.16:9229");
var indexFormat = $"{Assembly.GetExecutingAssembly().GetName().Name.ToLower().Replace(".", "-")}-{DateTime.UtcNow:yyyy-MM}";
var options = new ElasticsearchSinkOptions(elasticsearchUri)
{
FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog | EmitEventFailureHandling.RaiseCallback,
AutoRegisterTemplate = true,
AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6,
IndexFormat = indexFormat,
MinimumLogEventLevel = Serilog.Events.LogEventLevel.Verbose,
CustomFormatter = new ExceptionAsObjectJsonFormatter()
};
return(options);
}
/// <summary>
/// Use this instead of mocks to test against your instance.
/// </summary>
/// <returns></returns>
private ILogEventSink GetRealSink()
{
var options = new ElasticsearchSinkOptions(new Uri("http://your.elasticsearch.instance"));
options.TypeName = "idsrvevent";
return new ElasticsearchSink(options);
}
