public void VerifyAllConfigOptionsMappedToConfigObject()
{
var pipelineConfigObj = new Dictionary <string, object>
{
["outputs"] = new List <Dictionary <string, object> >
{
new Dictionary <string, object>
{
["type"] = "ElasticSearch",
["indexNamePrefix"] = "myprefix",
["serviceUri"] = "http://localhost:1000;http://localhost:1001;http://localhost:1002",
["basicAuthenticationUserName"] = "******",
["basicAuthenticationUserPassword"] = "******",
["numberOfShards"] = 10,
["numberOfReplicas"] = 20,
["refreshInterval"] = "60s",
["useSniffingConnectionPooling"] = "true",
["mappings"] = new Dictionary <string, object>
{
["properties"] = new Dictionary <string, object>
{
["timestamp"] = new Dictionary <string, object>
{
["type"] = "date_nanos"
}
}
}
}
}
};
using (var configFile = new TemporaryFile())
{
var pipelineConfig = JsonConvert.SerializeObject(pipelineConfigObj, EventFlowJsonUtilities.GetDefaultSerializerSettings());
configFile.Write(pipelineConfig);
var configBuilder = new ConfigurationBuilder();
configBuilder.AddJsonFile(configFile.FilePath);
var configuration = configBuilder.Build();
var outputConfigSection = configuration.GetSection("outputs");
var configFragments = outputConfigSection.GetChildren().ToList();
var esOutputConfiguration = new ElasticSearchOutputConfiguration();
configFragments[0].Bind(esOutputConfiguration);
Assert.Equal("myprefix", esOutputConfiguration.IndexNamePrefix);
Assert.Equal("http://localhost:1000;http://localhost:1001;http://localhost:1002", esOutputConfiguration.ServiceUri);
Assert.Equal("myesuser", esOutputConfiguration.BasicAuthenticationUserName);
Assert.Equal("myespass", esOutputConfiguration.BasicAuthenticationUserPassword);
Assert.Equal(10, esOutputConfiguration.NumberOfShards);
Assert.Equal(20, esOutputConfiguration.NumberOfReplicas);
Assert.Equal("60s", esOutputConfiguration.RefreshInterval);
Assert.NotNull(esOutputConfiguration.Mappings);
Assert.NotNull(esOutputConfiguration.Mappings.Properties);
Assert.NotNull(esOutputConfiguration.Mappings.Properties["timestamp"]);
Assert.Equal("date_nanos", esOutputConfiguration.Mappings.Properties["timestamp"].Type);
}
}
public void VerifyUriListParsedFromString()
{
//Test single node
var singleUri = "http://localhost:8080";
var elasticConfigSingleNode = new ElasticSearchOutputConfiguration
{
ServiceUri = singleUri,
ConnectionPoolType = ""
};
var singleExpected = new List <Node> {
new Node(new Uri(singleUri))
};
//Test many nodes
var manyUri = "http://localhost:8080;http://localhost:8081";
var elasticConfigManyNode = new ElasticSearchOutputConfiguration
{
ServiceUri = manyUri,
ConnectionPoolType = "Static"
};
var manyExpected = new List <Node>
{
new Node(new Uri("http://localhost:8080")),
new Node(new Uri("http://localhost:8081"))
};
var healthReporterMock = new Mock <IHealthReporter>();
var singleResult = elasticConfigSingleNode.GetConnectionPool(healthReporterMock.Object);
var manyResult = elasticConfigManyNode.GetConnectionPool(healthReporterMock.Object);
Assert.Equal(singleExpected, singleResult.Nodes);
Assert.Equal(manyExpected, manyResult.Nodes);
healthReporterMock.Verify(m => m.ReportHealthy("ElasticSearchOutput: Using default Static connection type.", EventFlowContextIdentifiers.Configuration));
}
internal CreateIndexRequestBuilder(Configuration.ElasticSearchOutputConfiguration configuration, IHealthReporter healthReporter)
{
Requires.NotNull(configuration, nameof(configuration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.configuration = configuration;
this.healthReporter = healthReporter;
}
private void Initialize(ElasticSearchOutputConfiguration esOutputConfiguration)
{
Debug.Assert(esOutputConfiguration != null);
Debug.Assert(this.healthReporter != null);
this.connectionData = new ElasticSearchConnectionData
{
Configuration = esOutputConfiguration
};
string userName = esOutputConfiguration.BasicAuthenticationUserName;
string password = esOutputConfiguration.BasicAuthenticationUserPassword;
bool credentialsIncomplete = string.IsNullOrWhiteSpace(userName) ^ string.IsNullOrWhiteSpace(password);
if (credentialsIncomplete)
{
var errorMessage = $"{nameof(ElasticSearchOutput)}: for basic authentication to work both user name and password must be specified";
healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Configuration);
userName = password = null;
}
IConnectionPool pool = esOutputConfiguration.GetConnectionPool(healthReporter);
ConnectionSettings connectionSettings = new ConnectionSettings(pool);
if (!string.IsNullOrWhiteSpace(userName) && !string.IsNullOrWhiteSpace(password))
{
connectionSettings = connectionSettings.BasicAuthentication(userName, password);
}
this.connectionData.Client = new ElasticClient(connectionSettings);
this.connectionData.LastIndexName = null;
if (string.IsNullOrWhiteSpace(esOutputConfiguration.IndexNamePrefix))
{
esOutputConfiguration.IndexNamePrefix = string.Empty;
}
else
{
string lowerCaseIndexNamePrefix = esOutputConfiguration.IndexNamePrefix.ToLowerInvariant();
if (lowerCaseIndexNamePrefix != esOutputConfiguration.IndexNamePrefix)
{
healthReporter.ReportWarning($"{nameof(ElasticSearchOutput)}: The chosen index name prefix '{esOutputConfiguration.IndexNamePrefix}' "
+ "contains uppercase characters, which are not allowed by Elasticsearch. The prefix will be converted to lowercase.",
EventFlowContextIdentifiers.Configuration);
}
esOutputConfiguration.IndexNamePrefix = lowerCaseIndexNamePrefix + Dash;
}
if (string.IsNullOrWhiteSpace(esOutputConfiguration.EventDocumentTypeName))
{
string warning = $"{nameof(ElasticSearchOutput)}: '{nameof(ElasticSearchOutputConfiguration.EventDocumentTypeName)}' configuration parameter "
+ "should not be empty";
healthReporter.ReportWarning(warning, EventFlowContextIdentifiers.Configuration);
esOutputConfiguration.EventDocumentTypeName = ElasticSearchOutputConfiguration.DefaultEventDocumentTypeName;
}
}
public ElasticSearchOutput(ElasticSearchOutputConfiguration elasticSearchOutputConfiguration, IHealthReporter healthReporter)
{
Requires.NotNull(elasticSearchOutputConfiguration, nameof(elasticSearchOutputConfiguration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
// Clone the configuration instance since we are going to hold onto it (via this.connectionData)
Initialize(elasticSearchOutputConfiguration.DeepClone());
}
public void VerifyValidConfigCreatesConnectionPool(string connectionType, Type expectedConnectionPool)
{
var testUriString = "http://localhost:8080";
var elasticConfig = new ElasticSearchOutputConfiguration
{
ServiceUri = testUriString,
ConnectionPoolType = connectionType
};
var healthReporterMock = new Mock <IHealthReporter>();
var result = elasticConfig.GetConnectionPool(healthReporterMock.Object);
Assert.IsType(expectedConnectionPool, result);
}
public ElasticSearchOutputConfiguration DeepClone()
{
var other = new ElasticSearchOutputConfiguration()
{
IndexNamePrefix = this.IndexNamePrefix,
ServiceUri = this.ServiceUri,
BasicAuthenticationUserName = this.BasicAuthenticationUserName,
BasicAuthenticationUserPassword = this.BasicAuthenticationUserPassword,
EventDocumentTypeName = this.EventDocumentTypeName
};
return(other);
}
public void InvalidServiceUriConfigThrows()
{
var invalidUri = "httpNotValidUri";
var elasticConfigSingleNode = new ElasticSearchOutputConfiguration
{
ServiceUri = invalidUri,
ConnectionPoolType = ""
};
var healthReporterMock = new Mock <IHealthReporter>();
var expectedExceptionMessage = "ElasticSearchOutput: required 'serviceUri' configuration parameter is invalid";
var exception = Assert.Throws <Exception>(() => elasticConfigSingleNode.GetConnectionPool(healthReporterMock.Object));
Assert.Equal(expectedExceptionMessage, exception.Message);
healthReporterMock.Verify(m => m.ReportProblem(expectedExceptionMessage, EventFlowContextIdentifiers.Configuration), Times.Once);
}
public ElasticSearchOutputConfiguration DeepClone()
{
var other = new ElasticSearchOutputConfiguration()
{
IndexNamePrefix = this.IndexNamePrefix,
ServiceUri = this.ServiceUri,
BasicAuthenticationUserName = this.BasicAuthenticationUserName,
BasicAuthenticationUserPassword = this.BasicAuthenticationUserPassword,
EventDocumentTypeName = this.EventDocumentTypeName,
NumberOfShards = this.NumberOfShards,
NumberOfReplicas = this.NumberOfReplicas,
RefreshInterval = this.RefreshInterval,
};
return(other);
}
public void VerifyAllConfigOptionsMappedToConfigObject()
{
var pipelineConfigObj = new Dictionary <string, object>
{
["outputs"] = new List <Dictionary <string, object> >
{
new Dictionary <string, object>
{
["type"] = "ElasticSearch",
["indexNamePrefix"] = "myprefix",
["eventDocumentTypeName"] = "mytype",
["serviceUri"] = "http://localhost:1000;http://localhost:1001;http://localhost:1002",
["basicAuthenticationUserName"] = "******",
["basicAuthenticationUserPassword"] = "******",
["numberOfShards"] = 10,
["numberOfReplicas"] = 20,
["refreshInterval"] = "60s",
}
}
};
using (var configFile = new TemporaryFile())
{
var pipelineConfig = JsonConvert.SerializeObject(pipelineConfigObj);
configFile.Write(pipelineConfig);
var configBuilder = new ConfigurationBuilder();
configBuilder.AddJsonFile(configFile.FilePath);
var configuration = configBuilder.Build();
var outputConfigSection = configuration.GetSection("outputs");
var configFragments = outputConfigSection.GetChildren().ToList();
var esOutputConfiguration = new ElasticSearchOutputConfiguration();
configFragments[0].Bind(esOutputConfiguration);
Assert.Equal("myprefix", esOutputConfiguration.IndexNamePrefix);
Assert.Equal("mytype", esOutputConfiguration.EventDocumentTypeName);
Assert.Equal("http://localhost:1000;http://localhost:1001;http://localhost:1002", esOutputConfiguration.ServiceUri);
Assert.Equal("myesuser", esOutputConfiguration.BasicAuthenticationUserName);
Assert.Equal("myespass", esOutputConfiguration.BasicAuthenticationUserPassword);
Assert.Equal(10, esOutputConfiguration.NumberOfShards);
Assert.Equal(20, esOutputConfiguration.NumberOfReplicas);
Assert.Equal("60s", esOutputConfiguration.RefreshInterval);
}
}
public ElasticSearchOutput(IConfiguration configuration, IHealthReporter healthReporter)
{
Requires.NotNull(configuration, nameof(configuration));
Requires.NotNull(healthReporter, nameof(healthReporter));
this.healthReporter = healthReporter;
var esOutputConfiguration = new ElasticSearchOutputConfiguration();
try
{
configuration.Bind(esOutputConfiguration);
}
catch
{
healthReporter.ReportProblem($"Invalid {nameof(ElasticSearchOutput)} configuration encountered: '{configuration.ToString()}'",
EventFlowContextIdentifiers.Configuration);
throw;
}
Initialize(esOutputConfiguration);
}
private void Initialize(ElasticSearchOutputConfiguration esOutputConfiguration)
{
Debug.Assert(esOutputConfiguration != null);
Debug.Assert(this.healthReporter != null);
this.connectionData = new ElasticSearchConnectionData
{
Configuration = esOutputConfiguration
};
var esServiceUris = esOutputConfiguration.ServiceUri
.Split(';')
.Where(x => Uri.IsWellFormedUriString(x, UriKind.Absolute))
.Select(x => new Uri(x))
.ToList();
string errorMessage;
if (!esServiceUris.Any())
{
errorMessage =
$"{nameof(ElasticSearchOutput)}: required 'serviceUri' configuration parameter is invalid";
this.healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Configuration);
throw new Exception(errorMessage);
}
string userName = esOutputConfiguration.BasicAuthenticationUserName;
string password = esOutputConfiguration.BasicAuthenticationUserPassword;
bool credentialsIncomplete = string.IsNullOrWhiteSpace(userName) ^ string.IsNullOrWhiteSpace(password);
if (credentialsIncomplete)
{
errorMessage = $"{nameof(ElasticSearchOutput)}: for basic authentication to work both user name and password must be specified";
healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Configuration);
userName = password = null;
}
IConnectionPool pool = esOutputConfiguration.UseSniffingConnectionPooling? new SniffingConnectionPool(esServiceUris) : new StaticConnectionPool(esServiceUris);
ConnectionSettings connectionSettings = new ConnectionSettings(pool);
if (!string.IsNullOrWhiteSpace(userName) && !string.IsNullOrWhiteSpace(password))
{
connectionSettings = connectionSettings.BasicAuthentication(userName, password);
}
this.connectionData.Client = new ElasticClient(connectionSettings);
this.connectionData.LastIndexName = null;
if (string.IsNullOrWhiteSpace(esOutputConfiguration.IndexNamePrefix))
{
esOutputConfiguration.IndexNamePrefix = string.Empty;
}
else
{
string lowerCaseIndexNamePrefix = esOutputConfiguration.IndexNamePrefix.ToLowerInvariant();
if (lowerCaseIndexNamePrefix != esOutputConfiguration.IndexNamePrefix)
{
healthReporter.ReportWarning($"{nameof(ElasticSearchOutput)}: The chosen index name prefix '{esOutputConfiguration.IndexNamePrefix}' "
+ "contains uppercase characters, which are not allowed by Elasticsearch. The prefix will be converted to lowercase.",
EventFlowContextIdentifiers.Configuration);
}
esOutputConfiguration.IndexNamePrefix = lowerCaseIndexNamePrefix + Dash;
}
if (string.IsNullOrWhiteSpace(esOutputConfiguration.EventDocumentTypeName))
{
string warning = $"{nameof(ElasticSearchOutput)}: '{nameof(ElasticSearchOutputConfiguration.EventDocumentTypeName)}' configuration parameter "
+ "should not be empty";
healthReporter.ReportWarning(warning, EventFlowContextIdentifiers.Configuration);
esOutputConfiguration.EventDocumentTypeName = ElasticSearchOutputConfiguration.DefaultEventDocumentTypeName;
}
}
public void SniffingConnectionPoolingIsDisabledByDefault()
{
var esOutputConfiguration = new ElasticSearchOutputConfiguration();
Assert.False(esOutputConfiguration.UseSniffingConnectionPooling);
}
public ElasticSearchOutputStream(ElasticSearchOutputConfiguration configuration)
{
this.configuration = configuration;
pool = new Semaphore(configuration.ConnectionLimit, configuration.ConnectionLimit);
logger = LogManager.GetCurrentClassLogger();
}
