public static int Main()
{
ElasticsearchVersion v = null;
// var clusterConfiguration = new EphemeralClusterConfiguration("6.0.0", numberOfNodes: 2);
// var ephemeralCluster = new EphemeralCluster(clusterConfiguration);
// var nodeConfiguration = new NodeConfiguration(clusterConfiguration, 9200);
// var elasticsearchNode = new ElasticsearchNode(nodeConfiguration);
////
// using (var node = new ElasticsearchNode("5.5.1"))
// {
// node.Subscribe(new ConsoleOutColorWriter());
// node.WaitForStarted(TimeSpan.FromMinutes(2));
// }
//
// using (var node = new ElasticsearchNode("6.0.0-beta2", @"c:\Data\elasticsearch-6.0.0-beta2"))
// {
// node.Subscribe();
// node.WaitForStarted(TimeSpan.FromMinutes(2));
// Console.ReadKey();
// }
// using (var cluster = new EphemeralCluster("6.0.0"))
// {
// cluster.Start();
// }
var config = new EphemeralClusterConfiguration("6.2.3", XPack | Security | SSL, null, 1)
{
PrintYamlFilesInConfigFolder = true,
ShowElasticsearchOutputAfterStarted = true
};
using (var cluster = new EphemeralCluster(config))
{
cluster.Start();
var nodes = cluster.NodesUris();
var connectionPool = new StaticConnectionPool(nodes);
var settings = new ConnectionSettings(connectionPool).EnableDebugMode();
if (config.EnableSecurity && !config.EnableSsl)
{
settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password);
}
if (config.EnableSsl)
{
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
settings = settings.ClientCertificate(new X509Certificate2(config.FileSystem.ClientCertificate));
}
var client = new ElasticClient(settings);
Console.Write(client.XPackInfo().DebugInformation);
}
//
// Console.WriteLine($".. DONE ...");
return(0);
}
private static void GenerateCertificate(EphemeralClusterConfiguration config, string name, string path, string zipLocation, string silentModeConfigFile, IConsoleLineHandler writer)
{
var @out = config.Version.Major < 6 ? $"{name}.zip" : zipLocation;
var fs = config.FileSystem;
var binary = config.Version >= "6.3.0"
? config.Version < "8.0.0"
? Path.Combine(fs.ElasticsearchHome, "bin", "elasticsearch-certgen") + BinarySuffix
: Path.Combine(fs.ElasticsearchHome, "bin", "elasticsearch-certutil") + BinarySuffix
: Path.Combine(fs.ElasticsearchHome, "bin", "x-pack", "certgen") + BinarySuffix;
if (!Directory.Exists(path))
{
if (config.Version < "7.0.0")
{
ExecuteBinary(config, writer, binary, "generating ssl certificates for this session",
"-in", silentModeConfigFile, "-out", @out);
}
else
{
ExecuteBinary(config, writer, binary, "generating ssl certificates for this session",
"cert",
"-in", silentModeConfigFile, "-out", @out);
}
}
var badLocation = Path.Combine(config.FileSystem.ElasticsearchHome, "config", "x-pack", @out);
//not necessary anymore now that we patch .in.bat i think
if (config.Version.Major < 6 && File.Exists(badLocation))
{
writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} moving {badLocation} to {@out}");
File.Move(badLocation, zipLocation);
}
}
static void Main(string[] args)
{
var clusterConfiguration = new EphemeralClusterConfiguration("7.9.0");
using var cluster = new EphemeralCluster(clusterConfiguration);
cluster.Start(TimeSpan.FromMinutes(2));
var uri = cluster.NodesUris().First();
var nestIndex = "logs-from-nest";
Announce("NEST + Elastic.CommonSchema", nestIndex);
var settings = new ConnectionSettings(uri).EnableDebugMode();
var client = new ElasticClient(settings);
var bulkAll = client.BulkAll(GetEcsEvents(10_000), b => b
.Index(nestIndex)
.Size(1_000)
.RefreshOnCompleted()
.BufferToBulk((bulk, items) => bulk.CreateMany(items))
);
bulkAll.Wait(TimeSpan.FromMinutes(2), r => Console.WriteLine("Indexed 1000 events"));
Check(client, nestIndex);
// Using Serilog ElasticsearchSink
var sinkIndex = "logs-from-sink";
Announce("Elastic.CommonSchema.Serilog", sinkIndex);
var options = new ElasticsearchSinkOptions(uri)
{
CustomFormatter = new EcsTextFormatter(),
IndexFormat = sinkIndex + "-{0:yyyy-MM-dd}",
DetectElasticsearchVersion = true,
BatchAction = ElasticOpType.Create,
BatchPostingLimit = 1_000,
};
var logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.Elasticsearch(options)
.CreateLogger();
Log.Logger = logger;
foreach (var e in GetEcsEvents(10_000))
{
Log.Debug("Reusing {field1} by {another_field}: {message}", "some value", true, e.Message);
}
Log.CloseAndFlush();
Check(client, $"{sinkIndex}-*");
}
private static void ManualConfigRun()
{
ElasticVersion version = "latest";
var plugins =
new ElasticsearchPlugins(ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.IngestAttachment);
var features = Security | XPack | SSL;
var config = new EphemeralClusterConfiguration(version, features, plugins, numberOfNodes: 1)
{
HttpFiddlerAware = true,
ShowElasticsearchOutputAfterStarted = true,
CacheEsHomeInstallation = false,
TrialMode = XPackTrialMode.Trial,
NoCleanupAfterNodeStopped = false,
};
using (var cluster = new EphemeralCluster(config))
{
cluster.Start();
var nodes = cluster.NodesUris();
var connectionPool = new StaticConnectionPool(nodes);
var settings = new ConnectionSettings(connectionPool).EnableDebugMode();
if (config.EnableSecurity)
{
settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username,
ClusterAuthentication.Admin.Password);
}
if (config.EnableSsl)
{
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
}
var client = new ElasticClient(settings);
var clusterConfiguration = new Dictionary <string, object>()
{
{ "cluster.routing.use_adaptive_replica_selection", true },
{ "cluster.remote.remote-cluster.seeds", "127.0.0.1:9300" }
};
var putSettingsResponse = client.ClusterPutSettings(new ClusterPutSettingsRequest
{
Transient = clusterConfiguration
});
Console.Write(client.XPackInfo().DebugInformation);
Console.WriteLine("Press any key to exit");
Console.ReadKey();
Console.WriteLine("Exitting..");
}
Console.WriteLine("Done!");
}
private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
string binary, string description, StartedHandler startedHandler, params string[] arguments)
{
var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
var timeout = TimeSpan.FromSeconds(420);
var processStartArguments = new StartArguments(binary, arguments)
{
Environment = new Dictionary <string, string>
{
{ config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath },
{ "ES_HOME", config.FileSystem.ElasticsearchHome }
}
};
var result = startedHandler != null
? Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter(), startedHandler)
: Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());
if (!result.Completed)
{
throw new Exception($"Timeout while executing {description} exceeded {timeout}");
}
if (result.ExitCode != 0)
{
throw new Exception(
$"Expected exit code 0 but received ({result.ExitCode}) while executing {description}: {command}");
}
var errorOut = result.ConsoleOut.Where(c => c.Error).ToList();
// this manifested when calling certgen on versions smaller then 5.2.0
if (errorOut.Any() && config.Version < "5.2.0")
{
errorOut = errorOut.Where(e => !e.Line.Contains("No log4j2 configuration file found")).ToList();
}
if (errorOut.Any(e =>
!string.IsNullOrWhiteSpace(e.Line) && !e.Line.Contains("usage of JAVA_HOME is deprecated")) &&
!binary.Contains("plugin") && !binary.Contains("cert"))
{
throw new Exception(
$"Received error out with exitCode ({result.ExitCode}) while executing {description}: {command}");
}
writer?.WriteDiagnostic(
$"{{{nameof(ExecuteBinary)}}} finished process [{description}] {{{result.ExitCode}}}");
}
private void AddClientCertificateUser(EphemeralClusterConfiguration config)
{
var file = Path.Combine(config.FileSystem.ConfigPath, "x-pack", "role_mapping") + ".yml";
var name = config.FileSystem.ClientCertificateName;
if (!File.Exists(file) || !File.ReadAllLines(file).Any(f => f.Contains(name)))
{
File.WriteAllLines(file, new[]
{
"admin:",
$" - \"{name}\""
});
}
}
private void GenerateUnusedCertificates(EphemeralClusterConfiguration config, string silentModeConfigFile, IConsoleLineWriter writer)
{
var name = config.FileSystem.UnusedCertificateFolderName;
var zipLocation = Path.Combine(config.FileSystem.ConfigPath, "x-pack", name) + ".zip";
var @out = config.Version.Major < 6 ? $"{name}.zip" : zipLocation;
if (!File.Exists(config.FileSystem.UnusedCaCertificate))
{
ExecuteBinary(config, writer, config.FileSystem.CertGenBinary, "generating ssl certificates for this session",
"-in", silentModeConfigFile, "-out", @out);
}
if (Directory.Exists(config.FileSystem.UnusedCertificatesPath))
{
return;
}
Directory.CreateDirectory(config.FileSystem.UnusedCertificatesPath);
ZipFile.ExtractToDirectory(zipLocation, config.FileSystem.UnusedCertificatesPath);
}
private static void ManualConfigRun()
{
ElasticVersion version = "latest-7";
var plugins =
new ElasticsearchPlugins(ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.AnalysisKuromoji);
var features = Security | XPack | SSL;
var config = new EphemeralClusterConfiguration(version, features, null, numberOfNodes: 1)
{
HttpFiddlerAware = true,
ShowElasticsearchOutputAfterStarted = true,
CacheEsHomeInstallation = true,
TrialMode = XPackTrialMode.Trial,
NoCleanupAfterNodeStopped = false,
};
using (var cluster = new EphemeralCluster(config))
{
cluster.Start();
var nodes = cluster.NodesUris();
var connectionPool = new StaticConnectionPool(nodes);
var settings = new ConnectionSettings(connectionPool).EnableDebugMode();
if (config.EnableSecurity)
{
settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username,
ClusterAuthentication.Admin.Password);
}
if (config.EnableSsl)
{
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
}
var client = new ElasticClient(settings);
Console.Write(client.XPackInfo().DebugInformation);
Console.WriteLine("Press any key to exit");
Console.ReadKey();
Console.WriteLine("Exitting..");
}
Console.WriteLine("Done!");
}
protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineWriter writer, string binary, string description, params string[] arguments)
{
var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
var timeout = TimeSpan.FromSeconds(420);
var processStartArguments = new StartArguments(binary, arguments)
{
Environment = new Dictionary <string, string>
{
{ "ES_PATH_CONF", config.FileSystem.ConfigPath },
{ "ES_HOME", config.FileSystem.ElasticsearchHome }
}
};
var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());
if (!result.Completed)
{
throw new Exception($"Timeout while executing {description} exceeded {timeout}");
}
if (result.ExitCode != 0)
{
throw new Exception($"Expected exit code 0 but recieved ({result.ExitCode}) while executing {description}: {command}");
}
var errorOut = result.ConsoleOut.Where(c => c.Error).ToList();
if (errorOut.Any())
{
throw new Exception($"Recieved error out with exitCode ({result.ExitCode}) while executing {description}: {command}");
}
writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} finished process [{description}] {{{result.ExitCode}}}");
}
public static void Run()
{
var plugins = new ElasticsearchPlugins(ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.AnalysisKuromoji);
var versions = new string[] { "7.0.0-beta1", "latest", "latest-7", "latest-6", "957e3089:7.2.0", "6.6.1", "5.6.15" };
var features = new[]
{
ClusterFeatures.None,
// ClusterFeatures.XPack,
// ClusterFeatures.XPack | ClusterFeatures.Security,
ClusterFeatures.XPack | ClusterFeatures.SSL | ClusterFeatures.Security
};
foreach (var v in versions)
{
foreach (var f in features)
{
Console.Clear();
var reset = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($"{v} {f}");
Console.ForegroundColor = reset;
var config = new EphemeralClusterConfiguration(v, f, plugins, numberOfNodes: 1)
{
HttpFiddlerAware = true,
};
using (var cluster = new EphemeralCluster(config))
{
try
{
cluster.Start();
var nodes = cluster.NodesUris();
var connectionPool = new StaticConnectionPool(nodes);
var settings = new ConnectionSettings(connectionPool).EnableDebugMode();
if (config.EnableSecurity)
{
settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password);
}
if (config.EnableSsl)
{
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
}
var client = new ElasticClient(settings);
Console.WriteLine(client.RootNodeInfo().Version.Number);
cluster.Dispose();
cluster.WaitForExit(TimeSpan.FromMinutes(1));
}
catch (Exception e)
{
Console.WriteLine(e);
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine($"{v} {f}");
Console.ForegroundColor = reset;
throw;
}
}
}
}
Console.WriteLine("Done!");
}
private static void GenerateCertGenConfigFiles(IEphemeralCluster <EphemeralClusterConfiguration> cluster, INodeFileSystem fileSystem, string silentModeConfigFile, EphemeralClusterConfiguration config)
{
var silentModeConfigFileDuplicate = Path.Combine(fileSystem.ConfigPath, "x-pack", "certgen") + ".yml";
var files = cluster.ClusterConfiguration.Version >= "6.3.0"
? new[] { silentModeConfigFile }
: new[] { silentModeConfigFile, silentModeConfigFileDuplicate };
cluster.Writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} creating config files");
foreach (var file in files)
{
if (!File.Exists(file))
{
File.WriteAllLines(file, new[]
{
"instances:",
$" - name : \"{config.FileSystem.CertificateNodeName}\"",
$" - name : \"{config.FileSystem.ClientCertificateName}\"",
$" ip:",
$" - \"127.0.0.1\"",
$" dns:",
$" - \"localhost\"",
$" - \"ipv4.fiddler\"",
$" filename : \"{config.FileSystem.ClientCertificateFilename}\"",
});
}
}
}
protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
string binary, string description, StartedHandler startedHandler, params string[] arguments) =>
ExecuteBinaryInternal(config, writer, binary, description, startedHandler, arguments);
