public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var to = Path.Combine(cluster.FileSystem.LocalFolder, "server_metrics.tar.gz");
if (!File.Exists(to))
{
var from = "https://download.elasticsearch.org/demos/machine_learning/gettingstarted/server_metrics.tar.gz";
Console.WriteLine($"Download machine learning sample data from: {from}");
DownloadFile(from, to);
Console.WriteLine($"Downloaded machine learning sample data to: {to}");
}
var directoryTarget = Path.Combine(cluster.FileSystem.LocalFolder, "server_metrics");
if (Directory.Exists(directoryTarget))
{
return;
}
Directory.CreateDirectory(directoryTarget);
Console.WriteLine($"Unzipping machine learning sample data: {to} ...");
using (var inStream = File.OpenRead(to))
using (var gzipStream = new GZipInputStream(inStream))
using (var tarArchive = TarArchive.CreateInputTarArchive(gzipStream))
{
tarArchive.ExtractContents(directoryTarget);
tarArchive.Close();
}
}
public static ElasticsearchClient GetOrAddClient <TConfig>(
this IEphemeralCluster <TConfig> cluster,
Func <ElasticsearchClientSettings, ElasticsearchClientSettings> modifySettings = null
)
where TConfig : EphemeralClusterConfiguration
{
modifySettings ??= s => s;
return(cluster.GetOrAddClient(c =>
{
var settings = modifySettings(cluster.CreateConnectionSettings());
var current = (ITransportClientConfigurationValues)settings;
var notAlreadyAuthenticated = current.Authentication == null &&
current.ClientCertificates == null;
var noCertValidation = current.ServerCertificateValidationCallback == null;
if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated)
{
settings = settings.Authentication(new BasicAuthentication(ClusterAuthentication.Admin.Username,
ClusterAuthentication.Admin.Password));
}
if (cluster.ClusterConfiguration.EnableSsl && noCertValidation)
{
//todo use CA callback instead of allowall
// ReSharper disable once UnusedVariable
var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate);
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
}
var client = new ElasticsearchClient(settings);
return client;
}));
}
private void StartTrial(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var w = cluster.Writer;
var c = cluster.ClusterConfiguration;
if (c.Version < "6.3.0" || c.TrialMode == XPackTrialMode.None)
{
cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} {c.Version} < 6.3.0 or opting out of explicit basic/trial license");
return;
}
var licenseUrl = cluster.ClusterConfiguration.Version.Major < 8 ? "_xpack/license" : "_license";
if (c.TrialMode == XPackTrialMode.Trial)
{
//TODO make this configurable for either trial or basic
cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} attempt to start trial license");
var postResponse = this.Post(cluster, $"{licenseUrl}/start_trial", "acknowledge=true", string.Empty);
if (postResponse != null && postResponse.IsSuccessStatusCode)
{
return;
}
}
if (c.TrialMode == XPackTrialMode.Basic)
{
//TODO make this configurable for either trial or basic
cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} attempt to start basic license");
var postResponse = this.Post(cluster, $"{licenseUrl}/start_basic", "acknowledge=true", string.Empty);
if (postResponse != null && postResponse.IsSuccessStatusCode)
{
return;
}
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var fs = cluster.FileSystem;
var w = cluster.Writer;
var v = cluster.ClusterConfiguration.Version;
DeleteDirectory(w, "cluster data", fs.DataPath);
DeleteDirectory(w, "cluster config", fs.ConfigPath);
DeleteDirectory(w, "cluster logs", fs.LogsPath);
DeleteDirectory(w, "repositories", fs.RepositoryPath);
var efs = fs as EphemeralFileSystem;
if (!string.IsNullOrWhiteSpace(efs?.TempFolder))
{
DeleteDirectory(w, "cluster temp folder", efs.TempFolder);
}
if (efs != null)
{
var extractedFolder = Path.Combine(fs.LocalFolder, v.FolderInZip);
if (extractedFolder != fs.ElasticsearchHome)
{
DeleteDirectory(w, "ephemeral ES_HOME", fs.ElasticsearchHome);
}
}
}
private string LicenseInfo(IEphemeralCluster <EphemeralClusterConfiguration> cluster, string filter,
int retries = 0)
{
var licenseUrl = cluster.ClusterConfiguration.Version.Major < 8 ? "_xpack/license" : "_license";
var getLicense = Get(cluster, licenseUrl, "filter_path=" + filter);
if ((getLicense == null || !getLicense.IsSuccessStatusCode) && retries >= 5)
{
throw new Exception(
$"Calling GET _xpack/license did not result in an OK response after trying {retries} {GetResponseException(getLicense)}");
}
if (getLicense == null || !getLicense.IsSuccessStatusCode)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
return(LicenseInfo(cluster, filter, ++retries));
}
var licenseStatusString = GetResponseString(getLicense)
.Where(c => !new[] { ' ', '\n', '{', '"', '}' }.Contains(c))
.ToArray();
var status = new string(licenseStatusString).Replace($"{filter.Replace(".", ":")}:", "");
return(status);
}
public static IElasticClient GetOrAddClient <TConfig>(
this IEphemeralCluster <TConfig> cluster,
Func <ConnectionSettings, ConnectionSettings> createSettings = null,
Func <ICollection <Uri>, IConnectionPool> createPool = null)
where TConfig : EphemeralClusterConfiguration
{
createSettings = createSettings ?? (s => s);
return(cluster.GetOrAddClient(c =>
{
var host = (RunningFiddler) ? "ipv4.fiddler" : "localhost";
createPool = createPool ?? (uris => new StaticConnectionPool(uris));
var connectionPool = createPool(c.NodesUris(host));
var connection = TestClient.Configuration.RunIntegrationTests ? TestClient.CreateLiveConnection() : new InMemoryConnection();
var settings = TestClient.CreateSettings(createSettings, connection, connectionPool);
var current = (IConnectionConfigurationValues)settings;
var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null;
var noCertValidation = current.ServerCertificateValidationCallback == null;
if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated)
{
settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password);
}
if (cluster.ClusterConfiguration.EnableSsl && noCertValidation)
{
var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate);
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
}
var client = new ElasticClient(settings);
return client;
}));
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var fs = cluster.FileSystem;
var v = cluster.ClusterConfiguration.Version;
if (Directory.Exists(fs.ElasticsearchHome))
{
cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} skipping [{fs.ElasticsearchHome}] already exists");
return;
}
var from = Path.Combine(fs.LocalFolder, v.Zip);
var extractedFolder = Path.Combine(fs.LocalFolder, v.FolderInZip);
if (!Directory.Exists(extractedFolder))
{
cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} unzipping version [{v}] {{{from}}}");
ZipFile.ExtractToDirectory(from, fs.LocalFolder);
cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} extracted version [{v}] to {{{fs.LocalFolder}}}");
}
if (extractedFolder == fs.ElasticsearchHome)
{
return;
}
cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} Copying extracted folder {{{extractedFolder}}} => {fs.ElasticsearchHome}");
Copy(new DirectoryInfo(extractedFolder), new DirectoryInfo(fs.ElasticsearchHome));
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (!cluster.ClusterConfiguration.CacheEsHomeInstallation)
{
return;
}
var fs = cluster.FileSystem;
var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName());
var cachedelasticsearchYaml = Path.Combine(cachedEsHomeFolder, "config", "elasticsearch.yml");
if (File.Exists(cachedelasticsearchYaml))
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(CacheElasticsearchInstallation)}}} cached home already exists [{cachedEsHomeFolder}]");
return;
}
var source = fs.ElasticsearchHome;
var target = cachedEsHomeFolder;
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(CacheElasticsearchInstallation)}}} caching {{{source}}} to [{target}]");
CopyFolder(source, target, false);
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var c = cluster.ClusterConfiguration;
var version = c.Version;
string F(ClusterFeatures feature) => c.Features.HasFlag(feature) ? Enum.GetName(typeof(ClusterFeatures), feature) : string.Empty;
var features = string.Join("|", new[] { F(Security), F(ClusterFeatures.XPack), F(SSL) }.Where(v => !string.IsNullOrWhiteSpace(v)));
features = string.IsNullOrWhiteSpace(features) ? "None" : features;
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} starting {{{version}}} with features [{features}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NumberOfNodes)}}} [{c.NumberOfNodes}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ClusterName)}}} [{c.ClusterName}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSsl)}}} [{c.EnableSsl}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSecurity)}}} [{c.EnableSecurity}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.XPackInstalled)}}} [{c.XPackInstalled}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.Plugins)}}} [{c.Plugins}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.TrialMode)}}} [{c.TrialMode}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.CacheEsHomeInstallation)}}} [{c.CacheEsHomeInstallation}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ShowElasticsearchOutputAfterStarted)}}} [{c.ShowElasticsearchOutputAfterStarted}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ValidatePluginsToInstall)}}} [{c.ValidatePluginsToInstall}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.PrintYamlFilesInConfigFolder)}}} [{c.PrintYamlFilesInConfigFolder}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.SkipBuiltInAfterStartTasks)}}} [{c.SkipBuiltInAfterStartTasks}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.HttpFiddlerAware)}}} [{c.HttpFiddlerAware}]");
cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NoCleanupAfterNodeStopped)}}} [{c.NoCleanupAfterNodeStopped}]");
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (!cluster.ClusterConfiguration.EnableSsl)
{
return;
}
var config = cluster.ClusterConfiguration;
var fileSystem = cluster.FileSystem;
//due to a bug in certgen this file needs to live in two places
var silentModeConfigFile = Path.Combine(fileSystem.ElasticsearchHome, "certgen") + ".yml";
var silentModeConfigFileDuplicate = Path.Combine(fileSystem.ConfigPath, "x-pack", "certgen") + ".yml";
cluster.Writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} creating config files");
foreach (var file in new[] { silentModeConfigFile, silentModeConfigFileDuplicate })
{
if (!File.Exists(file))
{
File.WriteAllLines(file, new[]
{
"instances:",
$" - name : \"{config.FileSystem.CertificateNodeName}\"",
$" - name : \"{config.FileSystem.ClientCertificateName}\"",
$" filename : \"{config.FileSystem.ClientCertificateFilename}\"",
});
}
}
this.GenerateCertificates(config, silentModeConfigFile, cluster.Writer);
this.GenerateUnusedCertificates(config, silentModeConfigFile, cluster.Writer);
this.AddClientCertificateUser(config);
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (cluster.CachingAndCachedHomeExists())
{
return;
}
var fs = cluster.FileSystem;
var v = cluster.ClusterConfiguration.Version;
var a = cluster.ClusterConfiguration.Artifact;
var from = v.Artifact(Product.Elasticsearch).DownloadUrl;
var to = Path.Combine(fs.LocalFolder, a.Archive);
if (File.Exists(to))
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(DownloadElasticsearchVersion)}}} {v} was already downloaded");
return;
}
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(DownloadElasticsearchVersion)}}} downloading Elasticsearch [{v}] from {{{from}}} {{{to}}}");
DownloadFile(from, to);
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(DownloadElasticsearchVersion)}}} downloaded Elasticsearch [{v}] from {{{from}}} {{{to}}}");
}
private static void NewOrCachedCertificates(IEphemeralCluster <EphemeralClusterConfiguration> cluster, string name, string path, string silentModeConfigFile, IConsoleLineHandler writer)
{
var config = cluster.ClusterConfiguration;
var cachedEsHomeFolder = Path.Combine(config.FileSystem.LocalFolder, cluster.GetCacheFolderName());
var zipLocationCache = Path.Combine(cachedEsHomeFolder, name) + ".zip";
if (File.Exists(zipLocationCache))
{
writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} using cached certificates from {zipLocationCache}");
UnpackCertificatesZip(zipLocationCache, path, writer);
return;
}
var zipLocation = config.Version >= "6.3.0"
? Path.Combine(config.FileSystem.ConfigPath, name) + ".zip"
: Path.Combine(config.FileSystem.ConfigPath, "x-pack", name) + ".zip";
GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer);
if (!File.Exists(zipLocationCache))
{
writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} caching {zipLocation} in ES_HOME {zipLocationCache}");
File.Copy(zipLocation, zipLocationCache);
}
UnpackCertificatesZip(zipLocation, path, writer);
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (!cluster.ClusterConfiguration.XPackInstalled)
{
return;
}
if (cluster.CachingAndCachedHomeExists())
{
return;
}
var config = cluster.ClusterConfiguration;
var v = config.Version;
var envBinary = Path.Combine(config.FileSystem.ElasticsearchHome, "bin", "x-pack", "x-pack-env") + BinarySuffix;
if (v.Major != 6 || File.Exists(envBinary))
{
return;
}
if (v >= "6.3.0")
{
return;
}
cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureSecurityUsersInDefaultRealmAreAdded)}}} {envBinary} does not exist, patching now.");
File.WriteAllText(envBinary, "set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/plugins/x-pack/*");
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (!cluster.ClusterConfiguration.XPackInstalled)
{
return;
}
if (string.IsNullOrWhiteSpace(cluster.ClusterConfiguration.XPackLicenseJson))
{
cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} no license file available to post");
StartTrial(cluster);
return;
}
cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} attempting to post license json");
var licenseUrl = cluster.ClusterConfiguration.Version.Major < 8 ? "_xpack/license" : "_license";
var postResponse = Post(cluster, licenseUrl, "", cluster.ClusterConfiguration.XPackLicenseJson);
if (postResponse != null && postResponse.IsSuccessStatusCode)
{
return;
}
var details = postResponse != null?GetResponseException(postResponse) : "";
throw new Exception($"The license that was posted was not accepted: {details}");
}
private static void GenerateCertGenConfigFiles(IEphemeralCluster <EphemeralClusterConfiguration> cluster, INodeFileSystem fileSystem, string silentModeConfigFile, EphemeralClusterConfiguration config)
{
var silentModeConfigFileDuplicate = Path.Combine(fileSystem.ConfigPath, "x-pack", "certgen") + ".yml";
var files = cluster.ClusterConfiguration.Version >= "6.3.0"
? new[] { silentModeConfigFile }
: new[] { silentModeConfigFile, silentModeConfigFileDuplicate };
cluster.Writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} creating config files");
foreach (var file in files)
{
if (!File.Exists(file))
{
File.WriteAllLines(file, new[]
{
"instances:",
$" - name : \"{config.FileSystem.CertificateNodeName}\"",
$" - name : \"{config.FileSystem.ClientCertificateName}\"",
$" ip:",
$" - \"127.0.0.1\"",
$" dns:",
$" - \"localhost\"",
$" - \"ipv4.fiddler\"",
$" filename : \"{config.FileSystem.ClientCertificateFilename}\"",
});
}
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (cluster.CachingAndCachedHomeExists())
{
return;
}
var config = cluster.ClusterConfiguration;
if (config.Version < "6.2.0" || config.Version >= "6.3.0")
{
return;
}
var batFile = cluster.FileSystem.Binary;
cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureElasticsearchBatWorksAcrossDrives)}}} patching {batFile} according to elastic/elasticsearch#29057");
var contents = File.ReadAllLines(batFile);
for (var i = 0; i < contents.Length; i++)
{
if (contents[i] != "cd \"%ES_HOME%\"")
{
continue;
}
contents[i] = "cd /d \"%ES_HOME%\"";
break;
}
File.WriteAllLines(batFile, contents);
}
public void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster, bool nodeStarted)
{
var fs = cluster.FileSystem;
var w = cluster.Writer;
var v = cluster.ClusterConfiguration.Version;
DeleteDirectory(w, "cluster data", fs.DataPath);
DeleteDirectory(w, "cluster config", fs.ConfigPath);
DeleteDirectory(w, "cluster logs", fs.LogsPath);
DeleteDirectory(w, "repositories", fs.RepositoryPath);
var efs = fs as EphemeralFileSystem;
if (!string.IsNullOrWhiteSpace(efs?.TempFolder))
{
DeleteDirectory(w, "cluster temp folder", efs.TempFolder);
}
if (efs != null)
{
var extractedFolder = Path.Combine(fs.LocalFolder, v.FolderInZip);
if (extractedFolder != fs.ElasticsearchHome)
{
DeleteDirectory(w, "ephemeral ES_HOME", fs.ElasticsearchHome);
}
}
//if the node did not start make sure we delete the cached folder as we can not assume its in a good state
if (cluster.ClusterConfiguration.CacheEsHomeInstallation && !nodeStarted)
{
var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName());
DeleteDirectory(w, "cached installation - node failed to start", cachedEsHomeFolder);
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var keystoreFile = Path.Combine(cluster.FileSystem.ConfigPath, "elasticsearch.keystore");
var binary = Path.Combine(cluster.FileSystem.ElasticsearchHome, "bin", "elasticsearch-keystore" + BinarySuffix);
if (!File.Exists(keystoreFile))
{
ExecuteBinary(
cluster.ClusterConfiguration,
cluster.Writer,
binary,
"creating elasticsearch.keystore",
"create");
}
ExecuteBinary(
cluster.ClusterConfiguration,
cluster.Writer,
binary,
"add xpack.notification.slack.account.monitoring.secure_url to elasticsearch.keystore",
input => input.WriteLine("https://hooks.slack.com/services/EXAMPLE/EXAMPLE/EXAMPLE"),
"add", "xpack.notification.slack.account.monitoring.secure_url", "-xf");
ExecuteBinary(
cluster.ClusterConfiguration,
cluster.Writer,
binary,
"add xpack.notification.pagerduty.account.my_pagerduty_account.secure_service_api_key to elasticsearch.keystore",
input => input.WriteLine("dummy_key"),
"add", "xpack.notification.pagerduty.account.my_pagerduty_account.secure_service_api_key", "-xf");
cluster.Writer.WriteDiagnostic(
$"{{{nameof(EnsureWatcherActionConfigurationSecretsInKeystore)}}} added watcher action secrets to elasticsearch.keystore");
}
private static void CopyConfigDirectoryToHomeCacheConfigDirectory(IEphemeralCluster <EphemeralClusterConfiguration> cluster, ElasticsearchPlugin plugin)
{
if (plugin.SubProductName == "x-pack")
{
return;
}
if (!cluster.ClusterConfiguration.CacheEsHomeInstallation)
{
return;
}
var fs = cluster.FileSystem;
var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName());
var configTarget = Path.Combine(cachedEsHomeFolder, "config");
var configPluginPath = Path.Combine(fs.ConfigPath, plugin.SubProductName);
var configPluginPathCached = Path.Combine(configTarget, plugin.SubProductName);
if (!Directory.Exists(configPluginPath) || Directory.Exists(configPluginPathCached))
{
return;
}
Directory.CreateDirectory(configPluginPathCached);
CopyFolder(configPluginPath, configPluginPathCached);
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var v = cluster.ClusterConfiguration.Version;
//on 2.x we do not support tests requiring plugins for 2.x since we can not reliably install them
if (v.Major == 2)
{
cluster.Writer?.WriteDiagnostic($"{{{nameof(InstallPlugins)}}} skipping install plugins on {{2.x}} version: [{v}]");
return;
}
var fs = cluster.FileSystem;
var requiredPlugins = cluster.ClusterConfiguration.Plugins;
var invalidPlugins = requiredPlugins.Where(p => !p.IsValid(v)).Select(p => p.Moniker).ToList();
if (invalidPlugins.Any())
{
throw new CleanExitException($"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} ");
}
var plugins =
from plugin in requiredPlugins
let validForCurrentVersion = plugin.IsValid(v)
let alreadyInstalled = AlreadyInstalled(fs, plugin.FolderName)
where !alreadyInstalled && validForCurrentVersion
select plugin;
foreach (var plugin in plugins)
{
//var installParameter = v.ReleaseState == ReleaseState.Released ? plugin.Moniker : UseHttpPluginLocation(cluster.Writer, fs, plugin, v);
var installParameter = UseHttpPluginLocation(cluster.Writer, fs, plugin, v);
ExecuteBinary(cluster.ClusterConfiguration, cluster.Writer, "cmd", $"install elasticsearch plugin: {plugin.Moniker}", $"/c CALL {fs.PluginBinary} install --batch", installParameter);
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (!cluster.ClusterConfiguration.EnableSsl)
{
return;
}
var config = cluster.ClusterConfiguration;
var v = config.Version;
var file = v >= "6.3.0"
? Path.Combine(config.FileSystem.ConfigPath, "role_mapping") + ".yml"
: Path.Combine(config.FileSystem.ConfigPath, "x-pack", "role_mapping") + ".yml";
var name = config.FileSystem.ClientCertificateName;
if (!File.Exists(file) || !File.ReadAllLines(file).Any(f => f.Contains(name)))
{
File.WriteAllLines(file, new[]
{
"admin:",
$" - \"{name}\""
});
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
if (!cluster.ClusterConfiguration.EnableSecurity)
{
return;
}
var configFile = Path.Combine(cluster.FileSystem.ConfigPath, "elasticsearch.yml");
cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureSecurityRealms)}}} attempting to add xpack realms to [{configFile}]");
var lines = File.ReadAllLines(configFile).ToList();
var saveFile = false;
var major = cluster.ClusterConfiguration.Version.Major;
saveFile = major >= 6
? (major >= 7 ? Write7XAndUpRealms(lines, cluster.ClusterConfiguration.EnableSsl) : Write6XAndUpRealms(lines))
: Write5XAndUpRealms(lines);
if (saveFile)
{
File.WriteAllLines(configFile, lines);
}
cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureSecurityRealms)}}} {(saveFile ? "saved" : "skipped saving")} xpack realms to [{configFile}]");
}
public static IElasticClient GetOrAddClient(this IEphemeralCluster cluster) =>
Clients.GetOrAdd(cluster, (c) =>
{
var connectionPool = new StaticConnectionPool(c.NodesUris());
var settings = new ConnectionSettings(connectionPool);
var client = new ElasticClient(settings);
return(client);
});
public static ConnectionSettings CreateConnectionSettings <TConfig>(this IEphemeralCluster <TConfig> cluster)
where TConfig : EphemeralClusterConfiguration
{
var clusterNodes = cluster.NodesUris(TestConnectionSettings.LocalOrProxyHost);
//we ignore the uri's that TestConnection provides and seed with the nodes the cluster dictates.
return(new TestConnectionSettings(uris => new StaticConnectionPool(clusterNodes)));
}
private static void GenerateCertificates(IEphemeralCluster <EphemeralClusterConfiguration> cluster, string silentModeConfigFile, IConsoleLineHandler writer)
{
var config = cluster.ClusterConfiguration;
var name = config.FileSystem.CertificateFolderName;
var path = config.FileSystem.CertificatesPath;
NewOrCachedCertificates(cluster, name, path, silentModeConfigFile, writer);
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
cluster.Writer.WriteDiagnostic($"{{{nameof(ValidateClusterStateTask)}}} waiting cluster to go into yellow health state");
var healthyResponse = this.Get(cluster, "_cluster/health", "wait_for_status=yellow&timeout=20s");
if (healthyResponse == null || healthyResponse.StatusCode != HttpStatusCode.OK)
{
throw new Exception($"Cluster health waiting for status yellow failed after 20s");
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var v = cluster.ClusterConfiguration.Version;
if (v.Major == 2)
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(ValidatePluginsTask)}}} skipping validate plugins on {{2.x}} version: [{v}]");
return;
}
if (v.Major < 7 && v.ArtifactBuildState == ArtifactBuildState.Snapshot)
{
cluster.Writer?.WriteDiagnostic(
$"{{{nameof(InstallPlugins)}}} skipping validate SNAPSHOT plugins on < {{7.x}} version: [{v}]");
return;
}
var requestPlugins = cluster.ClusterConfiguration.Plugins
.Where(p => p.IsValid(v))
.Where(p => !p.IsIncludedOutOfTheBox(v))
.Select(p => p.GetExistsMoniker(v))
.ToList();
if (!requestPlugins.Any())
{
return;
}
cluster.Writer.WriteDiagnostic(
$"{{{nameof(ValidatePluginsTask)}}} validating the cluster is running the requested plugins");
var catPlugins = Get(cluster, "_cat/plugins", "h=component");
if (catPlugins == null || !catPlugins.IsSuccessStatusCode)
{
throw new Exception(
$"Calling _cat/plugins did not result in an OK response {GetResponseException(catPlugins)}");
}
var installedPlugins = GetResponseString(catPlugins)
.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).ToList();
var missingPlugins = requestPlugins.Except(installedPlugins).ToList();
if (!missingPlugins.Any())
{
return;
}
var missingString = string.Join(", ", missingPlugins);
var pluginsString = string.Join(", ", installedPlugins);
throw new Exception(
$"Already running elasticsearch missed the following plugin(s): {missingString} currently installed: {pluginsString}.");
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var javaHome = Environment.GetEnvironmentVariable("JAVA_HOME");
if (string.IsNullOrWhiteSpace(javaHome))
{
cluster.Writer?.WriteDiagnostic($"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} JAVA_HOME is not SET exiting..");
throw new Exception("The elasticsearch bat files are resillient to JAVA_HOME not being set, however the shield tooling is not");
}
cluster.Writer?.WriteDiagnostic($"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} JAVA_HOME is set proceeding");
}
public void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster, bool nodeStarted)
{
var fs = cluster.FileSystem;
var w = cluster.Writer;
var v = cluster.ClusterConfiguration.Version;
var a = cluster.ClusterConfiguration.Artifact;
if (cluster.ClusterConfiguration.NoCleanupAfterNodeStopped)
{
w.WriteDiagnostic(
$"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} skipping cleanup as requested on cluster configuration");
return;
}
DeleteDirectory(w, "cluster data", fs.DataPath);
DeleteDirectory(w, "cluster config", fs.ConfigPath);
DeleteDirectory(w, "cluster logs", fs.LogsPath);
DeleteDirectory(w, "repositories", fs.RepositoryPath);
var efs = fs as EphemeralFileSystem;
if (!string.IsNullOrWhiteSpace(efs?.TempFolder))
{
DeleteDirectory(w, "cluster temp folder", efs.TempFolder);
}
if (efs != null)
{
var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip);
if (extractedFolder != fs.ElasticsearchHome)
{
DeleteDirectory(w, "ephemeral ES_HOME", fs.ElasticsearchHome);
}
//if the node was not started delete the cached extractedFolder
if (!nodeStarted)
{
DeleteDirectory(w, "cached extracted folder - node failed to start", extractedFolder);
}
}
//if the node did not start make sure we delete the cached folder as we can not assume its in a good state
var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName());
if (cluster.ClusterConfiguration.CacheEsHomeInstallation && !nodeStarted)
{
DeleteDirectory(w, "cached installation - node failed to start", cachedEsHomeFolder);
}
else
{
w.WriteDiagnostic(
$"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} Leaving [cached folder] on disk: {{{cachedEsHomeFolder}}}");
}
}
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster)
{
var configFile = Path.Combine(cluster.FileSystem.ConfigPath, "elasticsearch.yml");
var lines = File.ReadAllLines(configFile).ToList();
var saveFile = false;
var v = cluster.ClusterConfiguration.Version;
var prefix = v.Major >= 5 ? "xpack.notification" : "watcher.actions";
var postfix = v.Major >= 5 ? string.Empty : ".service";
// set up for Watcher Slack action
if (!lines.Any(line => line.StartsWith($"{prefix}.slack{postfix}:")))
{
lines.AddRange(new[]
{
string.Empty,
$"{prefix}.slack{postfix}:",
" account:",
" monitoring:",
" message_defaults:",
" from: x-pack",
string.Empty
});
saveFile = true;
}
// set up for Watcher PagerDuty action
if (!lines.Any(line => line.StartsWith($"{prefix}.pagerduty{postfix}:")))
{
lines.AddRange(new[]
{
string.Empty,
$"{prefix}.pagerduty{postfix}:",
" account:",
" my_pagerduty_account:",
" event_defaults:",
" description: pager_duty",
//" service_api_key: pager_duty_service_api_key",
string.Empty
});
saveFile = true;
}
if (saveFile)
{
File.WriteAllLines(configFile, lines);
}
cluster.Writer.WriteDiagnostic(
$"{{{nameof(EnsureWatcherActionConfigurationInElasticsearchYaml)}}} {(saveFile ? "saved" : "skipped saving")} watcher config [{configFile}]");
}