public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var to = Path.Combine(cluster.FileSystem.LocalFolder, "server_metrics.tar.gz"); if (!File.Exists(to)) { var from = "https://download.elasticsearch.org/demos/machine_learning/gettingstarted/server_metrics.tar.gz"; Console.WriteLine($"Download machine learning sample data from: {from}"); DownloadFile(from, to); Console.WriteLine($"Downloaded machine learning sample data to: {to}"); } var directoryTarget = Path.Combine(cluster.FileSystem.LocalFolder, "server_metrics"); if (Directory.Exists(directoryTarget)) { return; } Directory.CreateDirectory(directoryTarget); Console.WriteLine($"Unzipping machine learning sample data: {to} ..."); using (var inStream = File.OpenRead(to)) using (var gzipStream = new GZipInputStream(inStream)) using (var tarArchive = TarArchive.CreateInputTarArchive(gzipStream)) { tarArchive.ExtractContents(directoryTarget); tarArchive.Close(); } }
public static ElasticsearchClient GetOrAddClient <TConfig>( this IEphemeralCluster <TConfig> cluster, Func <ElasticsearchClientSettings, ElasticsearchClientSettings> modifySettings = null ) where TConfig : EphemeralClusterConfiguration { modifySettings ??= s => s; return(cluster.GetOrAddClient(c => { var settings = modifySettings(cluster.CreateConnectionSettings()); var current = (ITransportClientConfigurationValues)settings; var notAlreadyAuthenticated = current.Authentication == null && current.ClientCertificates == null; var noCertValidation = current.ServerCertificateValidationCallback == null; if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated) { settings = settings.Authentication(new BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password)); } if (cluster.ClusterConfiguration.EnableSsl && noCertValidation) { //todo use CA callback instead of allowall // ReSharper disable once UnusedVariable var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate); settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); } var client = new ElasticsearchClient(settings); return client; })); }
private void StartTrial(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var w = cluster.Writer; var c = cluster.ClusterConfiguration; if (c.Version < "6.3.0" || c.TrialMode == XPackTrialMode.None) { cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} {c.Version} < 6.3.0 or opting out of explicit basic/trial license"); return; } var licenseUrl = cluster.ClusterConfiguration.Version.Major < 8 ? "_xpack/license" : "_license"; if (c.TrialMode == XPackTrialMode.Trial) { //TODO make this configurable for either trial or basic cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} attempt to start trial license"); var postResponse = this.Post(cluster, $"{licenseUrl}/start_trial", "acknowledge=true", string.Empty); if (postResponse != null && postResponse.IsSuccessStatusCode) { return; } } if (c.TrialMode == XPackTrialMode.Basic) { //TODO make this configurable for either trial or basic cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} attempt to start basic license"); var postResponse = this.Post(cluster, $"{licenseUrl}/start_basic", "acknowledge=true", string.Empty); if (postResponse != null && postResponse.IsSuccessStatusCode) { return; } } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var fs = cluster.FileSystem; var w = cluster.Writer; var v = cluster.ClusterConfiguration.Version; DeleteDirectory(w, "cluster data", fs.DataPath); DeleteDirectory(w, "cluster config", fs.ConfigPath); DeleteDirectory(w, "cluster logs", fs.LogsPath); DeleteDirectory(w, "repositories", fs.RepositoryPath); var efs = fs as EphemeralFileSystem; if (!string.IsNullOrWhiteSpace(efs?.TempFolder)) { DeleteDirectory(w, "cluster temp folder", efs.TempFolder); } if (efs != null) { var extractedFolder = Path.Combine(fs.LocalFolder, v.FolderInZip); if (extractedFolder != fs.ElasticsearchHome) { DeleteDirectory(w, "ephemeral ES_HOME", fs.ElasticsearchHome); } } }
private string LicenseInfo(IEphemeralCluster <EphemeralClusterConfiguration> cluster, string filter, int retries = 0) { var licenseUrl = cluster.ClusterConfiguration.Version.Major < 8 ? "_xpack/license" : "_license"; var getLicense = Get(cluster, licenseUrl, "filter_path=" + filter); if ((getLicense == null || !getLicense.IsSuccessStatusCode) && retries >= 5) { throw new Exception( $"Calling GET _xpack/license did not result in an OK response after trying {retries} {GetResponseException(getLicense)}"); } if (getLicense == null || !getLicense.IsSuccessStatusCode) { Thread.Sleep(TimeSpan.FromSeconds(10)); return(LicenseInfo(cluster, filter, ++retries)); } var licenseStatusString = GetResponseString(getLicense) .Where(c => !new[] { ' ', '\n', '{', '"', '}' }.Contains(c)) .ToArray(); var status = new string(licenseStatusString).Replace($"{filter.Replace(".", ":")}:", ""); return(status); }
public static IElasticClient GetOrAddClient <TConfig>( this IEphemeralCluster <TConfig> cluster, Func <ConnectionSettings, ConnectionSettings> createSettings = null, Func <ICollection <Uri>, IConnectionPool> createPool = null) where TConfig : EphemeralClusterConfiguration { createSettings = createSettings ?? (s => s); return(cluster.GetOrAddClient(c => { var host = (RunningFiddler) ? "ipv4.fiddler" : "localhost"; createPool = createPool ?? (uris => new StaticConnectionPool(uris)); var connectionPool = createPool(c.NodesUris(host)); var connection = TestClient.Configuration.RunIntegrationTests ? TestClient.CreateLiveConnection() : new InMemoryConnection(); var settings = TestClient.CreateSettings(createSettings, connection, connectionPool); var current = (IConnectionConfigurationValues)settings; var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; var noCertValidation = current.ServerCertificateValidationCallback == null; if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated) { settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); } if (cluster.ClusterConfiguration.EnableSsl && noCertValidation) { var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate); settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); } var client = new ElasticClient(settings); return client; })); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var fs = cluster.FileSystem; var v = cluster.ClusterConfiguration.Version; if (Directory.Exists(fs.ElasticsearchHome)) { cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} skipping [{fs.ElasticsearchHome}] already exists"); return; } var from = Path.Combine(fs.LocalFolder, v.Zip); var extractedFolder = Path.Combine(fs.LocalFolder, v.FolderInZip); if (!Directory.Exists(extractedFolder)) { cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} unzipping version [{v}] {{{from}}}"); ZipFile.ExtractToDirectory(from, fs.LocalFolder); cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} extracted version [{v}] to {{{fs.LocalFolder}}}"); } if (extractedFolder == fs.ElasticsearchHome) { return; } cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipElasticsearch)}}} Copying extracted folder {{{extractedFolder}}} => {fs.ElasticsearchHome}"); Copy(new DirectoryInfo(extractedFolder), new DirectoryInfo(fs.ElasticsearchHome)); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (!cluster.ClusterConfiguration.CacheEsHomeInstallation) { return; } var fs = cluster.FileSystem; var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); var cachedelasticsearchYaml = Path.Combine(cachedEsHomeFolder, "config", "elasticsearch.yml"); if (File.Exists(cachedelasticsearchYaml)) { cluster.Writer?.WriteDiagnostic( $"{{{nameof(CacheElasticsearchInstallation)}}} cached home already exists [{cachedEsHomeFolder}]"); return; } var source = fs.ElasticsearchHome; var target = cachedEsHomeFolder; cluster.Writer?.WriteDiagnostic( $"{{{nameof(CacheElasticsearchInstallation)}}} caching {{{source}}} to [{target}]"); CopyFolder(source, target, false); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var c = cluster.ClusterConfiguration; var version = c.Version; string F(ClusterFeatures feature) => c.Features.HasFlag(feature) ? Enum.GetName(typeof(ClusterFeatures), feature) : string.Empty; var features = string.Join("|", new[] { F(Security), F(ClusterFeatures.XPack), F(SSL) }.Where(v => !string.IsNullOrWhiteSpace(v))); features = string.IsNullOrWhiteSpace(features) ? "None" : features; cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} starting {{{version}}} with features [{features}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NumberOfNodes)}}} [{c.NumberOfNodes}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ClusterName)}}} [{c.ClusterName}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSsl)}}} [{c.EnableSsl}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSecurity)}}} [{c.EnableSecurity}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.XPackInstalled)}}} [{c.XPackInstalled}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.Plugins)}}} [{c.Plugins}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.TrialMode)}}} [{c.TrialMode}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.CacheEsHomeInstallation)}}} [{c.CacheEsHomeInstallation}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ShowElasticsearchOutputAfterStarted)}}} [{c.ShowElasticsearchOutputAfterStarted}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ValidatePluginsToInstall)}}} [{c.ValidatePluginsToInstall}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.PrintYamlFilesInConfigFolder)}}} [{c.PrintYamlFilesInConfigFolder}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.SkipBuiltInAfterStartTasks)}}} [{c.SkipBuiltInAfterStartTasks}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.HttpFiddlerAware)}}} [{c.HttpFiddlerAware}]"); cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NoCleanupAfterNodeStopped)}}} [{c.NoCleanupAfterNodeStopped}]"); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (!cluster.ClusterConfiguration.EnableSsl) { return; } var config = cluster.ClusterConfiguration; var fileSystem = cluster.FileSystem; //due to a bug in certgen this file needs to live in two places var silentModeConfigFile = Path.Combine(fileSystem.ElasticsearchHome, "certgen") + ".yml"; var silentModeConfigFileDuplicate = Path.Combine(fileSystem.ConfigPath, "x-pack", "certgen") + ".yml"; cluster.Writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} creating config files"); foreach (var file in new[] { silentModeConfigFile, silentModeConfigFileDuplicate }) { if (!File.Exists(file)) { File.WriteAllLines(file, new[] { "instances:", $" - name : \"{config.FileSystem.CertificateNodeName}\"", $" - name : \"{config.FileSystem.ClientCertificateName}\"", $" filename : \"{config.FileSystem.ClientCertificateFilename}\"", }); } } this.GenerateCertificates(config, silentModeConfigFile, cluster.Writer); this.GenerateUnusedCertificates(config, silentModeConfigFile, cluster.Writer); this.AddClientCertificateUser(config); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (cluster.CachingAndCachedHomeExists()) { return; } var fs = cluster.FileSystem; var v = cluster.ClusterConfiguration.Version; var a = cluster.ClusterConfiguration.Artifact; var from = v.Artifact(Product.Elasticsearch).DownloadUrl; var to = Path.Combine(fs.LocalFolder, a.Archive); if (File.Exists(to)) { cluster.Writer?.WriteDiagnostic( $"{{{nameof(DownloadElasticsearchVersion)}}} {v} was already downloaded"); return; } cluster.Writer?.WriteDiagnostic( $"{{{nameof(DownloadElasticsearchVersion)}}} downloading Elasticsearch [{v}] from {{{from}}} {{{to}}}"); DownloadFile(from, to); cluster.Writer?.WriteDiagnostic( $"{{{nameof(DownloadElasticsearchVersion)}}} downloaded Elasticsearch [{v}] from {{{from}}} {{{to}}}"); }
private static void NewOrCachedCertificates(IEphemeralCluster <EphemeralClusterConfiguration> cluster, string name, string path, string silentModeConfigFile, IConsoleLineHandler writer) { var config = cluster.ClusterConfiguration; var cachedEsHomeFolder = Path.Combine(config.FileSystem.LocalFolder, cluster.GetCacheFolderName()); var zipLocationCache = Path.Combine(cachedEsHomeFolder, name) + ".zip"; if (File.Exists(zipLocationCache)) { writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} using cached certificates from {zipLocationCache}"); UnpackCertificatesZip(zipLocationCache, path, writer); return; } var zipLocation = config.Version >= "6.3.0" ? Path.Combine(config.FileSystem.ConfigPath, name) + ".zip" : Path.Combine(config.FileSystem.ConfigPath, "x-pack", name) + ".zip"; GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer); if (!File.Exists(zipLocationCache)) { writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} caching {zipLocation} in ES_HOME {zipLocationCache}"); File.Copy(zipLocation, zipLocationCache); } UnpackCertificatesZip(zipLocation, path, writer); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (!cluster.ClusterConfiguration.XPackInstalled) { return; } if (cluster.CachingAndCachedHomeExists()) { return; } var config = cluster.ClusterConfiguration; var v = config.Version; var envBinary = Path.Combine(config.FileSystem.ElasticsearchHome, "bin", "x-pack", "x-pack-env") + BinarySuffix; if (v.Major != 6 || File.Exists(envBinary)) { return; } if (v >= "6.3.0") { return; } cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureSecurityUsersInDefaultRealmAreAdded)}}} {envBinary} does not exist, patching now."); File.WriteAllText(envBinary, "set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/plugins/x-pack/*"); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (!cluster.ClusterConfiguration.XPackInstalled) { return; } if (string.IsNullOrWhiteSpace(cluster.ClusterConfiguration.XPackLicenseJson)) { cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} no license file available to post"); StartTrial(cluster); return; } cluster.Writer.WriteDiagnostic($"{{{nameof(PostLicenseTask)}}} attempting to post license json"); var licenseUrl = cluster.ClusterConfiguration.Version.Major < 8 ? "_xpack/license" : "_license"; var postResponse = Post(cluster, licenseUrl, "", cluster.ClusterConfiguration.XPackLicenseJson); if (postResponse != null && postResponse.IsSuccessStatusCode) { return; } var details = postResponse != null?GetResponseException(postResponse) : ""; throw new Exception($"The license that was posted was not accepted: {details}"); }
private static void GenerateCertGenConfigFiles(IEphemeralCluster <EphemeralClusterConfiguration> cluster, INodeFileSystem fileSystem, string silentModeConfigFile, EphemeralClusterConfiguration config) { var silentModeConfigFileDuplicate = Path.Combine(fileSystem.ConfigPath, "x-pack", "certgen") + ".yml"; var files = cluster.ClusterConfiguration.Version >= "6.3.0" ? new[] { silentModeConfigFile } : new[] { silentModeConfigFile, silentModeConfigFileDuplicate }; cluster.Writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} creating config files"); foreach (var file in files) { if (!File.Exists(file)) { File.WriteAllLines(file, new[] { "instances:", $" - name : \"{config.FileSystem.CertificateNodeName}\"", $" - name : \"{config.FileSystem.ClientCertificateName}\"", $" ip:", $" - \"127.0.0.1\"", $" dns:", $" - \"localhost\"", $" - \"ipv4.fiddler\"", $" filename : \"{config.FileSystem.ClientCertificateFilename}\"", }); } } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (cluster.CachingAndCachedHomeExists()) { return; } var config = cluster.ClusterConfiguration; if (config.Version < "6.2.0" || config.Version >= "6.3.0") { return; } var batFile = cluster.FileSystem.Binary; cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureElasticsearchBatWorksAcrossDrives)}}} patching {batFile} according to elastic/elasticsearch#29057"); var contents = File.ReadAllLines(batFile); for (var i = 0; i < contents.Length; i++) { if (contents[i] != "cd \"%ES_HOME%\"") { continue; } contents[i] = "cd /d \"%ES_HOME%\""; break; } File.WriteAllLines(batFile, contents); }
public void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster, bool nodeStarted) { var fs = cluster.FileSystem; var w = cluster.Writer; var v = cluster.ClusterConfiguration.Version; DeleteDirectory(w, "cluster data", fs.DataPath); DeleteDirectory(w, "cluster config", fs.ConfigPath); DeleteDirectory(w, "cluster logs", fs.LogsPath); DeleteDirectory(w, "repositories", fs.RepositoryPath); var efs = fs as EphemeralFileSystem; if (!string.IsNullOrWhiteSpace(efs?.TempFolder)) { DeleteDirectory(w, "cluster temp folder", efs.TempFolder); } if (efs != null) { var extractedFolder = Path.Combine(fs.LocalFolder, v.FolderInZip); if (extractedFolder != fs.ElasticsearchHome) { DeleteDirectory(w, "ephemeral ES_HOME", fs.ElasticsearchHome); } } //if the node did not start make sure we delete the cached folder as we can not assume its in a good state if (cluster.ClusterConfiguration.CacheEsHomeInstallation && !nodeStarted) { var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); DeleteDirectory(w, "cached installation - node failed to start", cachedEsHomeFolder); } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var keystoreFile = Path.Combine(cluster.FileSystem.ConfigPath, "elasticsearch.keystore"); var binary = Path.Combine(cluster.FileSystem.ElasticsearchHome, "bin", "elasticsearch-keystore" + BinarySuffix); if (!File.Exists(keystoreFile)) { ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, binary, "creating elasticsearch.keystore", "create"); } ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, binary, "add xpack.notification.slack.account.monitoring.secure_url to elasticsearch.keystore", input => input.WriteLine("https://hooks.slack.com/services/EXAMPLE/EXAMPLE/EXAMPLE"), "add", "xpack.notification.slack.account.monitoring.secure_url", "-xf"); ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, binary, "add xpack.notification.pagerduty.account.my_pagerduty_account.secure_service_api_key to elasticsearch.keystore", input => input.WriteLine("dummy_key"), "add", "xpack.notification.pagerduty.account.my_pagerduty_account.secure_service_api_key", "-xf"); cluster.Writer.WriteDiagnostic( $"{{{nameof(EnsureWatcherActionConfigurationSecretsInKeystore)}}} added watcher action secrets to elasticsearch.keystore"); }
private static void CopyConfigDirectoryToHomeCacheConfigDirectory(IEphemeralCluster <EphemeralClusterConfiguration> cluster, ElasticsearchPlugin plugin) { if (plugin.SubProductName == "x-pack") { return; } if (!cluster.ClusterConfiguration.CacheEsHomeInstallation) { return; } var fs = cluster.FileSystem; var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); var configTarget = Path.Combine(cachedEsHomeFolder, "config"); var configPluginPath = Path.Combine(fs.ConfigPath, plugin.SubProductName); var configPluginPathCached = Path.Combine(configTarget, plugin.SubProductName); if (!Directory.Exists(configPluginPath) || Directory.Exists(configPluginPathCached)) { return; } Directory.CreateDirectory(configPluginPathCached); CopyFolder(configPluginPath, configPluginPathCached); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var v = cluster.ClusterConfiguration.Version; //on 2.x we do not support tests requiring plugins for 2.x since we can not reliably install them if (v.Major == 2) { cluster.Writer?.WriteDiagnostic($"{{{nameof(InstallPlugins)}}} skipping install plugins on {{2.x}} version: [{v}]"); return; } var fs = cluster.FileSystem; var requiredPlugins = cluster.ClusterConfiguration.Plugins; var invalidPlugins = requiredPlugins.Where(p => !p.IsValid(v)).Select(p => p.Moniker).ToList(); if (invalidPlugins.Any()) { throw new CleanExitException($"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} "); } var plugins = from plugin in requiredPlugins let validForCurrentVersion = plugin.IsValid(v) let alreadyInstalled = AlreadyInstalled(fs, plugin.FolderName) where !alreadyInstalled && validForCurrentVersion select plugin; foreach (var plugin in plugins) { //var installParameter = v.ReleaseState == ReleaseState.Released ? plugin.Moniker : UseHttpPluginLocation(cluster.Writer, fs, plugin, v); var installParameter = UseHttpPluginLocation(cluster.Writer, fs, plugin, v); ExecuteBinary(cluster.ClusterConfiguration, cluster.Writer, "cmd", $"install elasticsearch plugin: {plugin.Moniker}", $"/c CALL {fs.PluginBinary} install --batch", installParameter); } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (!cluster.ClusterConfiguration.EnableSsl) { return; } var config = cluster.ClusterConfiguration; var v = config.Version; var file = v >= "6.3.0" ? Path.Combine(config.FileSystem.ConfigPath, "role_mapping") + ".yml" : Path.Combine(config.FileSystem.ConfigPath, "x-pack", "role_mapping") + ".yml"; var name = config.FileSystem.ClientCertificateName; if (!File.Exists(file) || !File.ReadAllLines(file).Any(f => f.Contains(name))) { File.WriteAllLines(file, new[] { "admin:", $" - \"{name}\"" }); } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { if (!cluster.ClusterConfiguration.EnableSecurity) { return; } var configFile = Path.Combine(cluster.FileSystem.ConfigPath, "elasticsearch.yml"); cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureSecurityRealms)}}} attempting to add xpack realms to [{configFile}]"); var lines = File.ReadAllLines(configFile).ToList(); var saveFile = false; var major = cluster.ClusterConfiguration.Version.Major; saveFile = major >= 6 ? (major >= 7 ? Write7XAndUpRealms(lines, cluster.ClusterConfiguration.EnableSsl) : Write6XAndUpRealms(lines)) : Write5XAndUpRealms(lines); if (saveFile) { File.WriteAllLines(configFile, lines); } cluster.Writer.WriteDiagnostic($"{{{nameof(EnsureSecurityRealms)}}} {(saveFile ? "saved" : "skipped saving")} xpack realms to [{configFile}]"); }
public static IElasticClient GetOrAddClient(this IEphemeralCluster cluster) => Clients.GetOrAdd(cluster, (c) => { var connectionPool = new StaticConnectionPool(c.NodesUris()); var settings = new ConnectionSettings(connectionPool); var client = new ElasticClient(settings); return(client); });
public static ConnectionSettings CreateConnectionSettings <TConfig>(this IEphemeralCluster <TConfig> cluster) where TConfig : EphemeralClusterConfiguration { var clusterNodes = cluster.NodesUris(TestConnectionSettings.LocalOrProxyHost); //we ignore the uri's that TestConnection provides and seed with the nodes the cluster dictates. return(new TestConnectionSettings(uris => new StaticConnectionPool(clusterNodes))); }
private static void GenerateCertificates(IEphemeralCluster <EphemeralClusterConfiguration> cluster, string silentModeConfigFile, IConsoleLineHandler writer) { var config = cluster.ClusterConfiguration; var name = config.FileSystem.CertificateFolderName; var path = config.FileSystem.CertificatesPath; NewOrCachedCertificates(cluster, name, path, silentModeConfigFile, writer); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { cluster.Writer.WriteDiagnostic($"{{{nameof(ValidateClusterStateTask)}}} waiting cluster to go into yellow health state"); var healthyResponse = this.Get(cluster, "_cluster/health", "wait_for_status=yellow&timeout=20s"); if (healthyResponse == null || healthyResponse.StatusCode != HttpStatusCode.OK) { throw new Exception($"Cluster health waiting for status yellow failed after 20s"); } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var v = cluster.ClusterConfiguration.Version; if (v.Major == 2) { cluster.Writer?.WriteDiagnostic( $"{{{nameof(ValidatePluginsTask)}}} skipping validate plugins on {{2.x}} version: [{v}]"); return; } if (v.Major < 7 && v.ArtifactBuildState == ArtifactBuildState.Snapshot) { cluster.Writer?.WriteDiagnostic( $"{{{nameof(InstallPlugins)}}} skipping validate SNAPSHOT plugins on < {{7.x}} version: [{v}]"); return; } var requestPlugins = cluster.ClusterConfiguration.Plugins .Where(p => p.IsValid(v)) .Where(p => !p.IsIncludedOutOfTheBox(v)) .Select(p => p.GetExistsMoniker(v)) .ToList(); if (!requestPlugins.Any()) { return; } cluster.Writer.WriteDiagnostic( $"{{{nameof(ValidatePluginsTask)}}} validating the cluster is running the requested plugins"); var catPlugins = Get(cluster, "_cat/plugins", "h=component"); if (catPlugins == null || !catPlugins.IsSuccessStatusCode) { throw new Exception( $"Calling _cat/plugins did not result in an OK response {GetResponseException(catPlugins)}"); } var installedPlugins = GetResponseString(catPlugins) .Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).ToList(); var missingPlugins = requestPlugins.Except(installedPlugins).ToList(); if (!missingPlugins.Any()) { return; } var missingString = string.Join(", ", missingPlugins); var pluginsString = string.Join(", ", installedPlugins); throw new Exception( $"Already running elasticsearch missed the following plugin(s): {missingString} currently installed: {pluginsString}."); }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var javaHome = Environment.GetEnvironmentVariable("JAVA_HOME"); if (string.IsNullOrWhiteSpace(javaHome)) { cluster.Writer?.WriteDiagnostic($"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} JAVA_HOME is not SET exiting.."); throw new Exception("The elasticsearch bat files are resillient to JAVA_HOME not being set, however the shield tooling is not"); } cluster.Writer?.WriteDiagnostic($"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} JAVA_HOME is set proceeding"); }
public void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster, bool nodeStarted) { var fs = cluster.FileSystem; var w = cluster.Writer; var v = cluster.ClusterConfiguration.Version; var a = cluster.ClusterConfiguration.Artifact; if (cluster.ClusterConfiguration.NoCleanupAfterNodeStopped) { w.WriteDiagnostic( $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} skipping cleanup as requested on cluster configuration"); return; } DeleteDirectory(w, "cluster data", fs.DataPath); DeleteDirectory(w, "cluster config", fs.ConfigPath); DeleteDirectory(w, "cluster logs", fs.LogsPath); DeleteDirectory(w, "repositories", fs.RepositoryPath); var efs = fs as EphemeralFileSystem; if (!string.IsNullOrWhiteSpace(efs?.TempFolder)) { DeleteDirectory(w, "cluster temp folder", efs.TempFolder); } if (efs != null) { var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); if (extractedFolder != fs.ElasticsearchHome) { DeleteDirectory(w, "ephemeral ES_HOME", fs.ElasticsearchHome); } //if the node was not started delete the cached extractedFolder if (!nodeStarted) { DeleteDirectory(w, "cached extracted folder - node failed to start", extractedFolder); } } //if the node did not start make sure we delete the cached folder as we can not assume its in a good state var cachedEsHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); if (cluster.ClusterConfiguration.CacheEsHomeInstallation && !nodeStarted) { DeleteDirectory(w, "cached installation - node failed to start", cachedEsHomeFolder); } else { w.WriteDiagnostic( $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} Leaving [cached folder] on disk: {{{cachedEsHomeFolder}}}"); } }
public override void Run(IEphemeralCluster <EphemeralClusterConfiguration> cluster) { var configFile = Path.Combine(cluster.FileSystem.ConfigPath, "elasticsearch.yml"); var lines = File.ReadAllLines(configFile).ToList(); var saveFile = false; var v = cluster.ClusterConfiguration.Version; var prefix = v.Major >= 5 ? "xpack.notification" : "watcher.actions"; var postfix = v.Major >= 5 ? string.Empty : ".service"; // set up for Watcher Slack action if (!lines.Any(line => line.StartsWith($"{prefix}.slack{postfix}:"))) { lines.AddRange(new[] { string.Empty, $"{prefix}.slack{postfix}:", " account:", " monitoring:", " message_defaults:", " from: x-pack", string.Empty }); saveFile = true; } // set up for Watcher PagerDuty action if (!lines.Any(line => line.StartsWith($"{prefix}.pagerduty{postfix}:"))) { lines.AddRange(new[] { string.Empty, $"{prefix}.pagerduty{postfix}:", " account:", " my_pagerduty_account:", " event_defaults:", " description: pager_duty", //" service_api_key: pager_duty_service_api_key", string.Empty }); saveFile = true; } if (saveFile) { File.WriteAllLines(configFile, lines); } cluster.Writer.WriteDiagnostic( $"{{{nameof(EnsureWatcherActionConfigurationInElasticsearchYaml)}}} {(saveFile ? "saved" : "skipped saving")} watcher config [{configFile}]"); }