public EnvironmentConfiguration()
{
//if env var NEST_INTEGRATION_VERSION is set assume integration mode
//used by the build script FAKE
var version = Environment.GetEnvironmentVariable("NEST_INTEGRATION_VERSION");
if (!string.IsNullOrEmpty(version))
{
Mode = TestMode.Integration;
}
this.ElasticsearchVersion = ElasticsearchVersion.From(string.IsNullOrWhiteSpace(version) ? DefaultVersion : version);
this.ClusterFilter = Environment.GetEnvironmentVariable("NEST_INTEGRATION_CLUSTER");
this.TestFilter = Environment.GetEnvironmentVariable("NEST_TEST_FILTER");
var newRandom = new Random().Next(1, 100000);
this.Seed = TryGetEnv("NEST_TEST_SEED", out var seed) ? int.Parse(seed) : newRandom;
var randomizer = new Randomizer(this.Seed);
this.Random = new RandomConfiguration
{
SourceSerializer = RandomBoolConfig("SOURCESERIALIZER", randomizer),
TypedKeys = RandomBoolConfig("TYPEDKEYS", randomizer),
#if FEATURE_HTTPWEBREQUEST
OldConnection = RandomBoolConfig("OLDCONNECTION", randomizer),
#else
OldConnection = false
#endif
};
}
public YamlConfiguration(string configurationFile)
{
if (!File.Exists(configurationFile))
{
return;
}
_config = File.ReadAllLines(configurationFile)
.Where(l => !l.Trim().StartsWith("#") && !string.IsNullOrWhiteSpace(l))
.ToDictionary(ConfigName, ConfigValue);
this.Mode = GetTestMode(_config["mode"]);
this.ElasticsearchVersion = ElasticsearchVersion.From(_config["elasticsearch_version"]);
this.ForceReseed = BoolConfig("force_reseed", false);
this.TestAgainstAlreadyRunningElasticsearch = BoolConfig("test_against_already_running_elasticsearch", false);
this.ShowElasticsearchOutputAfterStarted = BoolConfig("elasticsearch_out_after_started", false);
this.ClusterFilter = _config.ContainsKey("cluster_filter") ? _config["cluster_filter"] : null;
this.TestFilter = _config.ContainsKey("test_filter") ? _config["test_filter"] : null;
this.Seed = _config.TryGetValue("seed", out var seed) ? int.Parse(seed) : 1337;
var randomizer = new Randomizer(this.Seed);
this.Random = new RandomConfiguration
{
SourceSerializer = RandomBool("source_serializer", randomizer),
TypedKeys = RandomBool("typed_keys", randomizer),
OldConnection = RandomBool("old_connection", randomizer)
};
}
public YamlConfiguration(string configurationFile)
{
if (!File.Exists(configurationFile))
{
return;
}
_config = File.ReadAllLines(configurationFile)
.Where(l => !l.Trim().StartsWith("#") && !string.IsNullOrWhiteSpace(l))
.ToDictionary(ConfigName, ConfigValue);
this.Mode = GetTestMode(_config["mode"]);
this.ElasticsearchVersion = ElasticsearchVersion.GetOrAdd(_config["elasticsearch_version"]);
this.ForceReseed = bool.Parse(_config["force_reseed"]);
this.TestAgainstAlreadyRunningElasticsearch =
_config.TryGetValue("test_against_already_running_elasticsearch", out var tar) && bool.Parse(tar);
this.ClusterFilter = _config.ContainsKey("cluster_filter") ? _config["cluster_filter"] : null;
this.TestFilter = _config.ContainsKey("test_filter") ? _config["test_filter"] : null;
this.Seed = _config.TryGetValue("seed", out var seed) ? int.Parse(seed) : 1337;
Randomizer.Seed = new Random(this.Seed);
var randomizer = new Randomizer();
if (_config.TryGetValue("force_custom_source_serializer", out var v))
{
this.UsingCustomSourceSerializer = bool.Parse(v);
}
else
{
this.UsingCustomSourceSerializer = randomizer.Bool();
}
}
public ElasticsearchConsoleOut(ElasticsearchVersion version, bool error, string consoleLine) : base(error, consoleLine)
{
if (string.IsNullOrEmpty(consoleLine))
{
return;
}
var match = ConsoleLineParser.Match(consoleLine);
if (!match.Success)
{
return;
}
var dateString = match.Groups["date"].Value.Trim();
if (version.Major >= 5)
{
Date = DateTime.ParseExact(dateString, "yyyy-MM-ddTHH:mm:ss,fff", CultureInfo.CurrentCulture);
}
else
{
Date = DateTime.ParseExact(dateString, "yyyy-MM-dd HH:mm:ss,fff", CultureInfo.CurrentCulture);
}
Level = match.Groups["level"].Value.Trim();
Section = match.Groups["section"].Value.Trim().Replace("org.elasticsearch.", "");
Node = match.Groups["node"].Value.Trim();
Message = match.Groups["message"].Value.Trim();
}
public static int Main()
{
ElasticsearchVersion v = null;
// var clusterConfiguration = new EphemeralClusterConfiguration("6.0.0", numberOfNodes: 2);
// var ephemeralCluster = new EphemeralCluster(clusterConfiguration);
// var nodeConfiguration = new NodeConfiguration(clusterConfiguration, 9200);
// var elasticsearchNode = new ElasticsearchNode(nodeConfiguration);
////
// using (var node = new ElasticsearchNode("5.5.1"))
// {
// node.Subscribe(new ConsoleOutColorWriter());
// node.WaitForStarted(TimeSpan.FromMinutes(2));
// }
//
// using (var node = new ElasticsearchNode("6.0.0-beta2", @"c:\Data\elasticsearch-6.0.0-beta2"))
// {
// node.Subscribe();
// node.WaitForStarted(TimeSpan.FromMinutes(2));
// Console.ReadKey();
// }
// using (var cluster = new EphemeralCluster("6.0.0"))
// {
// cluster.Start();
// }
var config = new EphemeralClusterConfiguration("6.2.3", XPack | Security | SSL, null, 1)
{
PrintYamlFilesInConfigFolder = true,
ShowElasticsearchOutputAfterStarted = true
};
using (var cluster = new EphemeralCluster(config))
{
cluster.Start();
var nodes = cluster.NodesUris();
var connectionPool = new StaticConnectionPool(nodes);
var settings = new ConnectionSettings(connectionPool).EnableDebugMode();
if (config.EnableSecurity && !config.EnableSsl)
{
settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password);
}
if (config.EnableSsl)
{
settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
settings = settings.ClientCertificate(new X509Certificate2(config.FileSystem.ClientCertificate));
}
var client = new ElasticClient(settings);
Console.Write(client.XPackInfo().DebugInformation);
}
//
// Console.WriteLine($".. DONE ...");
return(0);
}
public NodeFileSystem(ElasticsearchVersion version, string elasticsearchHome = null)
{
this.Version = version;
this.LocalFolder = AppDataFolder(version);
this.ElasticsearchHome = elasticsearchHome ?? GetEsHomeVariable() ?? throw new ArgumentNullException(nameof(elasticsearchHome));
this.ConfigEnvironmentVariableName = version.Major >= 6 ? "ES_PATH_CONF" : "CONF_DIR";
}
public XunitClusterConfiguration(
ElasticsearchVersion version,
ClusterFeatures features = ClusterFeatures.None,
ElasticsearchPlugins plugins = null,
int numberOfNodes = 1)
: base(version, features, plugins, numberOfNodes)
{
this.AdditionalAfterStartedTasks.Add(new PrintXunitAfterStartedTask());
}
public NodeFileSystem(ElasticsearchVersion version, string clusterName, string nodeName)
{
this._version = version;
this._clusterName = clusterName;
var appData = GetApplicationDataDirectory() ?? "/tmp/NEST";
this.RoamingFolder = Path.Combine(appData, "NEST", this._version.FullyQualifiedVersion);
this.ElasticsearchHome = Path.Combine(this.RoamingFolder, this._version.FolderInZip);
}
private void AddClientCertificateUser(NodeFileSystem fileSystem, ElasticsearchVersion version)
{
var file = Path.Combine(fileSystem.ConfigPath, "x-pack", "role_mapping") + ".yml";
var name = fileSystem.ClientCertificateName;
if (!File.Exists(file) || !File.ReadAllLines(file).Any(f => f.Contains(name)))
{
File.WriteAllLines(file, new []
{
"admin:",
$" - \"{name}\""
});
}
}
private void HandleConsoleMessage(ElasticsearchConsoleOut consoleOut, XplatManualResetEvent handle)
{
//no need to snoop for metadata if we already started
if (!this._config.RunIntegrationTests || this.Started)
{
return;
}
//if we are running on CI and not started dump elasticsearch stdout/err
//before the started notification to help debug failures to start
if (this.RunningOnCI && !this.Started)
{
if (consoleOut.Error)
{
Console.Error.WriteLine(consoleOut.Data);
}
else
{
Console.WriteLine(consoleOut.Data);
}
}
if (consoleOut.Error && !this.Started && !string.IsNullOrWhiteSpace(consoleOut.Data))
{
throw new Exception(consoleOut.Data);
}
string version;
int? pid;
int port;
if (this.ProcessId == null && consoleOut.TryParseNodeInfo(out version, out pid))
{
var startedVersion = ElasticsearchVersion.GetOrAdd(version);
this.ProcessId = pid;
if (this.Version != startedVersion)
{
throw new Exception($"Booted elasticsearch is version {startedVersion} but the test config dictates {this.Version}");
}
}
else if (consoleOut.TryGetPortNumber(out port))
{
this.Port = port;
}
else if (consoleOut.TryGetStartedConfirmation())
{
this.Started = true;
handle.Set();
}
}
public static bool VersionSatisfiedBy(string range, ElasticsearchVersion version)
{
var versionRange = new SemVer.Range(range);
var satisfied = versionRange.IsSatisfied(version.Version);
if (!version.IsSnapshot || satisfied)
{
return(satisfied);
}
//Semver can only match snapshot version with ranges on the same major and minor
//anything else fails but we want to know e.g 2.4.5-SNAPSHOT satisfied by <5.0.0;
var wholeVersion = $"{version.Major}.{version.Minor}.{version.Patch}";
return(versionRange.IsSatisfied(wholeVersion));
}
public EnvironmentConfiguration()
{
//if env var NEST_INTEGRATION_VERSION is set assume integration mode
//used by the build script FAKE
var version = Environment.GetEnvironmentVariable("NEST_INTEGRATION_VERSION");
if (!string.IsNullOrEmpty(version))
{
Mode = TestMode.Integration;
}
this.ElasticsearchVersion = ElasticsearchVersion.GetOrAdd(string.IsNullOrWhiteSpace(version) ? "5.0.0" : version);
this.ClusterFilter = Environment.GetEnvironmentVariable("NEST_INTEGRATION_CLUSTER");
this.TestFilter = Environment.GetEnvironmentVariable("NEST_TEST_FILTER");
}
public EphemeralClusterConfiguration(ElasticsearchVersion version, ClusterFeatures features = ClusterFeatures.None, ElasticsearchPlugins plugins = null, int numberOfNodes = 1)
: base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName)
{
this.Features = features;
var pluginsList = plugins?.ToList() ?? new List <ElasticsearchPlugin>();
if (this.Features.HasFlag(ClusterFeatures.XPack) && !pluginsList.Any(p => p.Moniker == "x-pack"))
{
pluginsList.Add(ElasticsearchPlugin.XPack);
}
this.Plugins = new ElasticsearchPlugins(pluginsList);
AddDefaultXPackSettings();
}
public NodeConfiguration(ITestConfiguration configuration, ClusterBase cluster)
{
this._cluster = cluster;
this.EnableSsl = cluster.SkipValidation;
this.RequiredPlugins = ClusterRequiredPlugins(cluster);
this.Mode = configuration.Mode;
var v = configuration.ElasticsearchVersion;
this.ElasticsearchVersion = v;
this.ForceReseed = configuration.ForceReseed;
this.TestAgainstAlreadyRunningElasticsearch = configuration.TestAgainstAlreadyRunningElasticsearch;
this.RunIntegrationTests = configuration.RunIntegrationTests;
this.RunUnitTests = configuration.RunUnitTests;
this.ClusterFilter = configuration.ClusterFilter;
this.TestFilter = configuration.TestFilter;
this.FileSystem = new NodeFileSystem(configuration.ElasticsearchVersion, this.ClusterName, this.NodeName);
this.DesiredPort = cluster.DesiredPort;
var attr = v.Major >= 5 ? "attr." : "";
var indexedOrStored = v > ElasticsearchVersion.GetOrAdd("5.0.0-alpha1") ? "stored" : "indexed";
var shieldOrSecurity = v > ElasticsearchVersion.GetOrAdd("5.0.0-alpha1") ? "xpack.security" : "shield";
var es = v > ElasticsearchVersion.GetOrAdd("5.0.0-alpha2") ? "" : "es.";
var b = this.XPackEnabled.ToString().ToLowerInvariant();
var sslEnabled = this.EnableSsl.ToString().ToLowerInvariant();
this.DefaultNodeSettings = new List <string>
{
$"{es}cluster.name={this.ClusterName}",
$"{es}node.name={this.NodeName}",
$"{es}path.repo={this.FileSystem.RepositoryPath}",
$"{es}path.data={this.FileSystem.DataPath}",
$"{es}http.port={this.DesiredPort}",
$"{es}script.inline=true",
$"{es}script.max_compilations_per_minute=10000",
$"{es}script.{indexedOrStored}=true",
$"{es}node.{attr}testingcluster=true",
$"{es}node.{attr}gateway=true",
$"{es}{shieldOrSecurity}.enabled={b}",
$"{es}{shieldOrSecurity}.http.ssl.enabled={sslEnabled}",
$"{es}{shieldOrSecurity}.authc.realms.pki1.enabled={sslEnabled}",
$"{es}search.remote.connect=true"
};
}
public override void Validate(IElasticClient client, NodeConfiguration configuration)
{
var alreadyUp = client.RootNodeInfo();
if (!alreadyUp.IsValid)
{
return;
}
var v = configuration.ElasticsearchVersion;
var alreadyUpVersion = new ElasticsearchVersion(alreadyUp.Version.Number);
var alreadyUpSnapshotVersion = new ElasticsearchVersion(alreadyUp.Version.Number + "-SNAPSHOT");
if (v != alreadyUpVersion && v != alreadyUpSnapshotVersion)
{
throw new Exception($"running elasticsearch is version {alreadyUpVersion} but the test config dictates {v}");
}
}
public YamlConfiguration(string configurationFile)
{
if (!File.Exists(configurationFile))
{
return;
}
var config = File.ReadAllLines(configurationFile)
.Where(l => !l.Trim().StartsWith("#") && !string.IsNullOrWhiteSpace(l))
.ToDictionary(ConfigName, ConfigValue);
this.Mode = GetTestMode(config["mode"]);
this.ElasticsearchVersion = ElasticsearchVersion.GetOrAdd(config["elasticsearch_version"]);
this.ForceReseed = bool.Parse(config["force_reseed"]);
this.TestAgainstAlreadyRunningElasticsearch = bool.Parse(config["test_against_already_running_elasticsearch"]);
this.ClusterFilter = config.ContainsKey("cluster_filter") ? config["cluster_filter"] : null;
this.TestFilter = config.ContainsKey("test_filter") ? config["test_filter"] : null;
}
private bool ValidateRunningVersion(IElasticClient client, XplatManualResetEvent handle)
{
var alreadyUp = client.RootNodeInfo();
if (!alreadyUp.IsValid)
{
return(false);
}
var alreadyUpVersion = new ElasticsearchVersion(alreadyUp.Version.Number);
var alreadyUpSnapshotVersion = new ElasticsearchVersion(alreadyUp.Version.Number + "-SNAPSHOT");
if (this.Version == alreadyUpVersion || this.Version == alreadyUpSnapshotVersion)
{
return(true);
}
var e = new Exception($"running elasticsearch is version {alreadyUpVersion} but the test config dictates {this.Version}");
this.Fatal(handle, e);
return(false);
}
protected ClusterBase(ElasticsearchVersion version, int instanceCount = 1, string clusterName = null, string[] additionalSettings = null)
{
var n = new NodeConfiguration(version, clusterName, null);
this.NodeConfiguration = n;
var nodeSettings = n.CreateSettings(additionalSettings);
this.NodeSettings = nodeSettings;
var fs = this.NodeConfiguration.FileSystem;
var nodes = Enumerable.Range(9200, instanceCount)
.Select(p => new ElasticsearchNode(fs.Binary, n.CreateSettings(nodeSettings, NodeSpecificSettings(p)))
{
DesiredPort = p,
AssumeStartedOnNotEnoughMasterPing = instanceCount > 1
})
.ToList();
this.Nodes = new ReadOnlyCollection <ElasticsearchNode>(nodes);
this.TaskRunner = new NodeTaskRunner(this.NodeConfiguration);
}
private static ClientTestClusterConfiguration CreateConfiguration()
{
var plugins = new List <ElasticsearchPlugin>
{
IngestGeoIp,
IngestAttachment,
AnalysisKuromoji,
AnalysisIcu,
AnalysisPhonetic,
MapperMurmur3,
};
// TODO: temporary until https://github.com/elastic/elasticsearch-net-abstractions/commit/3977ccb6449870fb4f1e6059be960e12ec5e5125 is released
if (ElasticsearchVersion.From(TestClient.Configuration.ElasticsearchVersion) >= "6.4.0")
{
plugins.Add(new ElasticsearchPlugin("analysis-nori", v => v >= "6.4.0"));
}
return(new ClientTestClusterConfiguration(plugins.ToArray())
{
MaxConcurrency = 4
});
}
private void HandleConsoleMessage(ElasticsearchConsoleOut consoleOut, XplatManualResetEvent handle)
{
if (consoleOut.Error && !this.Started)
{
this.Fatal(handle, new Exception(consoleOut.Data));
return;
}
//no need to snoop for metadata if we already started
if (!this._config.RunIntegrationTests || this.Started)
{
return;
}
string version; int?pid; int port;
if (this.ProcessId == null && consoleOut.TryParseNodeInfo(out version, out pid))
{
var startedVersion = new ElasticsearchVersion(version);
this.ProcessId = pid;
if (this.Version != startedVersion)
{
this.Fatal(handle, new Exception($"Booted elasticsearch is version {startedVersion} but the test config dictates {this.Version}"));
}
}
else if (consoleOut.TryGetPortNumber(out port))
{
this.Port = port;
}
else if (consoleOut.TryGetStartedConfirmation())
{
var client = this.GetPrivateClient(null, false, this.Port);
this.ValidatePlugins(client, handle);
this.ValidateLicense(client, handle);
this.WaitForClusterBootstrap(client, handle, alreadyRunningInstance: false);
}
}
private void HandleConsoleMessage(ElasticsearchConsoleOut consoleOut, XplatManualResetEvent handle)
{
Console.WriteLine(consoleOut.Data);
//no need to snoop for metadata if we already started
if (!this._config.RunIntegrationTests || this.Started)
{
return;
}
if (consoleOut.Error && !this.Started && !string.IsNullOrWhiteSpace(consoleOut.Data))
{
throw new Exception("Error out:" + consoleOut.Data);
}
string version;
int? pid;
int port;
if (this.ProcessId == null && consoleOut.TryParseNodeInfo(out version, out pid))
{
var startedVersion = new ElasticsearchVersion(version);
this.ProcessId = pid;
if (this.Version != startedVersion)
{
throw new Exception($"Booted elasticsearch is version {startedVersion} but the test config dictates {this.Version}");
}
}
else if (consoleOut.TryGetPortNumber(out port))
{
this.Port = port;
}
else if (consoleOut.TryGetStartedConfirmation())
{
this.Started = true;
handle.Set();
}
}
public EnvironmentConfiguration()
{
//if env var NEST_INTEGRATION_VERSION is set assume integration mode
//used by the build script FAKE
var version = Environment.GetEnvironmentVariable("NEST_INTEGRATION_VERSION");
if (!string.IsNullOrEmpty(version))
{
Mode = TestMode.Integration;
}
this.ElasticsearchVersion = ElasticsearchVersion.GetOrAdd(string.IsNullOrWhiteSpace(version) ? DefaultVersion : version);
this.ClusterFilter = Environment.GetEnvironmentVariable("NEST_INTEGRATION_CLUSTER");
this.TestFilter = Environment.GetEnvironmentVariable("NEST_TEST_FILTER");
var newRandom = new Random().Next(1, 100000);
this.Seed = TryGetEnv("NEST_TEST_SEED", out var seed) ? int.Parse(seed) : newRandom;
Randomizer.Seed = new Random(this.Seed);
var randomizer = new Randomizer();
this.UsingCustomSourceSerializer = (TryGetEnv("NEST_SOURCE_SERIALIZER", out var source) && bool.Parse(source)) ||
randomizer.Bool();
}
public EphemeralCluster(ElasticsearchVersion version, int numberOfNodes = 1)
: base(new EphemeralClusterConfiguration(version, ClusterFeatures.None, numberOfNodes: numberOfNodes))
{
}
public virtual string DownloadUrl(ElasticsearchVersion version) => version.DownloadLocations.PluginDownloadUrl(this.Moniker);
public bool IsValid(ElasticsearchVersion version) => _isValid(version);
/// <summary> what name to check under /_cat/plugins for the plugins existense</summary> public string ListedPluginName(ElasticsearchVersion version) => _getListedPluginName(version);
public ElasticsearchVersion Create(string version) => ElasticsearchVersion.Create(version, new MockElasticsearchVersionResolver());
private void DownloadPluginSnapshot(string downloadLocation, ElasticsearchPluginConfiguration plugin, ElasticsearchVersion v)
{
if (File.Exists(downloadLocation))
{
return;
}
var downloadUrl = plugin.SnapshotDownloadUrl(v);
Console.WriteLine($"Download plugin snapshot {plugin.Moniker}: {downloadUrl}");
try
{
this.DownloadFile(downloadUrl, downloadLocation);
Console.WriteLine($"Downloaded plugin snapshot {plugin.Moniker}");
}
catch (Exception)
{
Console.WriteLine($"Failed downloading plugin snapshot {plugin.Moniker}");
}
}
private string DownloadSnapshotIfNeeded(NodeFileSystem fileSystem, ElasticsearchPluginConfiguration plugin, ElasticsearchVersion v)
{
var downloadLocation = Path.Combine(fileSystem.RoamingFolder, plugin.SnapshotZip(v));
this.DownloadPluginSnapshot(downloadLocation, plugin, v);
//transform downloadLocation to file uri and use that to install from
return(new Uri(downloadLocation).AbsoluteUri);
}
public EphemeralClusterConfiguration(ElasticsearchVersion version, ElasticsearchPlugins plugins = null, int numberOfNodes = 1)
: this(version, ClusterFeatures.None, plugins, numberOfNodes)
{
}
