/// <summary>
/// Validates the DNS endpoint. Any warning/errors will be appended to <paramref name="warnings"/>.
/// </summary>
/// <param name="warnings">Any warnings will be appended here.</param>
/// <param name="hiveDefinition">The current hive definition,</param>
/// <param name="nodeGroups">The hive node groups.</param>
/// <param name="entryHostname">The parent <see cref="DnsEntry"/>'s hostname.</param>
public void Validate(List <string> warnings, HiveDefinition hiveDefinition, Dictionary <string, List <NodeDefinition> > nodeGroups, string entryHostname)
{
Covenant.Requires <ArgumentException>(hiveDefinition != null);
Covenant.Requires <ArgumentException>(nodeGroups != null);
if (string.IsNullOrEmpty(Target))
{
warnings.Add($"Invalid [{nameof(DnsEndpoint)}.{nameof(Target)}={Target}] for [{nameof(DnsEntry)}={entryHostname}].");
}
var groupName = GetGroupName();
if (groupName != null)
{
if (string.IsNullOrEmpty(groupName))
{
warnings.Add($"Invalid [{nameof(DnsEndpoint)}.{nameof(Target)}={Target}] for [{nameof(DnsEntry)}={entryHostname}].");
}
else if (!nodeGroups.ContainsKey(groupName))
{
warnings.Add($"Node group [{groupName}] not found for [{nameof(DnsEntry)}={entryHostname}].");
}
}
else
{
if (!IPAddress.TryParse(Target, out var address) && !HiveDefinition.DnsHostRegex.IsMatch(Target))
{
warnings.Add($"Invalid [{nameof(DnsEndpoint)}.{nameof(Target)}={Target}] is not a valid IP address or DNS hostname for [{nameof(DnsEntry)}={entryHostname}].");
}
}
}
public void Validate(HiveDefinition hiveDefinition)
{
if (Name == AzureCloudEnvironments.Custom)
{
Uri uri;
if (string.IsNullOrEmpty(AuthenticationEndpoint) || !Uri.TryCreate(AuthenticationEndpoint, UriKind.Absolute, out uri))
{
throw new HiveDefinitionException($"Invalid Azure environment [{nameof(AuthenticationEndpoint)}={AuthenticationEndpoint}].");
}
if (string.IsNullOrEmpty(ResourceManagerEndpoint) || !Uri.TryCreate(ResourceManagerEndpoint, UriKind.Absolute, out uri))
{
throw new HiveDefinitionException($"Invalid Azure environment [{nameof(ResourceManagerEndpoint)}={ResourceManagerEndpoint}].");
}
if (string.IsNullOrEmpty(GraphEndpoint) || !Uri.TryCreate(GraphEndpoint, UriKind.Absolute, out uri))
{
throw new HiveDefinitionException($"Invalid Azure environment [{nameof(GraphEndpoint)}={GraphEndpoint}].");
}
if (string.IsNullOrEmpty(AuthenticationEndpoint) || !Uri.TryCreate(AuthenticationEndpoint, UriKind.Absolute, out uri))
{
throw new HiveDefinitionException($"Invalid Azure environment [{nameof(ManagementEnpoint)}={ManagementEnpoint}].");
}
}
}
public void Validate(HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
if (!System.Version.TryParse(Version, out var version))
{
throw new HiveDefinitionException($"Invalid version [{nameof(ConsulOptions)}.{nameof(Version)}={Version}].");
}
if (version < minVersion)
{
throw new HiveDefinitionException($"Minumim acceptable [{nameof(ConsulOptions)}.{nameof(Version)}={minVersion}].");
}
if (string.IsNullOrEmpty(EncryptionKey))
{
EncryptionKey = Convert.ToBase64String(NeonHelper.RandBytes(16));
}
if (DnsTTL < 0)
{
throw new HiveDefinitionException($"[{nameof(ConsulOptions)}.{nameof(DnsTTL)}={DnsTTL}] is not valid.");
}
if (DnsMaxStale < 0)
{
throw new HiveDefinitionException($"[{nameof(ConsulOptions)}.{nameof(DnsMaxStale)}={DnsMaxStale}] is not valid.");
}
HiveDefinition.VerifyEncryptionKey(EncryptionKey);
}
/// <summary>
/// Validates the options and also ensures that all <c>null</c> properties are
/// initialized to their default values.
/// </summary>
/// <param name="hiveDefinition">The hive definition.</param>
/// <exception cref="HiveDefinitionException">Thrown if the definition is not valid.</exception>
public void Validate(HiveDefinition hiveDefinition)
{
if (StepStaggerSeconds < 0)
{
throw new HiveDefinitionException($"[{nameof(SetupOptions)}.{nameof(StepStaggerSeconds)}={StepStaggerSeconds}] cannot be negative.");
}
}
/// <summary>
/// Creates or updates a hive Docker string config.
/// </summary>
/// <param name="configName">The config name.</param>
/// <param name="value">The config value.</param>
/// <param name="options">Optional command run options.</param>
/// <exception cref="HiveException">Thrown if the operation failed.</exception>
public void Set(string configName, string value, RunOptions options = RunOptions.None)
{
Covenant.Requires <ArgumentException>(HiveDefinition.IsValidName(configName));
Covenant.Requires <ArgumentNullException>(value != null);
Set(configName, Encoding.UTF8.GetBytes(value), options);
}
/// <summary>
/// Returns the Consul key for the named global.
/// </summary>
/// <param name="name"></param>
/// <returns>The Consul key path.</returns>
public string GetKey(string name)
{
Covenant.Requires(!string.IsNullOrEmpty(name));
Covenant.Requires(HiveDefinition.IsValidName(name));
return($"{HiveConst.GlobalKey}/{name}");
}
public void Validate(HiveDefinition hiveDefinition)
{
if (PasswordLength > 0 && PasswordLength < 8)
{
throw new HiveDefinitionException($"[{nameof(HiveNodeOptions)}.{nameof(PasswordLength)}={PasswordLength}] is not zero and is less than the minimum [8].");
}
}
/// <summary>
/// Removes a hive certificate if it exists.
/// </summary>
/// <param name="name">The certificate name.</param>
public void Remove(string name)
{
Covenant.Requires <ArgumentException>(HiveDefinition.IsValidName(name));
hive.Vault.Client.DeleteAsync(HiveHelper.GetVaultCertificateKey(name)).Wait();
hive.SignalTrafficManagerUpdate();
}
public void Validate(HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
if (!Enabled)
{
return;
}
var esNodeCount = hiveDefinition.Nodes.Count(n => n.Labels.LogEsData);
if (esNodeCount == 0)
{
throw new HiveDefinitionException($"Invalid Log Configuration: At least one node must be labeled with [{NodeLabels.LabelLogEsData}=true].");
}
if (RetentionDays <= 0)
{
throw new HiveDefinitionException($"Invalid [{nameof(LogOptions)}.{nameof(RetentionDays)}={RetentionDays}]: This must be >= 0.");
}
if (!NeonHelper.TryParseCount(EsMemory, out var esMemoryBytes))
{
throw new HiveDefinitionException($"Invalid [{nameof(LogOptions)}.{nameof(EsMemory)}={EsMemory}].");
}
if (esMemoryBytes < 1.5 * NeonHelper.Giga)
{
throw new HiveDefinitionException($"[{nameof(LogOptions)}.{nameof(EsMemory)}={EsMemory}] cannot be less than [1.5GB].");
}
}
/// <summary>
/// Attempts to retrieve a named hive global setting as a <typeparamref name="T"/>
/// deserialized from JSON.
/// </summary>
/// <typeparam name="T">The type being deserialized.</typeparam>
/// <param name="name">The setting name.</param>
/// <param name="output">Returns as the setting value.</param>
/// <param name="strict">Optionally require that all input properties map to <typeparamref name="T"/> properties.</param>
/// <returns><c>true</c> if the setting exists and was returned.</returns>
/// <exception cref="FormatException">Thrown if the setting value could not be parsed.</exception>
/// <remarks>
/// <note>
/// Well known hive setting names are defined in <see cref="HiveGlobals"/>.
/// </note>
/// </remarks>
public bool TryGet <T>(string name, out T output, bool strict = false)
{
Covenant.Requires(!string.IsNullOrEmpty(name));
Covenant.Requires(HiveDefinition.IsValidName(name));
output = default(T);
var key = $"{HiveConst.GlobalKey}/{name}";
var value = hive.Consul.Client.KV.GetStringOrDefault(key).Result;
if (value == null)
{
return(false);
}
try
{
output = NeonHelper.JsonDeserialize <T>(value, strict);
}
catch
{
return(false);
}
return(true);
}
public void Validate(HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
HostXvaUri = HostXvaUri ?? defaultHostXvaUri;
TemplateName = TemplateName ?? defaultTemplate;
StorageRepository = StorageRepository ?? defaultStorageRepository;
OsdStorageRepository = OsdStorageRepository ?? defaultStorageRepository;
if (!hiveDefinition.Network.StaticIP)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(NetworkOptions.StaticIP)}] must be [true] when deploying to XenServer.");
}
if (string.IsNullOrEmpty(HostXvaUri) || !Uri.TryCreate(HostXvaUri, UriKind.Absolute, out Uri uri))
{
throw new HiveDefinitionException($"[{nameof(XenServerOptions)}.{nameof(HostXvaUri)}] is required when deploying to XenServer.");
}
if (string.IsNullOrEmpty(StorageRepository))
{
throw new HiveDefinitionException($"[{nameof(XenServerOptions)}.{nameof(StorageRepository)}] is required when deploying to XenServer.");
}
if (string.IsNullOrEmpty(OsdStorageRepository))
{
OsdStorageRepository = StorageRepository;
}
hiveDefinition.ValidatePrivateNodeAddresses(); // Private node IP addresses must be assigned and valid.
hiveDefinition.Hosting.ValidateHypervisor(hiveDefinition, remoteHypervisors: true); // Hypervisor options must be valid.
}
public void Validate(HiveDefinition hiveDefinition, string nodeName)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
var caps = AzureVmCapabilities.Get(VmSize);
if (!caps.LoadBalancing)
{
throw new HiveDefinitionException($"Hive node [{nodeName}] configures [{nameof(VmSize)}={VmSize}] which does not support load balancing and cannot be used for a neonHIVE.");
}
if (!caps.SupportsDataStorageType(StorageType))
{
throw new HiveDefinitionException($"Hive node [{nodeName}] configures [{nameof(VmSize)}={VmSize}] which does not support [{StorageType}] managed data drives.");
}
if (HardDriveCount > 1)
{
throw new HiveDefinitionException($"Hive node [{nodeName}] configures [{nameof(HardDriveCount)}={HardDriveCount}] managed data drives. Only zero or one managed drive is currently supported.");
}
if (caps.MaxDataDrives < HardDriveCount)
{
throw new HiveDefinitionException($"Hive node [{nodeName}]configures [{nameof(HardDriveCount)}={HardDriveCount}] managed data drives. Only up to [{caps.MaxDataDrives}] drives are allowed.");
}
AzureHelper.GetDiskSizeGB(StorageType, HardDriveSizeGB);
}
/// <summary>
/// Creates or updates a hive Docker object secret serialized as JSON.
/// </summary>
/// <typeparam name="T">The secret type.</typeparam>
/// <param name="secretName">The secret name.</param>
/// <param name="value">The secret value.</param>
/// <param name="options">Optional command run options.</param>
/// <exception cref="HiveException">Thrown if the operation failed.</exception>
public void Set <T>(string secretName, T value, RunOptions options = RunOptions.None)
where T : class, new()
{
Covenant.Requires <ArgumentException>(HiveDefinition.IsValidName(secretName));
Covenant.Requires <ArgumentNullException>(value != null);
Set(secretName, NeonHelper.JsonSerialize(value, Formatting.Indented), options);
}
/// <summary>
/// Adds or updates a hive certificate.
/// </summary>
/// <param name="name">The certificate name.</param>
/// <param name="certificate">The certificate.</param>
/// <exception cref="ArgumentException">Thrown if the certificate is not valid.</exception>
/// <remarks>
/// <note>
/// The <paramref name="certificate"/> must be fully parsed (it's
/// <see cref="TlsCertificate.Parse()"/> method must have been called at
/// some point to load the <see cref="TlsCertificate.Hosts"/>,
/// <see cref="TlsCertificate.ValidFrom"/> and <see cref="TlsCertificate.ValidUntil"/>
/// properties).
/// </note>
/// </remarks>
public void Set(string name, TlsCertificate certificate)
{
Covenant.Requires <ArgumentException>(HiveDefinition.IsValidName(name));
Covenant.Requires <ArgumentNullException>(certificate != null);
hive.Vault.Client.WriteJsonAsync(HiveHelper.GetVaultCertificateKey(name), certificate).Wait();
hive.SignalTrafficManagerUpdate();
}
/// <inheritdoc/>
public override void Validate(HiveDefinition hiveDefinition)
{
// Identify the OSD Bluestore block device for OSD nodes.
if (hive.Definition.HiveFS.Enabled)
{
throw new NotImplementedException("$todo(jeff.lill): Implement this.");
}
}
/// <summary>
/// Returns the maximum number of bytes to disk allocate to for this node when
/// hosted on a hypervisor.
/// </summary>
/// <param name="hiveDefinition">The hive definition.</param>
/// <returns>The size in bytes.</returns>
public long GetVmDisk(HiveDefinition hiveDefinition)
{
if (!string.IsNullOrEmpty(VmDisk))
{
return(HiveDefinition.ValidateSize(VmDisk, this.GetType(), nameof(VmDisk)));
}
else
{
return(HiveDefinition.ValidateSize(hiveDefinition.Hosting.VmDisk, hiveDefinition.Hosting.GetType(), nameof(hiveDefinition.Hosting.VmDisk)));
}
}
/// <summary>
/// Returns the maximum number processors to allocate for this node when
/// hosted on a hypervisor.
/// </summary>
/// <param name="hiveDefinition">The hive definition.</param>
/// <returns>The number of cores.</returns>
public int GetVmProcessors(HiveDefinition hiveDefinition)
{
if (VmProcessors != 0)
{
return(VmProcessors);
}
else
{
return(hiveDefinition.Hosting.VmProcessors);
}
}
public void Validate(HiveDefinition hiveDefinition)
{
if (!NetHelper.IsValidPort(FrontendPort))
{
throw new HiveDefinitionException($"[{nameof(HostedEndpoint)}.{nameof(FrontendPort)}] value [{FrontendPort}] is outside the range of a valid network port.");
}
if (!NetHelper.IsValidPort(FrontendPort))
{
throw new HiveDefinitionException($"[{nameof(HostedEndpoint)}.{nameof(BackendPort)}] value [{BackendPort}] is outside the range of a valid network port.");
}
}
/// <inheritdoc/>
public override void Validate(HiveDefinition hiveDefinition)
{
// Identify the OSD Bluestore block device for OSD nodes.
if (hive.Definition.HiveFS.Enabled)
{
foreach (var node in hive.Definition.Nodes.Where(n => n.Labels.CephOSD))
{
node.Labels.CephOSDDevice = "/dev/sdb";
}
}
}
/// <summary>
/// <b>INTERNAL USE ONLY:</b> Indicates whether the Docker ingress network should be used
/// for traffic manager instances based on <see cref="AvoidIngressNetwork"/> and the current
/// hosting environment.
/// </summary>
/// <param name="hiveDefinition">The current hive definition.</param>
public bool GetAvoidIngressNetwork(HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
if (AvoidIngressNetwork.HasValue)
{
return(AvoidIngressNetwork.Value);
}
else
{
return(false);
}
}
public void Validate(HiveDefinition hiveDefinition, string parentOptionName)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
parentOptionName = parentOptionName ?? string.Empty;
if (IsSecured)
{
if (!File.Exists(Path))
{
throw new FileNotFoundException($"[{parentOptionName}] TLS certificate file [{Path}] does not exist.");
}
}
}
/// <inheritdoc/>
public override void Validate(HiveDefinition hiveDefinition)
{
// Ensure that the OSD Bluestore block device have been specified.
if (hive.Definition.HiveFS.Enabled)
{
foreach (var node in hive.Definition.Nodes.Where(n => n.Labels.CephOSD))
{
if (string.IsNullOrEmpty(node.Labels.CephOSDDevice))
{
throw new HiveDefinitionException($"Hosting environment [{HostingEnvironments.Machine}] requires that the OSD node [{node.Name}] explicitly set [{nameof(NodeLabels)}.{nameof(NodeLabels.CephOSDDevice)}] to the target OSD block device (like: [/dev/sdb]).");
}
}
}
}
/// <summary>
/// Returns a traffic manager rule if it exists.
/// </summary>
/// <param name="ruleName">The rule name.</param>
/// <returns>The <see cref="TrafficRule"/> or <c>null</c>.</returns>
public TrafficRule GetRule(string ruleName)
{
Covenant.Requires <ArgumentException>(HiveDefinition.IsValidName(ruleName));
var ruleKey = GetProxyRuleKey(ruleName);
if (hive.Consul.Client.KV.Exists(ruleKey).Result)
{
return(TrafficRule.ParseJson(hive.Consul.Client.KV.GetString(ruleKey).Result));
}
else
{
return(null);
}
}
/// <summary>
/// Returns the prefix to be used when provisioning virtual machines in hypervisor environments.
/// </summary>
/// <param name="hiveDefinition">The hive definition.</param>
/// <returns>The prefix.</returns>
public string GetVmNamePrefix(HiveDefinition hiveDefinition)
{
if (VmNamePrefix == null)
{
return($"{hiveDefinition.Name}-".ToLowerInvariant());
}
else if (string.IsNullOrWhiteSpace(VmNamePrefix))
{
return(string.Empty);
}
else
{
return($"{VmNamePrefix}-".ToLowerInvariant());
}
}
/// <summary>
/// Returns credentials for a Vault AppRole.
/// </summary>
/// <param name="roleName">The role name.</param>
/// <param name="cancellationToken">The optional <see cref="CancellationToken"/>.</param>
/// <returns>The <see cref="HiveCredentials"/>.</returns>
/// <exception cref="HttpException">Thrown for Vault communication problems.</exception>
public async Task <HiveCredentials> GetAppRoleCredentialsAsync(string roleName, CancellationToken cancellationToken = default)
{
Covenant.Requires <ArgumentException>(HiveDefinition.IsValidName(roleName));
string roleId;
string secretId;
// Verify that the role exists.
try
{
await ReadDynamicAsync($"auth/approle/role/{roleName}", cancellationToken : cancellationToken);
}
catch (Exception e)
{
throw new HttpException($"Unable to access Vault [AppRole={roleName}]", e);
}
// Fetch the role ID.
try
{
var response = await ReadDynamicAsync($"auth/approle/role/{roleName}/role-id", cancellationToken : cancellationToken);
roleId = response.role_id;
}
catch (Exception e)
{
throw new HttpException($"Unable to fetch the role ID for Vault [AppRole={roleName}]", e);
}
// Fetch a secret ID.
try
{
var response = (await WriteJsonAsync($"auth/approle/role/{roleName}/secret-id", cancellationToken)).data;
secretId = response.secret_id;
}
catch (Exception e)
{
throw new HttpException($"Unable to fetch the role ID for Vault [AppRole={roleName}]", e);
}
// Return the credentials.
return(HiveCredentials.FromVaultRole(roleId, secretId));
}
/// <inheritdoc/>
public void Validate(HiveDefinition hiveDefinition)
{
CheckInitialized();
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
var hive = new HiveProxy(hiveDefinition);
var manager = GetManager(hive);
if (manager == null)
{
throw new HiveException($"Cannot locate a [{nameof(IHostingManager)}] implementation for the [{hiveDefinition.Hosting.Environment}] hosting environment.");
}
manager.Validate(hiveDefinition);
}
/// <summary>
/// Sets or removes a named hive global setting, serialized saved objects as JSON.
/// </summary>
/// <param name="name">The setting name.</param>
/// <param name="value">The setting value or <c>null</c> to remove the setting if it exists.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
/// <remarks>
/// <note>
/// Well known hive setting names are defined in <see cref="HiveGlobals"/>.
/// </note>
/// </remarks>
public void Set(string name, object value)
{
Covenant.Requires(!string.IsNullOrEmpty(name));
Covenant.Requires(HiveDefinition.IsValidName(name));
var key = $"{HiveConst.GlobalKey}/{name}";
if (value == null)
{
hive.Consul.Client.KV.Delete(key).Wait();
}
else
{
hive.Consul.Client.KV.PutString(key, NeonHelper.JsonSerialize(value, Formatting.Indented)).Wait();
}
}
/// <summary>
/// Sets or removes a named <see cref="TimeSpan"/> hive global setting.
/// </summary>
/// <param name="name">The setting name.</param>
/// <param name="value">The setting value or <c>null</c> to remove the setting if it exists.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
/// <remarks>
/// <note>
/// Well known hive setting names are defined in <see cref="HiveGlobals"/>.
/// </note>
/// </remarks>
public void Set(string name, TimeSpan?value)
{
Covenant.Requires(!string.IsNullOrEmpty(name));
Covenant.Requires(HiveDefinition.IsValidName(name));
var key = $"{HiveConst.GlobalKey}/{name}";
if (value == null)
{
hive.Consul.Client.KV.Delete(key).Wait();
}
else
{
hive.Consul.Client.KV.PutString(key, value.Value.ToString()).Wait();
}
}
/// <summary>
/// Returns the minimum number of bytes of memory allocate to for this node when
/// hosted on a hypervisor.
/// </summary>
/// <param name="hiveDefinition">The hive definition.</param>
/// <returns>The size in bytes.</returns>
public long GetVmMinimumMemory(HiveDefinition hiveDefinition)
{
if (!string.IsNullOrEmpty(VmMinimumMemory))
{
return(HiveDefinition.ValidateSize(VmMinimumMemory, this.GetType(), nameof(VmMinimumMemory)));
}
else if (!string.IsNullOrEmpty(hiveDefinition.Hosting.VmMinimumMemory))
{
return(HiveDefinition.ValidateSize(hiveDefinition.Hosting.VmMinimumMemory, hiveDefinition.Hosting.GetType(), nameof(hiveDefinition.Hosting.VmMinimumMemory)));
}
else
{
// Return [VmMemory] otherwise.
return(GetVmMemory(hiveDefinition));
}
}
public void Validate(HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
if (!hiveDefinition.Network.StaticIP)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(NetworkOptions.StaticIP)}] must be [true] when deploying to Hyper-V.");
}
if (string.IsNullOrEmpty(HostVhdxUri) || !Uri.TryCreate(HostVhdxUri, UriKind.Absolute, out Uri uri))
{
throw new HiveDefinitionException($"[{nameof(LocalHyperVOptions)}.{nameof(HostVhdxUri)}] is required when deploying to Hyper-V.");
}
hiveDefinition.ValidatePrivateNodeAddresses(); // Private node IP addresses must be assigned and valid.
hiveDefinition.Hosting.ValidateHypervisor(hiveDefinition, remoteHypervisors: false); // Hypervisor options must be valid.
}
