/// <summary>
/// <para>
/// Used to temporarily modify the <b>hosts</b> file used by the DNS resolver
/// for debugging or other purposes.
/// </para>
/// <note>
/// <b>WARNING:</b> Modifying the <b>hosts</b> file will impact all processes
/// on the system, not just the current one and this is designed to be used by
/// a single process at a time.
/// </note>
/// </summary>
/// <param name="hostEntries">A dictionary mapping the hostnames to an IP address or <c>null</c>.</param>
/// <param name="section">
/// <para>
/// Optionally specifies the string to use to mark the hostnames section. This
/// defaults to <b>MODIFY</b> which will delimit the section with <b># NEON-BEGIN-MODIFY</b>
/// and <b># NEON-END-MODIFY</b>. You may pass a different string to identify a custom section.
/// </para>
/// <note>
/// The string passed must be a valid DNS hostname label that must begin with a letter
/// followed by letters, digits or dashes. The maximum length is 63 characters.
/// </note>
/// </param>
/// <remarks>
/// <note>
/// This method requires elevated administrative privileges.
/// </note>
/// <para>
/// This method adds or removes a temporary section of host entry definitions
/// delimited by special comment lines. When <paramref name="hostEntries"/> is
/// non-null and non-empty, the section will be added or updated. Otherwise, the
/// section will be removed.
/// </para>
/// <para>
/// You can remove all host sections by passing both <paramref name="hostEntries"/>
/// and <paramref name="section"/> as <c>null</c>.
/// </para>
/// </remarks>
public static void ModifyLocalHosts(Dictionary <string, IPAddress> hostEntries = null, string section = "MODIFY")
{
#if XAMARIN
throw new NotSupportedException();
#else
if (hostEntries != null && string.IsNullOrWhiteSpace(section))
{
throw new ArgumentNullException(nameof(section));
}
if (section != null)
{
var sectionOK = char.IsLetter(section[0]) && section.Length <= 63;
if (sectionOK)
{
foreach (var ch in section)
{
if (!char.IsLetterOrDigit(ch) && ch != '-')
{
sectionOK = false;
break;
}
}
}
if (!sectionOK)
{
throw new ArgumentException("Suffix is not a valid DNS host name label.", nameof(section));
}
section = section.ToUpperInvariant();
}
string hostsPath;
if (NeonHelper.IsWindows)
{
hostsPath = Path.Combine(Environment.GetEnvironmentVariable("windir"), "System32", "drivers", "etc", "hosts");
}
else if (NeonHelper.IsLinux || NeonHelper.IsOSX)
{
hostsPath = "/etc/hosts";
}
else
{
throw new NotSupportedException();
}
// We're seeing transient file locked errors when trying to update the [hosts] file.
// My guess is that this is cause by the Window DNS resolver opening the file as
// READ/WRITE to prevent it from being modified while the resolver is reading any
// changes.
//
// We're going to mitigate this by retrying a few times.
//
// It can take a bit of time for the Windows DNS resolver to pick up the change.
//
// https://github.com/nforgeio/neonKUBE/issues/244
//
// We're going to mitigate this by writing a [neon-modify-local-hosts.nhive.io] record with
// a random IP address and then wait for for the DNS resolver to report the correct address.
//
// Note that this only works on Windows and perhaps OSX. This doesn't work on
// Linux because there's no central DNS resolver there. See the issue below for
// more information:
//
// https://github.com/nforgeio/neonKUBE/issues/271
var updateHost = section != null ? $"{section.ToLowerInvariant()}.neonforge-marker" : $"H-{Guid.NewGuid().ToString("d")}.neonforge-marker";
var addressBytes = NeonHelper.GetCryptoRandomBytes(4);
var updateAddress = GetRandomAddress();
var lines = new List <string>();
var existingHosts = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
var different = false;
retryFile.InvokeAsync(
async() =>
{
var beginMarker = $"# NEON-BEGIN-";
var endMarker = $"# NEON-END-";
if (section != null)
{
beginMarker += section;
endMarker += section;
}
var inputLines = File.ReadAllLines(hostsPath);
var inSection = false;
// Load lines of text from the current [hosts] file, without
// any lines for the named section. We're going to parse those
// lines instead, so we can compare them against the [hostEntries]
// passed to determine whether we actually need to update the
// [hosts] file.
lines.Clear();
existingHosts.Clear();
foreach (var line in inputLines)
{
var trimmed = line.Trim();
if (trimmed == beginMarker || (section == null && trimmed.StartsWith(beginMarker)))
{
inSection = true;
}
else if (trimmed == endMarker || (section == null && trimmed.StartsWith(endMarker)))
{
inSection = false;
}
else
{
if (inSection)
{
// The line is within the named section, so we're going to parse
// the host entry (if any) and add it to [existingHosts].
if (trimmed.Length == 0 || trimmed.StartsWith("#"))
{
// Ignore empty or comment lines (just to be safe).
continue;
}
// We're going to simply assume that the address and hostname
// are separated by whitespace and that there's no other junk
// on the line (like comments added by the operator). If there
// is any junk, we'll capture that too and then the entries
// won't match and we'll just end up rewriting the section
// (which is reasonable).
//
// Note that we're going to ignore the special marker entry.
var fields = line.Split(new char[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
var address = fields[0];
var hostname = fields.Length > 1 ? fields[1] : string.Empty;
if (!hostname.EndsWith(".neonforge-marker"))
{
existingHosts[hostname] = address;
}
}
else
{
// The line is not in the named section, so we'll
// include it as as.
lines.Add(line);
}
}
}
// Compare the existing entries against the new ones and rewrite
// the [hosts] file only if they are different.
if (hostEntries != null && hostEntries.Count == existingHosts.Count)
{
foreach (var item in hostEntries)
{
if (!existingHosts.TryGetValue(item.Key, out var existingAddress) ||
item.Value.ToString() != existingAddress)
{
different = true;
break;
}
}
if (!different)
{
return;
}
}
// Append the section if it has any host entries.
if (hostEntries?.Count > 0)
{
lines.Add(beginMarker);
// Append the special update host with a random IP address.
var address = updateAddress.ToString();
lines.Add($" {address}{new string(' ', 16 - address.Length)} {updateHost}");
// Append the new entries.
foreach (var item in hostEntries)
{
address = item.Value.ToString();
lines.Add($" {address}{new string(' ', 16 - address.Length)} {item.Key}");
}
lines.Add(endMarker);
}
File.WriteAllLines(hostsPath, lines.ToArray());
await Task.CompletedTask;
}).Wait();
if (!different)
{
// We didn't detect any changes to the section above so we're going to
// exit without rewriting the [hosts] file.
return;
}
if (NeonHelper.IsWindows)
{
// Flush the DNS cache (and I believe this reloads the [hosts] file too).
var response = NeonHelper.ExecuteCapture("ipconfig", "/flushdns");
if (response.ExitCode != 0)
{
throw new ToolException($"ipconfig [exitcode={response.ExitCode}]: {response.ErrorText}");
}
}
else if (NeonHelper.IsOSX)
{
// $todo(jefflill):
//
// We may need to clear the OSX DNS cache here. Here's some information on
// how to do this:
//
// https://help.dreamhost.com/hc/en-us/articles/214981288-Flushing-your-DNS-cache-in-Mac-OS-X-and-Linux
throw new NotImplementedException("$todo(jefflill): Purge the OSX DNS cache.");
}
if (NeonHelper.IsWindows || NeonHelper.IsOSX)
{
// Poll the local DNS resolver until it reports the correct address for the
// [neon-modify-local-hosts.nhive.io].
//
// If [hostEntries] is not null and contains at least one entry, we'll lookup
// [neon-modify-local-hosts.neon] and compare the IP address to ensure that the
// resolver has loaded the new entries.
//
// If [hostEntries] is null or empty, we'll wait until there are no records
// for [neon-modify-local-hosts.neon] to ensure that the resolver has reloaded
// the hosts file after we removed the entries.
//
// Note that we're going to count the retries and after the 20th (about 2 second's
// worth of 100ms polling), we're going to rewrite the [hosts] file. I've seen
// situations where at appears that the DNS resolver isn't re-reading [hosts]
// after it's been updated. I believe this is due to the file being written
// twice, once to remove the section and then shortly again there after to
// write the section again. I believe there's a chance that the resolver may
// miss the second file change notification. Writing the file again should
// trigger a new notification.
var retryCount = 0;
retryReady.InvokeAsync(
async() =>
{
var addresses = await GetHostAddressesAsync(updateHost);
if (hostEntries?.Count > 0)
{
// Ensure that the new records have been loaded by the resolver.
if (addresses.Length != 1)
{
RewriteOn20thRetry(hostsPath, lines, ref retryCount);
throw new NotReadyException($"[{updateHost}] lookup is returning [{addresses.Length}] results. There should be [1].");
}
if (addresses[0].ToString() != updateAddress.ToString())
{
RewriteOn20thRetry(hostsPath, lines, ref retryCount);
throw new NotReadyException($"DNS is [{updateHost}={addresses[0]}] rather than [{updateAddress}].");
}
}
else
{
// Ensure that the resolver recognizes that we removed the records.
if (addresses.Length != 0)
{
RewriteOn20thRetry(hostsPath, lines, ref retryCount);
throw new NotReadyException($"[{updateHost}] lookup is returning [{addresses.Length}] results. There should be [0].");
}
}
}).Wait();
}
#endif
}
/// <summary>
/// Converts an array of reflected parameter info into a C# parameter
/// definition.
/// </summary>
/// <param name="parameters">The parameters.</param>
/// <returns>The C# parameter definition source code.</returns>
private static string GetParameterDefinition(ParameterInfo[] parameters)
{
var sb = new StringBuilder();
foreach (var parameter in parameters)
{
var typeReference = ResolveTypeReference(parameter.ParameterType);
var hasParamsAttribute = parameter.GetCustomAttribute <ParamArrayAttribute>() != null;
if (hasParamsAttribute)
{
sb.AppendWithSeparator($"params {typeReference} {NormalizeParameterName(parameter.Name)}", ", ");
}
else
{
var defaultValueAssignment = string.Empty;
if (parameter.HasDefaultValue)
{
var defaultValue = parameter.DefaultValue;
var valueLiteral = string.Empty;
var type = parameter.ParameterType;
if (type.IsClass && defaultValue == null)
{
valueLiteral = "null";
}
else if (type.IsPrimitive || type == typeof(string) || type == typeof(Decimal) || type.IsEnum || type.FullName.StartsWith("System.Nullable`"))
{
if (type == typeof(string))
{
if (defaultValue == null)
{
valueLiteral = "null";
}
else
{
valueLiteral = $"\"{defaultValue}\"";
}
}
else
{
if (type.IsEnum)
{
valueLiteral = $"{type.Name}.{defaultValue}";
}
else if (type == typeof(bool))
{
valueLiteral = NeonHelper.ToBoolString((bool)defaultValue);
}
else if (defaultValue == null)
{
valueLiteral = "null";
}
else
{
valueLiteral = defaultValue.ToString();
}
}
}
else if (!type.IsClass)
{
valueLiteral = $"default({typeReference})";
}
else
{
// Other types can't have default values.
valueLiteral = null;
}
if (!string.IsNullOrEmpty(valueLiteral))
{
defaultValueAssignment = $" = {valueLiteral}";
}
}
sb.AppendWithSeparator($"{typeReference} {NormalizeParameterName(parameter.Name)}{defaultValueAssignment}", ", ");
}
}
return(sb.ToString());
}
/// <summary>
/// Decrypts a stream to another stream.
/// </summary>
/// <param name="source">The source stream.</param>
/// <param name="target">The target stream.</param>
/// <exception cref="CryptographicException">Thrown if the password was not found or for other decryption problems.</exception>
public void Decrypt(Stream source, Stream target)
{
// Read the first 512 bytes of the source stream to detect whether
// this is a [NeonVault] encrypted file. We'll simply copy the
// source stream to the target if it isn't one of our files.
var isVaultFile = true;
var startPos = source.Position;
var buffer = new byte[512];
var cb = source.Read(buffer, 0, buffer.Length);
var afterBOMPos = 0;
// Skip over any UTF-8 byte order marker (BOM) bytes.
if (cb > 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF)
{
afterBOMPos = 3;
}
if (cb - afterBOMPos < MagicBytes.Length)
{
isVaultFile = false;
}
for (int i = 0; i < MagicBytes.Length; i++)
{
if (buffer[i + afterBOMPos] != MagicBytes[i])
{
isVaultFile = false;
break;
}
}
if (!isVaultFile)
{
source.Position = startPos;
source.CopyTo(target);
return;
}
// This is a [NeonVault] file. We need to validate the remaining parameters
// on the header line, extract the password name.
var lfPos = -1;
for (int i = afterBOMPos; i < buffer.Length; i++)
{
if (buffer[i] == (char)'\n')
{
lfPos = i + 1;
break;
}
}
if (lfPos == -1)
{
throw new CryptographicException($"Invalid [{nameof(NeonVault)}] file: Invalid header line.");
}
var headerLine = Encoding.UTF8.GetString(buffer, 0, lfPos).Trim();
var fields = headerLine.Split(';');
if (fields.Length != 5)
{
throw new CryptographicException($"Invalid [{nameof(NeonVault)}] file: Unexpected number of header line parameters.");
}
if (fields[2] != "1.0")
{
throw new CryptographicException($"Unsupported [{nameof(NeonVault)}] file: Unexpected version number: {fields[2]}");
}
if (fields[3] != "AES256")
{
throw new CryptographicException($"Unsupported [{nameof(NeonVault)}] file: Unexpected cipher: {fields[3]}");
}
var passwordName = fields[4].ToLowerInvariant();
var key = GetKeyFromPassword(passwordName);
// Read the HEX lines and convert them into bytes and then write then
// to a MemoryStream to be decrypted.
source.Position = startPos + lfPos;
using (var encrypted = new MemoryStream())
{
while (true)
{
if (source.Read(buffer, 0, 1) == 0)
{
break;
}
var firstHex = char.ToUpperInvariant((char)buffer[0]);
if (char.IsWhiteSpace(firstHex))
{
continue;
}
if (source.Read(buffer, 0, 1) == 0)
{
throw new CryptographicException($"Invalid [{nameof(NeonVault)}] file: Odd numer of HEX digits on a line.");
}
var secondHex = char.ToUpperInvariant((char)buffer[0]);
if (char.IsWhiteSpace(firstHex))
{
throw new CryptographicException($"Invalid [{nameof(NeonVault)}] file: Odd numer of HEX digits on a line.");
}
if (!NeonHelper.IsHex(firstHex) || !NeonHelper.IsHex(secondHex))
{
throw new CryptographicException($"Invalid [{nameof(NeonVault)}] file: Invalid HEX digit.");
}
byte value;
if ('0' <= firstHex && firstHex <= '9')
{
value = (byte)(firstHex - '0');
}
else
{
value = (byte)(firstHex - 'A' + 10);
}
value <<= 4;
if ('0' <= secondHex && secondHex <= '9')
{
value |= (byte)(secondHex - '0');
}
else
{
value |= (byte)(secondHex - 'A' + 10);
}
encrypted.WriteByte(value);
}
encrypted.Position = 0;
using (var cipher = new AesCipher(key))
{
cipher.DecryptStream(encrypted, target);
}
}
}
/// <summary>
/// Commits the DNS records to the hosts file.
/// </summary>
public void Commit()
{
// Use a static lock to ensure that only once fixture instance
// at a time can munge the [hosts] file.
lock (syncLock)
{
var sectionHostname = GetSectionHostname(fixtureId);
// Remove any existing section for this instance.
RemoveSection(fixtureId);
// Append the fixture section to the end of the [hosts] file.
var sb = new StringBuilder();
sb.AppendLine($"# START-NEON-HOSTS-FIXTURE-{fixtureId}");
sb.AppendLine($"{dummyIP, -15} {sectionHostname}");
foreach (var record in records)
{
sb.AppendLine($"{record.Item2, -15} {record.Item1}");
}
sb.AppendLine($"# END-NEON-HOSTS-FIXTURE-{fixtureId}");
retryFile.InvokeAsync(
async() =>
{
File.AppendAllText(HostsPath, sb.ToString());
await Task.CompletedTask;
}).Wait();
if (NeonHelper.IsWindows)
{
// Flush the DNS cache (and I believe this reloads the [hosts] file too).
var response = NeonHelper.ExecuteCapture("ipconfig", "/flushdns");
if (response.ExitCode != 0)
{
throw new ToolException($"ipconfig [exitcode={response.ExitCode}]: {response.ErrorText}");
}
}
else if (NeonHelper.IsOSX)
{
// $todo(jeff.lill):
//
// We may need to clear the OSX DNS cache here.
//
// Here's some information on how to do this:
//
// https://help.dreamhost.com/hc/en-us/articles/214981288-Flushing-your-DNS-cache-in-Mac-OS-X-and-Linux
throw new NotImplementedException("$todo(jeff.lill): Purge the OSX DNS cache.");
}
// Wait for the local DNS resolver to indicate that it's picked
// up the changes by verifying that the section hostname resolves.
retryReady.InvokeAsync(
async() =>
{
var addresses = await GetHostAddressesAsync(sectionHostname);
if (addresses.Length == 0)
{
throw new NotReadyException($"Waiting for [{sectionHostname}] to resolve by the local DNS resolver.");
}
}).Wait();
}
}
/// <summary>
/// Computes the SHA512 hash for a stream from the current position'
/// until the end and returns the result formatted as a lowercase
/// hex string.
/// </summary>
/// <param name="input">The stream.</param>
/// <returns>The hash HEX string.</returns>
public static string ComputeSHA512String(Stream input)
{
return NeonHelper.ToHex(ComputeSHA512Bytes(input));
}
/// <summary>
/// Lists the packages for an organization.
/// </summary>
/// <param name="organization">The GitHub organization name.</param>
/// <param name="nameOrPattern">The matching pattern (see <see cref="NeonHelper.FileWildcardRegex(string)"/>).</param>
/// <param name="packageType">Optionally specifies the package type. This defaults to <see cref="GitHubPackageType.Container"/>.</param>
/// <param name="visibility">Optionally specifies the visibility of the package. This defaults to <see cref="GitHubPackageVisibility.All"/></param>
/// <param name="includeVersions">Optionally queries for the package versions as well. This defaults to <c>false</c>.</param>
/// <returns>The list of package information as a list of <see cref="GitHubPackage"/> instance.</returns>
public async Task <List <GitHubPackage> > ListAsync(
string organization,
string nameOrPattern = null,
GitHubPackageType packageType = GitHubPackageType.Container,
GitHubPackageVisibility visibility = GitHubPackageVisibility.All,
bool includeVersions = false)
{
await SyncContext.Clear;
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(organization), nameof(organization));
using (var client = GitHub.CreateJsonClient())
{
nameOrPattern = nameOrPattern ?? "*";
// List the packages that match the name and visibility parameters.
var regex = NeonHelper.FileWildcardRegex(nameOrPattern);
var packages = new List <GitHubPackage>();
var type = NeonHelper.EnumToString(packageType);
var rawPackages = await client.GetPaginatedAsync($"/orgs/{organization}/packages?package_type={type}");
foreach (var rawPackage in rawPackages)
{
string packageName = rawPackage.name;
if (regex.IsMatch(packageName))
{
var package = new GitHubPackage()
{
Name = packageName,
Type = NeonHelper.ParseEnum <GitHubPackageType>((string)rawPackage.package_type),
Visibility = NeonHelper.ParseEnum <GitHubPackageVisibility>((string)rawPackage.visibility)
};
if (visibility == GitHubPackageVisibility.All || package.Visibility == visibility)
{
packages.Add(package);
}
}
}
// Fetch the package versions when requested.
// $todo(jefflill):
//
// We should be using [Async.ParallelForEachAsync()) to speed this up.
if (includeVersions)
{
foreach (var package in packages)
{
var rawPackageVersions = await client.GetPaginatedAsync($"/orgs/{organization}/packages/{type}/{package.Name}/versions");
foreach (var rawVersion in rawPackageVersions)
{
// $hack(jefflill):
//
// We're special-casing container images by parsing the image
// tags from the package metadata. We're also going to ignore
// versions without any tags for now.
var packageVersion = new GitHubPackageVersion()
{
Id = (string)rawVersion.id,
Name = (string)rawVersion.name
};
if (packageType == GitHubPackageType.Container)
{
foreach (var rawTag in rawVersion.metadata.container.tags)
{
packageVersion.Tags.Add((string)rawTag);
}
}
if (packageVersion.Tags.Count > 0)
{
package.Versions.Add(packageVersion);
}
}
}
}
return(packages);
}
}
/// <summary>
/// Executes a <b>neon-cli</b> command against the current test cluster.
/// </summary>
/// <param name="args">The command arguments.</param>
/// <returns>An <see cref="ExecuteResponse"/> with the exit code and output text.</returns>
/// <remarks>
/// <para>
/// <b>neon-cli</b> is a wrapper around the <b>kubectl</b> and <b>helm</b> tools.
/// </para>
/// <para><b>KUBECTL COMMANDS:</b></para>
/// <para>
/// <b>neon-cli</b> implements <b>kubectl</b> commands directly like:
/// </para>
/// <code>
/// neon get pods
/// neon apply -f myapp.yaml
/// </code>
/// <para><b>HELM COMMANDS:</b></para>
/// <para>
/// <b>neon-cli</b> implements <b>helm</b> commands like <b>neon helm...</b>:
/// </para>
/// <code>
/// neon helm install -f values.yaml myapp .
/// neon helm uninstall myapp
/// </code>
/// <para><b>THROW EXCEPTION ON ERRORS</b></para>
/// <para>
/// Rather than explicitly checking the <see cref="ExecuteResponse.ExitCode"/> and throwing
/// exceptions yourself, you can call <see cref="ExecuteResponse.EnsureSuccess()"/> which
/// throws an <see cref="ExecuteException"/> for non-zero exit codes or you can use
/// <see cref="NeonExecuteWithCheck(string[])"/> which does this for you.
/// </para>
/// </remarks>
public ExecuteResponse NeonExecute(params string[] args)
{
return(NeonHelper.ExecuteCapture("neon", args));
}
/// <summary>
/// Removes then local Docker registry from the hive.
/// </summary>
/// <param name="progress">Optional action that will be called with a progress message.</param>
/// <exception cref="HiveException">Thrown if no registry is deployed or there was an error removing it.</exception>
public void RemoveLocalRegistry(Action <string> progress = null)
{
if (!HasLocalRegistry)
{
throw new HiveException("The [neon-registry] service is not deployed.");
}
var syncLock = new object();
var manager = hive.GetReachableManager();
var hostname = hive.Registry.GetLocalHostname();
// Logout of the registry.
progress?.Invoke($"Logging the hive out of the [{hostname}] registry.");
hive.Registry.Logout(hostname);
// Delete the [neon-registry] service and volume. Note that
// the volume should exist on all of the manager nodes.
progress?.Invoke($"Removing the [neon-registry] service.");
manager.DockerCommand(RunOptions.None, "docker", "service", "rm", "neon-registry");
progress?.Invoke($"Removing the [neon-registry] volumes.");
var volumeRemoveActions = new List <Action>();
var volumeRetryPolicy = new LinearRetryPolicy(typeof(TransientException), maxAttempts: 10, retryInterval: TimeSpan.FromSeconds(2));
foreach (var node in hive.Managers)
{
volumeRemoveActions.Add(
() =>
{
// $hack(jeff.lill):
//
// Docker service removal appears to be synchronous but the removal of the
// actual service task containers is not. We're going to detect this and
// throw a [TransientException] and then retry.
using (var clonedNode = node.Clone())
{
lock (syncLock)
{
progress?.Invoke($"Removing [neon-registry] volume on [{clonedNode.Name}].");
}
volumeRetryPolicy.InvokeAsync(
async() =>
{
var response = clonedNode.DockerCommand(RunOptions.None, "docker", "volume", "rm", "neon-registry");
if (response.ExitCode != 0)
{
if (response.AllText.Contains("volume is in use"))
{
throw new TransientException($"Error removing [neon-registry] volume from [{clonedNode.Name}: {response.ErrorText}");
}
}
else
{
lock (syncLock)
{
progress?.Invoke($"Removed [neon-registry] volume on [{clonedNode.Name}].");
}
}
await Task.Delay(0);
}).Wait();
}
});
}
NeonHelper.WaitForParallel(volumeRemoveActions);
// Remove the traffic manager rule and certificate.
progress?.Invoke($"Removing the [neon-registry] traffic manager rule.");
hive.PublicTraffic.RemoveRule("neon-registry");
progress?.Invoke($"Removing the [neon-registry] traffic manager certificate.");
hive.Certificate.Remove("neon-registry");
// Remove any related Consul state.
progress?.Invoke($"Removing the [neon-registry] Consul [hostname] and [secret].");
hive.Registry.SetLocalHostname(null);
hive.Registry.SetLocalSecret(null);
// Logout the hive from the registry.
progress?.Invoke($"Logging the hive out of the [{hostname}] registry.");
hive.Registry.Logout(hostname);
// Remove the hive DNS host entry.
progress?.Invoke($"Removing the [{hostname}] registry DNS hosts entry.");
hive.Dns.Remove(hostname);
}
/// <summary>
/// Logs the hive into a Docker registry or updates the registry credentials
/// if already logged in.
/// </summary>
/// <param name="registry">The registry hostname or <c>null</c> to specify the Docker public registry.</param>
/// <param name="username">The username.</param>
/// <param name="password">The password.</param>
/// <exception cref="HiveException">Thrown if one or more of the hive nodes could not be logged in.</exception>
public void Login(string registry, string username, string password)
{
Covenant.Requires <ArgumentNullException>(string.IsNullOrEmpty(registry) || HiveDefinition.DnsHostRegex.IsMatch(registry));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(username));
Covenant.Requires <ArgumentNullException>(password != null);
// Update the registry credentials in Vault.
hive.Vault.Client.WriteStringAsync($"{HiveConst.VaultRegistryCredentialsKey}/{registry}", $"{username}/{password}").Wait();
// Login all of the hive nodes in parallel.
var sleepSeconds = 5;
var maxAttempts = 75 / sleepSeconds;
var actions = new List <Action>();
var errors = new List <string>();
foreach (var node in hive.Nodes)
{
actions.Add(
() =>
{
using (var clonedNode = node.Clone())
{
for (int attempt = 1; attempt <= maxAttempts; attempt++)
{
var response = clonedNode.SudoCommand("docker login", RunOptions.None, "--username", username, "--password", password, registry);
if (response.ExitCode == 0)
{
SyncDockerConf(node);
return;
}
if (attempt == maxAttempts)
{
// This is the last attempt.
if (response.ExitCode != 0)
{
lock (errors)
{
errors.Add($"{clonedNode.Name}: {response.ErrorSummary}");
}
}
else
{
SyncDockerConf(node);
}
break;
}
else
{
// Pause for 5 seconds to mitigate transient errors.
Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds));
}
}
}
});
}
NeonHelper.WaitForParallel(actions);
if (errors.Count > 0)
{
var sb = new StringBuilder();
sb.AppendLine($"Could not login [{errors.Count}] nodes to the [{registry}] Docker registry.");
sb.AppendLine($"Your hive may now be in an inconsistent state.");
sb.AppendLine();
foreach (var error in errors)
{
sb.AppendLine(error);
}
throw new HiveException(sb.ToString());
}
}
/// <summary>
/// Program entry point.
/// </summary>
/// <param name="args">The command line arguments.</param>
/// <returns>The exit code.</returns>
public static int Main(params string[] args)
{
string usage = $@"
neonKUBE Management Tool: neon [v{Program.Version}]
{Build.Copyright}
USAGE:
neon [OPTIONS] COMMAND [ARG...]
COMMAND SUMMARY:
neon help COMMAND
neon cluster prepare [CLUSTER-DEF]
neon cluster setup [CLUSTER-DEF]
neon couchbase COMMNAND
neon generate models [OPTIONS] ASSEMBLY-PATH [OUTPUT-PATH]
neon login COMMAND
neon logout
neon password COMMAND
neon prepare COMMAND
neon run -- COMMAND
neon scp [NODE]
neon ssh [NODE]
neon vault COMMAND
neon version [-n] [--git] [--minimum=VERSION]
ARGUMENTS:
CLUSTER-DEF - Path to a cluster definition file. This is
optional for some commands when logged in
COMMAND - Subcommand and arguments.
NODE - A node name.
OPTIONS:
--help - Display help
--insecure - Use insecure temporary folder
(see remarks)
--log-folder=LOG-FOLDER - Optional log folder path
-m=COUNT, --max-parallel=COUNT - Maximum number of nodes to be
configured in parallel [default=6]
--machine-password=PASSWORD - Overrides default initial machine
password: sysadmin0000
--machine-username=USERNAME - Overrides default initial machine
username: sysadmin
-q, --quiet - Disables operation progress
-w=SECONDS, --wait=SECONDS - Seconds to delay for cluster stablization
(defaults to 60s).
REMARKS:
By default, any tempory files generated will be written to your
[$HOME/.neonkube/temp] folder which is encrypted at rest for
Windows 10 PRO and Windows Server workstations. This encryption
may cause some problems (e.g. a [neon run ...] command that
needs to mount a decrypted file to a local Docker container.
You can disable the use of this encrypted folder by specifying
[--insecure] for any command.
";
// Disable any logging that might be performed by library classes.
LogManager.Default.LogLevel = LogLevel.None;
// Use the version of Powershell Core installed with the application,
// if present.
PowerShell.PwshPath = KubeHelper.PwshPath;
// We need to verify that we're running with elevated permissions if we're not
// shimmed into a Docker container.
// $todo(jefflill):
//
// We're currently requiring elevated permissions for all commands, even those
// that don't actually require elevated permissions. We may wish to relax this
// in the future.
if (!KubeHelper.InToolContainer)
{
if (NeonHelper.IsWindows)
{
var identity = WindowsIdentity.GetCurrent();
var principal = new WindowsPrincipal(identity);
if (!principal.IsInRole(WindowsBuiltInRole.Administrator))
{
Console.Error.WriteLine("*** ERROR: This command requires elevated administrator permissions.");
Program.Exit(1);
}
}
else if (NeonHelper.IsOSX)
{
// $todo(jefflill): Do we really need this?
// throw new NotImplementedException("$todo(jefflill): Implement OSX elevated permissions check.");
}
}
// Ensure that all of the cluster hosting manager implementations are loaded.
new HostingManagerFactory(() => HostingLoader.Initialize());
// Process the command line.
try
{
ICommand command;
CommandLine = new CommandLine(args);
LeftCommandLine = CommandLine.Split("--").Left;
foreach (var cmdLine in new CommandLine[] { CommandLine, LeftCommandLine })
{
cmdLine.DefineOption("--machine-username");
cmdLine.DefineOption("--machine-password");
cmdLine.DefineOption("-os").Default = "Ubuntu-18.04";
cmdLine.DefineOption("-q", "--quiet");
cmdLine.DefineOption("-m", "--max-parallel").Default = "6";
cmdLine.DefineOption("-w", "--wait").Default = "60";
cmdLine.DefineOption("--log-folder").Default = string.Empty;
}
var validOptions = new HashSet <string>();
validOptions.Add("--debug");
validOptions.Add("--help");
validOptions.Add("--insecure");
validOptions.Add("--log-folder");
validOptions.Add("--machine-username");
validOptions.Add("--machine-password");
validOptions.Add("-m");
validOptions.Add("--max-parallel");
validOptions.Add("-q");
validOptions.Add("--quiet");
validOptions.Add("-w");
validOptions.Add("--wait");
validOptions.Add("-b");
validOptions.Add("--branch");
if (CommandLine.Arguments.Length == 0)
{
Console.WriteLine(usage);
Program.Exit(0);
}
if (!CommandLine.HasOption("--insecure"))
{
// Ensure that temporary files are written to the user's temporary folder because
// there's a decent chance that this folder will be encrypted at rest.
if (KubeTestManager.Current == null)
{
TempFile.Root = KubeHelper.TempFolder;
TempFolder.Root = KubeHelper.TempFolder;
}
}
var commands = new List <ICommand>()
{
new ClusterCommand(),
new ClusterPrepareCommand(),
new ClusterSetupCommand(),
new ClusterVerifyCommand(),
new CouchbaseCommand(),
new CouchbaseQueryCommand(),
new CouchbaseUpsertCommand(),
new GenerateCommand(),
new GenerateModelsCommand(),
new LoginCommand(),
new LoginExportCommand(),
new LoginImportCommand(),
new LoginListCommand(),
new LoginRemoveCommand(),
new LogoutCommand(),
new PasswordCommand(),
new PasswordExportCommand(),
new PasswordGenerateCommand(),
new PasswordGetCommand(),
new PasswordImportCommand(),
new PasswordListCommand(),
new PasswordRemoveCommand(),
new PasswordSetCommand(),
new PrepareCommand(),
new PrepareNodeTemplateCommand(),
new RunCommand(),
new ScpCommand(),
new SshCommand(),
new VaultCommand(),
new VaultCreateCommand(),
new VaultDecryptCommand(),
new VaultEditCommand(),
new VaultEncryptCommand(),
new VaultPasswordNameCommand(),
new VersionCommand()
};
// Short-circuit the help command.
if (CommandLine.Arguments[0] == "help")
{
if (CommandLine.Arguments.Length == 1)
{
Console.WriteLine(usage);
Program.Exit(0);
}
CommandLine = CommandLine.Shift(1);
command = GetCommand(CommandLine, commands);
if (command == null)
{
Console.Error.WriteLine($"*** ERROR: Unexpected [{CommandLine.Arguments[0]}] command.");
Program.Exit(1);
}
command.Help();
Program.Exit(0);
}
// Lookup the command.
command = GetCommand(CommandLine, commands);
if (command == null)
{
Console.Error.WriteLine($"*** ERROR: Unexpected [{CommandLine.Arguments[0]}] command.");
Program.Exit(1);
}
// Handle the logging options.
LogPath = LeftCommandLine.GetOption("--log-folder");
Quiet = LeftCommandLine.GetFlag("--quiet");
if (KubeHelper.InToolContainer)
{
// We hardcode logging to [/log] inside [neon-cli] containers.
LogPath = "/log";
}
else if (!string.IsNullOrEmpty(LogPath))
{
LogPath = Path.GetFullPath(LogPath);
Directory.CreateDirectory(LogPath);
}
else
{
LogPath = KubeHelper.LogFolder;
// We can clear this folder because we know that there shouldn't be
// any other files in here.
NeonHelper.DeleteFolderContents(LogPath);
}
//-------------------------------------------------------------
// Process the standard command line options.
// Load the user name and password from the command line options, if present.
MachineUsername = LeftCommandLine.GetOption("--machine-username", "sysadmin");
MachinePassword = LeftCommandLine.GetOption("--machine-password", "sysadmin0000");
// Handle the other options.
var maxParallelOption = LeftCommandLine.GetOption("--max-parallel");
int maxParallel;
if (!int.TryParse(maxParallelOption, out maxParallel) || maxParallel < 1)
{
Console.Error.WriteLine($"*** ERROR: [--max-parallel={maxParallelOption}] option is not valid.");
Program.Exit(1);
}
Program.MaxParallel = maxParallel;
var waitSecondsOption = LeftCommandLine.GetOption("--wait");
double waitSeconds;
if (!double.TryParse(waitSecondsOption, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out waitSeconds) || waitSeconds < 0)
{
Console.Error.WriteLine($"*** ERROR: [--wait={waitSecondsOption}] option is not valid.");
Program.Exit(1);
}
Program.WaitSeconds = waitSeconds;
Debug = LeftCommandLine.HasOption("--debug");
if (command.CheckOptions)
{
// Ensure that there are no unexpected command line options.
foreach (var optionName in command.ExtendedOptions)
{
validOptions.Add(optionName);
}
foreach (var option in LeftCommandLine.Options)
{
if (!validOptions.Contains(option.Key))
{
var commandWords = string.Empty;
foreach (var word in command.Words)
{
if (commandWords.Length > 0)
{
commandWords += " ";
}
commandWords += word;
}
Console.Error.WriteLine($"*** ERROR: [{commandWords}] command does not support [{option.Key}].");
Program.Exit(1);
}
}
}
// Run the command.
if (command.NeedsSshCredentials(CommandLine))
{
if (string.IsNullOrWhiteSpace(MachineUsername) || string.IsNullOrEmpty(MachinePassword))
{
Console.WriteLine();
Console.WriteLine(" Enter cluster SSH credentials:");
Console.WriteLine(" -------------------------------");
}
while (string.IsNullOrWhiteSpace(MachineUsername))
{
Console.Write(" username: "******" password: "******"*** ERROR: {NeonHelper.ExceptionError(e)}");
Console.Error.WriteLine(e.StackTrace);
Console.Error.WriteLine(string.Empty);
return(1);
}
return(0);
}
/// <summary>
/// Logs the hive out of a Docker registry.
/// </summary>
/// <param name="registry">The registry hostname.</param>
/// <exception cref="HiveException">Thrown if one or more of the hive nodes could not be logged out.</exception>
public void Logout(string registry)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(registry));
// Remove the registry credentials from Vault.
hive.Vault.Client.DeleteAsync($"{HiveConst.VaultRegistryCredentialsKey}/{registry}").Wait();
// Logout of all of the hive nodes in parallel.
var actions = new List <Action>();
var errors = new List <string>();
foreach (var node in hive.Nodes)
{
actions.Add(
() =>
{
using (var clonedNode = node.Clone())
{
// We need to special case logging out of the Docker public registry
// by not passing a registry hostname.
var registryArg = registry;
if (registryArg.Equals("docker.io", StringComparison.InvariantCultureIgnoreCase) ||
registryArg.Equals("registry-1.docker.io", StringComparison.InvariantCultureIgnoreCase))
{
registryArg = null;
}
var response = clonedNode.SudoCommand($"docker logout", RunOptions.None, registryArg);
if (response.ExitCode != 0)
{
lock (errors)
{
errors.Add($"{clonedNode.Name}: {response.ErrorSummary}");
}
}
else
{
SyncDockerConf(node);
}
}
});
}
NeonHelper.WaitForParallel(actions);
if (errors.Count > 0)
{
var sb = new StringBuilder();
sb.AppendLine($"Could not logout [{errors.Count}] nodes from the [{registry}] Docker registry.");
sb.AppendLine($"Your hive may now be in an inconsistent state.");
sb.AppendLine();
foreach (var error in errors)
{
sb.AppendLine(error);
}
throw new HiveException(sb.ToString());
}
}
/// <summary>
/// Add an object to the cache.
/// </summary>
/// <param name="key"></param>
/// <param name="value"></param>
/// <returns></returns>
public async Task SetAsync(string key, object value)
{
await SyncContext.Clear;
await cache.SetAsync(key, NeonHelper.JsonSerializeToBytes(value), cacheEntryOptions);
}
public async Task PostAsync()
{
// Ensure that POST sending and returning an explict types works.
RequestDoc requestDoc = null;
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (request.Method != "POST")
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
if (request.Path.ToString() != "/info")
{
response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
requestDoc = NeonHelper.JsonDeserialize <RequestDoc>(request.GetBodyText());
var output = new ReplyDoc()
{
Value1 = "Hello World!"
};
response.ContentType = "application/json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var doc = new RequestDoc()
{
Operation = "FOO",
Arg0 = "Hello",
Arg1 = "World"
};
var reply = (await jsonClient.PostAsync(baseUri + "info", doc)).As <ReplyDoc>();
Assert.Equal("FOO", requestDoc.Operation);
Assert.Equal("Hello", requestDoc.Arg0);
Assert.Equal("World", requestDoc.Arg1);
Assert.Equal("Hello World!", reply.Value1);
reply = await jsonClient.PostAsync <ReplyDoc>(baseUri + "info", doc);
Assert.Equal("FOO", requestDoc.Operation);
Assert.Equal("Hello", requestDoc.Arg0);
Assert.Equal("World", requestDoc.Arg1);
Assert.Equal("Hello World!", reply.Value1);
reply = (await jsonClient.PostAsync(baseUri + "info", @"{""Operation"":""FOO"", ""Arg0"":""Hello"", ""Arg1"":""World""}")).As <ReplyDoc>();
Assert.Equal("FOO", requestDoc.Operation);
Assert.Equal("Hello", requestDoc.Arg0);
Assert.Equal("World", requestDoc.Arg1);
Assert.Equal("Hello World!", reply.Value1);
}
};
}
/// <inheritdoc/>
public override async Task RunAsync(CommandLine commandLine)
{
if (commandLine.HasHelpOption || commandLine.Arguments.Length == 0)
{
Console.WriteLine(usage);
Program.Exit(0);
}
var sourceFolder = commandLine.Arguments.ElementAtOrDefault(0);
var outputPath = commandLine.Arguments.ElementAtOrDefault(1);
var sbGccArgs = new StringBuilder();
if (string.IsNullOrEmpty(sourceFolder) || string.IsNullOrEmpty(outputPath))
{
Console.WriteLine(usage);
Program.Exit(1);
}
foreach (var arg in commandLine.Arguments.Skip(2))
{
sbGccArgs.AppendWithSeparator(arg);
}
// We're going to build this within the distro at [/tmp/wsl-util/GUID] by
// recursively copying the contents of SOURCE-FOLDER to this directory,
// running GCC to build the thing, passing [*.c] to include all of the C
// source files and generating the binary as [output.bin] with the folder.
//
// We're also going to clear the [/tmp/wsl-util] folder first to ensure that we don't
// accumulate any old build files over time and we'll also ensure that
// [gcc] is installed.
var defaultDistro = Wsl2Proxy.GetDefault();
if (string.IsNullOrEmpty(defaultDistro))
{
Console.Error.WriteLine("*** ERROR: There is no default WSL2 distro.");
Program.Exit(1);
}
var distro = new Wsl2Proxy(defaultDistro);
if (!distro.IsDebian)
{
Console.Error.WriteLine($"*** ERROR: Your default WSL2 distro [{distro.Name}] is running: {distro.OSRelease["ID"]}/{distro.OSRelease["ID_LIKE"]}");
Console.Error.WriteLine($" The CRI-O build requires an Debian/Ubuntu based distribution.");
Program.Exit(1);
}
var linuxUtilFolder = LinuxPath.Combine("/", "tmp", "wsl-util");
var linuxBuildFolder = LinuxPath.Combine(linuxUtilFolder, Guid.NewGuid().ToString("d"));
var linuxOutputPath = LinuxPath.Combine(linuxBuildFolder, "output.bin");
var windowsUtilFolder = distro.ToWindowsPath(linuxUtilFolder);
var windowsBuildFolder = distro.ToWindowsPath(linuxBuildFolder);
try
{
// Delete the [/tmp/wsl-util] folder on Linux and the copy the
// source from the Windows side into a fresh distro folder.
NeonHelper.DeleteFolder(windowsUtilFolder);
NeonHelper.CopyFolder(sourceFolder, windowsBuildFolder);
// Install [safe-apt-get] if it's not already present. We're using this
// because it's possible that projects build in parallel and it's possible
// that multiple GCC commands could also be running in parallel.
var linuxSafeAptGetPath = "/usr/bin/safe-apt-get";
var windowsSafeAptGetPath = distro.ToWindowsPath(linuxSafeAptGetPath);
if (!File.Exists(windowsSafeAptGetPath))
{
var resources = Assembly.GetExecutingAssembly().GetResourceFileSystem("WslUtil.Resources");
var toolScript = resources.GetFile("/safe-apt-get.sh").ReadAllText();
// Note that we need to escape all "$" characters in the script
// so the upload script won't attempt to replace any variables
// (with blanks).
toolScript = toolScript.Replace("$", "\\$");
var uploadScript =
$@"
cat <<EOF > {linuxSafeAptGetPath}
{toolScript}
EOF
chmod 754 {linuxSafeAptGetPath}
";
distro.SudoExecuteScript(uploadScript).EnsureSuccess();
}
// Perform the build.
var buildScript =
$@"
set -euo pipefail
safe-apt-get install -yq gcc
cd {linuxBuildFolder}
gcc *.c -o {linuxOutputPath} {sbGccArgs}
";
distro.SudoExecuteScript(buildScript).EnsureSuccess();
// Copy the build output to the Windows output path.
Directory.CreateDirectory(Path.GetDirectoryName(outputPath));
NeonHelper.DeleteFile(outputPath);
File.Copy(distro.ToWindowsPath(linuxOutputPath), outputPath);
}
finally
{
// Remove the temporary distro folder.
NeonHelper.DeleteFolder(windowsBuildFolder);
}
await Task.CompletedTask;
}
/// <summary>
/// Writes an object value as JSON to a Consul key.
/// </summary>
/// <param name="kv">The key/value endpoint.</param>
/// <param name="key">The key.</param>
/// <param name="value">The value.</param>
/// <param name="formatting">Optional JSON formatting (defaults to <b>None</b>).</param>
/// <param name="cancellationToken">The optional cancellation token.</param>
/// <returns><c>true</c> on success.</returns>
public static async Task <bool> PutObject(this IKVEndpoint kv, string key, object value, Formatting formatting = Formatting.None, CancellationToken cancellationToken = default)
{
return(await PutString(kv, key, NeonHelper.JsonSerialize(value, formatting), cancellationToken));
}
/// <summary>
/// Handles incoming client connections on a background thread.
/// </summary>
/// <param name="pipeIndexObject">Passes as the index into the [pipes] array this thread will use for its server side pipe.</param>
private void ServerThread(object pipeIndexObject)
{
try
{
var pipeIndex = (int)pipeIndexObject;
while (true)
{
lock (syncLock)
{
if (disposing)
{
// The server is shutting down, so exit the thread.
return;
}
if (pipes[pipeIndex] != null)
{
pipes[pipeIndex].Dispose();
pipes[pipeIndex] = null;
}
#pragma warning disable CA1416
pipes[pipeIndex] = new NamedPipeServerStream(pipeName, PipeDirection.InOut, maxNumberOfServerInstances: threads.Length, PipeTransmissionMode.Message, PipeOptions.CurrentUserOnly);
#pragma warning restore CA1416
}
var pipe = pipes[pipeIndex];
pipe.WaitForConnection();
if (disposing)
{
return;
}
var reader = new StreamReader(pipe);
var writer = new StreamWriter(pipe);
writer.AutoFlush = true;
var requestLine = reader.ReadLine();
var request = (ProfileRequest)null;
var handlerResult = (ProfileHandlerResult)null;
try
{
request = ProfileRequest.Parse(requestLine);
}
catch (FormatException)
{
// Report an malformed request to the client and then continue
// listening for the next request.
writer.WriteLine(ProfileResponse.CreateError(ProfileStatus.BadRequest, "Malformed request"));
return;
}
if (GetIsReady != null)
{
handlerResult = GetIsReady();
if (handlerResult != null)
{
writer.WriteLine(handlerResult.ToResponse());
#pragma warning disable CA1416
pipe.WaitForPipeDrain();
#pragma warning restore CA1416
pipe.Dispose();
pipes[pipeIndex] = null;
continue;
}
}
request.Args.TryGetValue("name", out var name);
request.Args.TryGetValue("vault", out var vault);
request.Args.TryGetValue("masterpassword", out var masterPassword);
try
{
switch (request.Command)
{
case "GET-PROFILE-VALUE":
if (name == null)
{
handlerResult = ProfileHandlerResult.CreateError(request, ProfileStatus.MissingArg, $"GET-PROFILE-VALUE: [name] argument is required.");
break;
}
handlerResult = GetProfileValueHandler(request, name);
break;
case "GET-SECRET-PASSWORD":
if (name == null)
{
handlerResult = ProfileHandlerResult.CreateError(request, ProfileStatus.MissingArg, $"GET-SECRET-PASSWORD: [name] argument is required.");
break;
}
handlerResult = GetSecretPasswordHandler(request, name, vault, masterPassword);
break;
case "GET-SECRET-VALUE":
if (name == null)
{
handlerResult = ProfileHandlerResult.CreateError(request, ProfileStatus.MissingArg, $"GET-SECRET-VALUE: [name] argument is required.");
break;
}
handlerResult = GetSecretValueHandler(request, name, vault, masterPassword);
break;
case "CALL":
if (CallHandler == null)
{
handlerResult = ProfileHandlerResult.CreateError(request, ProfileStatus.BadCommand, $"Server has no call handler.");
}
else
{
handlerResult = CallHandler(request);
}
break;
default:
handlerResult = ProfileHandlerResult.CreateError(request, ProfileStatus.BadCommand, $"Unexpected command: {request.Command}");
break;
}
}
catch (Exception e)
{
handlerResult = ProfileHandlerResult.CreateError(request, ProfileStatus.BadCommand, NeonHelper.ExceptionError(e));
}
writer.WriteLine(handlerResult.ToResponse());
#pragma warning disable CA1416
pipe.WaitForPipeDrain();
#pragma warning restore CA1416
pipe.Disconnect();
}
}
catch
{
// Handle all exceptions by exiting the thread.
return;
}
}
/// <summary>
/// Reads and parses a key as a <c>bool</c>, throwing an exception if the key doesn't exist.
/// </summary>
/// <param name="kv">The key/value endpoint.</param>
/// <param name="key">The key.</param>
/// <param name="cancellationToken">The optional cancellation token.</param>
/// <returns>The parsed value.</returns>
/// <exception cref="KeyNotFoundException">Thrown if <paramref name="key"/> could not be found.</exception>
/// <exception cref="FormatException">Thrown if the value is not valid.</exception>
/// <remarks>
/// <note>
/// Any exceptions thrown will be wrapped within an <see cref="AggregateException"/>.
/// </note>
/// </remarks>
public static async Task <bool> GetBool(this IKVEndpoint kv, string key, CancellationToken cancellationToken = default)
{
var value = await GetString(kv, key, cancellationToken);
return(NeonHelper.ParseBool(value));
}
/// <summary>
/// Cleanly terminates the current process (for internal use).
/// </summary>
/// <param name="exitCode">Optional process exit code (defaults to <b>0</b>).</param>
/// <param name="explicitTermination">Optionally indicates that termination is not due to receiving an external signal.</param>
private void ExitInternal(int exitCode = 0, bool explicitTermination = false)
{
if (readyToExit)
{
// Application has already indicated that it has terminated.
return;
}
var isTerminating = terminating;
terminating = true;
if (isTerminating)
{
return; // Already terminating.
}
var terminationStopwatch = new Stopwatch();
terminationStopwatch.Start();
if (explicitTermination)
{
log?.LogInfo(() => $"INTERNAL stop request: [timeout={GracefulShutdownTimeout}]");
}
else
{
log?.LogInfo(() => $"SIGTERM received: Stopping process [timeout={GracefulShutdownTimeout}]");
}
cts.Cancel();
lock (handlers)
{
foreach (var handler in handlers)
{
NeonHelper.StartThread(handler);
}
}
// Hold here for up to [MinShutdownTime] to allow any straggling requests from
// Kubernetes service load balancers that haven't removed the pod yet.
var remainingWaitTime = MinShutdownTime - terminationStopwatch.Elapsed;
if (remainingWaitTime > TimeSpan.Zero)
{
Thread.Sleep(remainingWaitTime);
}
StopEvent.Set();
try
{
NeonHelper.WaitFor(() => readyToExit, GracefulShutdownTimeout);
log?.LogInfo(() => "Process stopped gracefully.");
}
catch (TimeoutException)
{
log?.LogWarn(() => $"Process did not signal a graceful stop by calling [{nameof(ReadyToExit)}()] within [{GracefulShutdownTimeout}].");
}
if (!DisableProcessExit)
{
Environment.Exit(exitCode);
}
}
/// <summary>
/// Verify that we can create an TCP traffic manager rule for a
/// site on the public port using a specific hostname and then
/// verify that that the traffic manager actually works by spinning
/// up a [vegomatic] based service to accept the traffic.
/// </summary>
/// <param name="testName">Simple name (without spaces) used to ensure that URIs cached for different tests won't conflict.</param>
/// <param name="proxyPort">The inbound proxy port.</param>
/// <param name="network">The proxy network.</param>
/// <param name="trafficManager">The traffic manager.</param>
/// <param name="serviceName">Optionally specifies the backend service name (defaults to <b>vegomatic</b>).</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task TestTcpRule(string testName, int proxyPort, string network, TrafficManager trafficManager, string serviceName = "vegomatic")
{
// Append a GUID to the test name to ensure that we won't
// conflict with what any previous test runs may have loaded
// into the cache.
testName += "-" + Guid.NewGuid().ToString("D");
// Verify that we can create an TCP traffic manager rule for a
// site on the public port using a specific hostname and then
// verify that that the traffic manager actually works by spinning
// up a [vegomatic] based service to accept the traffic.
var queryCount = 100;
var manager = hive.GetReachableManager();
var hostname = manager.PrivateAddress.ToString();
manager.Connect();
using (var client = new TestHttpClient(disableConnectionReuse: true))
{
// Setup the client to query the [vegomatic] service through the
// proxy without needing to configure a hive DNS entry.
client.BaseAddress = new Uri($"http://{manager.PrivateAddress}:{proxyPort}/");
client.DefaultRequestHeaders.Host = testHostname;
// Configure the traffic manager rule.
var rule = new TrafficTcpRule()
{
Name = "vegomatic",
CheckSeconds = 1,
};
rule.Frontends.Add(
new TrafficTcpFrontend()
{
ProxyPort = proxyPort
});
rule.Backends.Add(
new TrafficTcpBackend()
{
Server = serviceName,
Port = 80
});
trafficManager.SetRule(rule);
// Spin up a single [vegomatic] service instance.
manager.SudoCommand($"docker service create --name vegomatic --network {network} --replicas 1 {vegomaticImage} test-server").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Query the service several times to verify that we get a response and
// also that all of the responses are the same (because we have only
// a single [vegomatic] instance returning its UUID).
var uniqueResponses = new HashSet <string>();
for (int i = 0; i < queryCount; i++)
{
var response = await client.GetAsync($"/{testName}/pass-1/{i}?body=server-id&expires=60");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = await response.Content.ReadAsStringAsync();
if (!uniqueResponses.Contains(body))
{
uniqueResponses.Add(body);
}
}
Assert.Single(uniqueResponses);
// Spin up a second replica and repeat the query test to verify
// that we see two unique responses.
//
// Note that we're going to pass a new set of URLs to avoid having
// any responses cached so we'll end up seeing all of the IDs.
//
// Note also that we need to perform these requests in parallel
// to try to force Varnish to establish more than one connection
// to the [vegomatic] service. If we don't do this, Varnish will
// establish a single connection to one of the service instances
// and keep sending traffic there resulting in us seeing only
// one response UUID.
manager.SudoCommand($"docker service update --replicas 2 vegomatic").EnsureSuccess();
await WaitUntilReadyAsync(client.BaseAddress, hostname);
// Reset the response info and do the requests.
uniqueResponses.Clear();
var tasks = new List <Task>();
var uris = new List <string>();
for (int i = 0; i < queryCount; i++)
{
uris.Add($"/{testName}/pass-2/{i}?body=server-id&expires=60&delay=0.250");
}
foreach (var uri in uris)
{
tasks.Add(Task.Run(
async() =>
{
var response = await client.GetAsync(uri);
var body = await response.Content.ReadAsStringAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}));
}
await NeonHelper.WaitAllAsync(tasks, TimeSpan.FromSeconds(30));
Assert.Equal(2, uniqueResponses.Count);
}
}
/// <summary>
/// Expands any variables and TABs in the string passed.
/// </summary>
/// <param name="input">The input text.</param>
/// <returns>The expanded result.</returns>
private string Expand(string input)
{
if (!ExpandVariables)
{
return(input);
}
var output = input;
var matchCount = 0;
while (true)
{
var match = VariableExpansionRegex.Match(output);
if (!match.Success)
{
break;
}
var group = match.Groups["name"];
if (!group.Success)
{
throw new FormatException($"Line {lineNumber}: The [{nameof(VariableExpansionRegex)}] expression does not define the [name] pattern group: {VariableExpansionRegex} ");
}
var name = group.Value;
if (matchCount++ > 128)
{
throw new FormatException($"Line {lineNumber}: More than 128 variable expansions required. Verify that there are no recursively defined variables on line: {input}");
}
string value;
if (name.First() == match.Value[1] || name.Last() == match.Value.Last())
{
// This is an environment variable.
if (name.First() != match.Value[1] || name.Last() != match.Value.Last())
{
throw new FormatException($"Line {lineNumber}: Invalid variable reference [{match.Value}].");
}
name = name.Substring(1, name.Length - 2);
value = Environment.GetEnvironmentVariable(name);
if (value == null)
{
if (DefaultEnvironmentVariable == null)
{
throw new KeyNotFoundException($"Line {lineNumber}: Undefined environment variable reference [{match.Value}].");
}
value = DefaultEnvironmentVariable;
}
}
else
{
// This is a normal variable.
if (!variables.TryGetValue(name, out value))
{
if (DefaultVariable == null)
{
throw new KeyNotFoundException($"Line {lineNumber}: Undefined variable reference [{match.Value}].");
}
value = DefaultVariable;
}
}
output = output.Substring(0, match.Index) + value + output.Substring(match.Index + match.Length);
}
if (TabStop == 0)
{
return(IndentLine(output));
}
else
{
return(IndentLine(NeonHelper.ExpandTabs(output, TabStop)));
}
}
/// <summary>
/// Starts the fixture if it hasn't already been started including invoking the optional
/// <see cref="Action"/> when the first time <see cref="Start(Action)"/> is called for
/// a fixture instance.
/// </summary>
/// <param name="action">
/// <para>
/// The optional custom start action.
/// </para>
/// <note>
/// This is generally intended for use when developing custom test fixtures.
/// </note>
/// </param>
/// <returns>
/// <see cref="TestFixtureStatus.Started"/> if the fixture wasn't previously started and
/// this method call started it or <see cref="TestFixtureStatus.AlreadyRunning"/> if the
/// fixture was already running.
/// </returns>
/// <exception cref="InvalidOperationException">Thrown if this is called from within the <see cref="Action"/>.</exception>
public override TestFixtureStatus Start(Action action = null)
{
CheckDisposed();
if (InAction)
{
throw new InvalidOperationException($"[{nameof(Start)}()] cannot be called recursively from within the fixture initialization action.");
}
if (IsRunning)
{
return(TestFixtureStatus.AlreadyRunning);
}
// Initialize this fixture.
try
{
InAction = true;
action?.Invoke();
// Start any fixture groups from lowest to highest in parallel
// on separate threads.
var groups = new HashSet <int>();
foreach (var group in fixtureList
.Where(fixture => fixture.Group >= 0)
.Select(fixture => fixture.Group))
{
if (!groups.Contains(group))
{
groups.Add(group);
}
}
if (groups.Count > 0)
{
var fixtureThreads = new List <Thread>();
var fixtureExceptions = new List <Exception>();
foreach (var group in groups.OrderBy(group => group))
{
foreach (var subFixture in fixtureList.Where(fixture => fixture.Group == group))
{
var thread = new Thread(
new ParameterizedThreadStart(
arg =>
{
try
{
var sf = (SubFixture)arg;
if (sf.IsNeonService)
{
// $hack(jefflill):
//
// Using reflection, locate the correct subfixture [Start(Func<TService>, TimeSpan]
// method and then call it, passing the service creator function and running timeout.
var startMethod = (MethodInfo)null;
foreach (var method in sf.Fixture.GetType().GetMethods().Where(mi => mi.Name == "Start"))
{
var paramTypes = method.GetParameterTypes();
if (paramTypes.Length != 3)
{
continue;
}
if (paramTypes[0].Name == "Func`1" && paramTypes[1] == typeof(ServiceMap) && paramTypes[2] == typeof(TimeSpan))
{
startMethod = method;
break;
}
}
Covenant.Assert(startMethod != null, "The fixture's [Start(Func<TService>, ServiceMap, TimeSpan)] method signature must have changed.");
startMethod.Invoke(sf.Fixture, new object[] { sf.ServiceCreator, sf.ServiceMap, sf.StartTimeout });
}
else
{
// $hack(jefflill): Using reflection to call the fixture's start method.
sf.Fixture.Start(() => sf.ActionMethod?.Invoke(sf.ActionTarget, new object[] { subFixture.Fixture }));
}
}
catch (Exception e)
{
lock (fixtureExceptions)
{
fixtureExceptions.Add(e);
}
}
}));
fixtureThreads.Add(thread);
thread.Start(subFixture);
// Wait for all fixtured in the group to complete.
NeonHelper.WaitAll(fixtureThreads);
}
if (fixtureExceptions.Count > 0)
{
throw new AggregateException(fixtureExceptions);
}
fixtureThreads.Clear();
}
}
}
finally
{
InAction = false;
IsRunning = true; // Setting this even if the action failed.
}
return(TestFixtureStatus.Started);
}
public static async Task WaitUntilRunningAsync()
{
await NeonHelper.WaitForAsync(async() => await Task.FromResult(isRunning), TimeSpan.FromSeconds(maxWaitSeconds));
}
/// <summary>
/// Generates an opaque globally unique activity ID.
/// </summary>
/// <returns>The activity ID string.</returns>
public static string GenerateActivityId()
{
return(NeonHelper.UrlTokenEncode(Guid.NewGuid().ToByteArray()));
}
public async Task S3_MultiPart(bool publicReadAccess)
{
CheckCredentials();
// Verify that uploading a multi-part file to S3 works.
using (var tempFolder = new TempFolder())
{
// We're going to upload a 9900 byte file with maximum
// part size of 1000 bytes. This should result in nine
// 1000 byte parts and one 900 byte part being uploaded
// (the last part).
var tempPath = Path.Combine(tempFolder.Path, "multi-part.test");
var tempName = Path.GetFileName(tempPath);
var uploadBytes = NeonHelper.GetCryptoRandomBytes(9900);
File.WriteAllBytes(tempPath, uploadBytes);
var upload = AwsCli.S3UploadMultiPart(tempPath, TestBucketHttpsRef, "1.0", maxPartSize: 1000, publicReadAccess: publicReadAccess);
// Validate the Download information.
var manifest = upload.manifest;
Assert.Equal(tempName, manifest.Name);
Assert.Equal("1.0", manifest.Version);
Assert.Equal(tempName, manifest.Filename);
Assert.Equal(9900, manifest.Size);
Assert.Equal(10, manifest.Parts.Count);
Assert.Equal(CryptoHelper.ComputeMD5String(uploadBytes), manifest.Md5);
// Verify that the download information matches our expections.
using (var uploadStream = new MemoryStream(uploadBytes))
{
var partOffset = 0L;
for (int partNumber = 0; partNumber < manifest.Parts.Count; partNumber++)
{
var part = manifest.Parts[partNumber];
Assert.Equal(partNumber, part.Number);
Assert.Equal($"{TestBucketHttpsRef}/{tempName}.parts/part-{partNumber:000#}", part.Uri);
if (partNumber < 9)
{
Assert.Equal(1000, part.Size);
}
else
{
Assert.Equal(900, part.Size);
}
using (var substream = new SubStream(uploadStream, partOffset, part.Size))
{
Assert.Equal(part.Md5, CryptoHelper.ComputeMD5String(substream));
}
partOffset += part.Size;
}
}
using (var uploadStream = new MemoryStream(uploadBytes))
{
using (var httpClient = new HttpClient())
{
// Verify that the actual download file on S3 matches the download information returned.
var response = await httpClient.GetSafeAsync(upload.manifestUri);
Assert.Equal(DeploymentHelper.DownloadManifestContentType, response.Content.Headers.ContentType.MediaType);
var remoteDownload = NeonHelper.JsonDeserialize <DownloadManifest>(await response.Content.ReadAsStringAsync());
Assert.Equal(NeonHelper.JsonSerialize(upload.manifest, Formatting.Indented), NeonHelper.JsonSerialize(remoteDownload, Formatting.Indented));
// Verify that the uploaded parts match what we sent.
var partOffset = 0L;
for (int partNumber = 0; partNumber < manifest.Parts.Count; partNumber++)
{
var part = manifest.Parts[partNumber];
response = await httpClient.GetSafeAsync(part.Uri);
Assert.Equal(part.Md5, CryptoHelper.ComputeMD5String(await response.Content.ReadAsByteArrayAsync()));
partOffset += part.Size;
}
}
}
using (var tempFile = new TempFile())
{
// Verify that [DownloadMultiPartAsync()] checks the [Content-Type] header.
await Assert.ThrowsAsync <FormatException>(async() => await DeploymentHelper.DownloadMultiPartAsync("https://www.google.com", Path.Combine(tempFolder.Path, "test1.dat")));
// Verify that [DownloadMultiPartAsync()] actually works.
var targetPath = Path.Combine(tempFolder.Path, "test2.dat");
await DeploymentHelper.DownloadMultiPartAsync($"{TestBucketHttpsRef}/{tempName}.manifest", targetPath);
Assert.True(File.Exists(targetPath));
Assert.Equal(9900L, new FileInfo(targetPath).Length);
}
}
}
public void Base_PingAttack()
{
// Measure througput with 4 threads hammering the proxy with pings.
var syncLock = new object();
var totalTps = 0.0;
var threads = new Thread[4];
var iterations = 5000;
var exception = (Exception)null;
for (int i = 0; i < threads.Length; i++)
{
threads[i] = NeonHelper.StartThread(
() =>
{
var stopwatch = new Stopwatch();
stopwatch.Start();
for (int j = 0; j < iterations; j++)
{
if (exception != null)
{
return;
}
try
{
client.PingAsync().WaitWithoutAggregate();
}
catch (Exception e)
{
lock (syncLock)
{
if (exception != null)
{
exception = e;
}
}
return;
}
}
stopwatch.Stop();
var tps = iterations * (1.0 / stopwatch.Elapsed.TotalSeconds);
lock (syncLock)
{
totalTps += tps;
}
});
}
foreach (var thread in threads)
{
thread.Join();
}
if (exception != null)
{
throw exception;
}
testWriter.WriteLine($"Transactions/sec: {totalTps}");
testWriter.WriteLine($"Latency (average): {1.0 / totalTps}");
}
public async Task PostUnsafeAsync_Headers()
{
// Ensure that POST with query arguments work.
RequestDoc requestDoc = null;
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (request.Method != "POST")
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
if (request.Path.ToString() != "/info")
{
response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
requestDoc = NeonHelper.JsonDeserialize <RequestDoc>(request.GetBodyText());
var output = new ReplyDoc()
{
Value1 = request.Headers["arg1"],
Value2 = request.Headers["arg2"]
};
response.ContentType = "application/json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var headers = new ArgDictionary()
{
{ "arg1", "test1" },
{ "arg2", "test2" }
};
var doc = new RequestDoc()
{
Operation = "FOO",
Arg0 = "Hello",
Arg1 = "World"
};
var reply = (await jsonClient.PostUnsafeAsync(baseUri + "info", doc, headers: headers)).As <ReplyDoc>();
Assert.Equal("test1", reply.Value1);
Assert.Equal("test2", reply.Value2);
Assert.Equal("FOO", requestDoc.Operation);
Assert.Equal("Hello", requestDoc.Arg0);
Assert.Equal("World", requestDoc.Arg1);
}
};
}
/// <inheritdoc/>
public override void Run(CommandLine commandLine)
{
Console.Write(Convert.ToBase64String(NeonHelper.RandBytes(16)));
}
/// <summary>
/// Returns a Couchbase bucket connection using specified settings and the cluster credentials.
/// </summary>
/// <param name="settings">The Couchbase settings.</param>
/// <param name="username">The username.</param>
/// <param name="password">The password.</param>
/// <param name="timeout">The optional timeout (defaults to 60 seconds).</param>
/// <param name="ignoreDurability">Optionally configure the bucket to ignore durability parameters.</param>
/// <returns>The connected <see cref="NeonBucket"/>.</returns>
/// <exception cref="TimeoutException">Thrown if the bucket is not ready after waiting <paramref name="timeout"/>.</exception>
/// <remarks>
/// <para>
/// You may explicitly pass <paramref name="ignoreDurability"/><c>=true</c> for
/// development and test environments where there may not be enough cluster nodes to
/// satisfy durability constraints. If this is <c>null</c> (the default) then the bucket
/// will look for the presence of the <b>DEV_WORKSTATION</b> environment variable
/// and ignore durability constraints if this variable exists, otherwise durability
/// constraints will be honored.
/// </para>
/// </remarks>
public static NeonBucket OpenBucket(this CouchbaseSettings settings,
string username,
string password,
TimeSpan timeout = default(TimeSpan),
bool?ignoreDurability = null)
{
var bucketConfig =
new BucketConfiguration()
{
BucketName = settings.Bucket,
UseEnhancedDurability = settings.UseEnhancedDurability,
PoolConfiguration = new PoolConfiguration()
{
ConnectTimeout = settings.ConnectTimeout,
SendTimeout = settings.SendTimeout,
MaxSize = settings.MaxPoolConnections,
MinSize = settings.MinPoolConnections
}
};
bucketConfig.PoolConfiguration.ClientConfiguration =
new ClientConfiguration()
{
QueryRequestTimeout = (uint)settings.QueryRequestTimeout,
ViewRequestTimeout = settings.ViewRequestTimeout,
Serializer = () => new EntitySerializer()
};
var config = settings.ToClientConfig();
config.BucketConfigs.Clear();
config.BucketConfigs.Add(settings.Bucket, bucketConfig);
var cluster = new Cluster(config);
// We have to wait for three Couchbase operations to complete
// successfully:
//
// 1. Authenticate
// 2. Open Bucket
// 3. List Indexes
//
// Each of these can fail if Couchbase isn't ready. The Open Bucket
// can fail after the Authenticate succeeded because the bucket is
// still warming up in the cluster.
if (timeout <= TimeSpan.Zero)
{
timeout = NeonBucket.ReadyTimeout;
}
//-----------------------------------------------------------------
// Authenticate against the Couchbase cluster.
var stopwatch = new Stopwatch();
stopwatch.Start();
NeonHelper.WaitFor(
() =>
{
try
{
cluster.Authenticate(username, password);
return(true);
}
catch
{
return(false);
}
},
timeout: timeout,
pollTime: TimeSpan.FromSeconds(0.5));
//-----------------------------------------------------------------
// Open the bucket. Note that we're going to recreate the bucket after
// each failed attempt because it doesn't seem to recover from connection
// failures.
//
// I believe this may be due to the index and or query services not being
// ready yet after Couchbase has just been spun up and the bucket isn't
// notified when these become ready, even after some time passes.
timeout = timeout - stopwatch.Elapsed; // Adjust the timeout downward by the time taken to authenticate.
stopwatch.Restart();
var bucket = (NeonBucket)null;
NeonHelper.WaitFor(
() =>
{
try
{
bucket = new NeonBucket(cluster.OpenBucket(settings.Bucket), settings, ignoreDurability);
return(true);
}
catch
{
return(false);
}
},
timeout: timeout,
pollTime: TimeSpan.FromSeconds(0.5));
//-----------------------------------------------------------------
// Wait until the bucket can perform a simple read operation without
// failing. It appears that we can see early failures for Couchbase
// clusters that have just been created (e.g. during unit tests).
timeout = timeout - stopwatch.Elapsed; // Adjust the timeout downward by the time taken to connect.
stopwatch.Restart();
bucket.WaitUntilReadyAsync(timeout).Wait();
return(bucket);
}
/// <summary>
/// Handles received requests.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task RequestProcessor()
{
while (true)
{
try
{
var newContext = await listener.AcceptAsync();
// Process the request in its own task.
var task = Task.Factory.StartNew(
async (object arg) =>
{
var context = (RequestContext)arg;
var request = context.Request;
var response = context.Response;
using (context)
{
try
{
// Let the request handler have a look.
requestHandler?.Invoke(context);
// Copy the headers, body, and other state from the received request to the remote request.
var remoteRequest = new HttpRequestMessage(new HttpMethod(request.Method), $"{request.Path}{request.QueryString}");
remoteRequest.Version = request.ProtocolVersion;
foreach (var header in request.Headers
.Where(h => !ignoredRequestHeaders.Contains(h.Key)))
{
remoteRequest.Headers.Add(header.Key, header.Value.ToArray());
}
if (request.ContentLength.HasValue && request.ContentLength > 0 ||
request.Headers.TryGetValue("Transfer-Encoding", out var values))
{
// Looks like the client is transmitting content.
remoteRequest.Content = new StreamContent(request.Body);
// Copy the important content related headers.
if (request.Headers.TryGetValue("Content-Length", out var requestContentLengthHeader) &&
long.TryParse(requestContentLengthHeader.First(), out var requestContentLength))
{
remoteRequest.Content.Headers.ContentLength = requestContentLength;
}
if (request.Headers.TryGetValue("Content-Type", out var requestContentTypeHeader))
{
remoteRequest.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(requestContentTypeHeader.First());
}
// $todo(jeff.lill):
//
// Not going to worry about these for now. This will probably
// never be an issue.
//remoteRequest.Content.Headers.ContentMD5
//remoteRequest.Content.Headers.ContentRange
}
// Forward the request to the remote endpoint.
var remoteResponse = await client.SendAsync(remoteRequest, HttpCompletionOption.ResponseHeadersRead);
// Copy the remote response headers, body, and other state to the client response.
//
// Don't copy the "Server" header because the [WebListener] adds its own server
// header and we'd end up with multiple values.
response.StatusCode = (int)remoteResponse.StatusCode;
response.ReasonPhrase = remoteResponse.ReasonPhrase;
foreach (var header in remoteResponse.Headers
.Where(h => !ignoredResponseHeaders.Contains(h.Key)))
{
response.Headers.Add(header.Key, header.Value.ToArray());
}
foreach (var header in remoteResponse.Content.Headers)
{
response.Headers.Add(header.Key, header.Value.ToArray());
}
// Use a buffer from the pool write the data returned from the
// remote endpoint to the client response.
var buffer = GetBuffer();
using (var remoteStream = await remoteResponse.Content.ReadAsStreamAsync())
{
try
{
while (true)
{
var cb = await remoteStream.ReadAsync(buffer, 0, buffer.Length);
if (cb == 0)
{
break;
}
await response.Body.WriteAsync(buffer, 0, cb);
}
}
finally
{
ReleaseBuffer(buffer);
}
}
// Let the response handler have a look.
responseHandler?.Invoke(context);
}
catch (Exception e)
{
response.StatusCode = 503;
response.ReasonPhrase = "service unavailable";
response.ContentType = "text/plain";
response.Body.Write(Encoding.UTF8.GetBytes(NeonHelper.ExceptionError(e)));
}
}
},
newContext);
}
catch (ObjectDisposedException)
{
return; // We're going to use this as the signal to stop.
}
}
}
//--------------------------------------------------------------------
// Static members
/// <summary>
/// Creates a <see cref="ProjectProperties"/> instance holding the
/// necessary properties from a <see cref="Project"/>. This must
/// be called on a UI thread.
/// </summary>
/// <param name="solution">The current solution.</param>
/// <param name="project">The source project.</param>
/// <returns>The cloned <see cref="ProjectProperties"/>.</returns>
public static ProjectProperties CopyFrom(Solution solution, Project project)
{
Covenant.Requires <ArgumentNullException>(solution != null, nameof(solution));
Covenant.Requires <ArgumentNullException>(project != null, nameof(project));
ThreadHelper.ThrowIfNotOnUIThread();
var projectFolder = Path.GetDirectoryName(project.FullName);
var projectFile = File.ReadAllText(project.FullName);
var isNetCore = true;
var netVersion = (SemanticVersion)null;
var sdkName = (string)null;
// Read the properties we care about from the project.
var targetFrameworkMonikers = (string)project.Properties.Item("TargetFrameworkMoniker").Value;
var outputType = (int)project.Properties.Item("OutputType").Value;
var monikers = targetFrameworkMonikers.Split(',');
isNetCore = monikers[0] == ".NETCoreApp";
// Extract the version from the moniker. This looks like: "Version=v5.0"
var versionRegex = new Regex(@"(?<version>[0-9\.]+)$");
netVersion = SemanticVersion.Parse(versionRegex.Match(monikers[1]).Groups["version"].Value);
#if DOESNT_WORK
// The [dotnet --info] command doesn't work as I expected because it doesn't
// appear to examine the project file when determining the SDK version./
//
// https://github.com/nforgeio/RaspberryDebugger/issues/16
// We're going to execute [dotnet --info] in the project directory, to obtain the
// SDK version which will be on the second line which will look something like:
//
// Version: 3.1.301
//
// The cool thing is that this will honor any [global.json] files in the solution.
var orgDirectory = Environment.CurrentDirectory;
Environment.CurrentDirectory = projectFolder;
try
{
var response = NeonHelper.ExecuteCapture("dotnet", new object[] { "--info" });
// Note that we'll stick with the version we extracted from the TargetFrameworkMoniker
// above on the off chance that this call fails.
if (response.ExitCode == 0)
{
using (var reader = new StringReader(response.OutputText))
{
var versionLine = reader.Lines().Skip(1).Take(1).FirstOrDefault().Trim();
Covenant.Assert(versionLine != null);
Covenant.Assert(versionLine.StartsWith("Version:"));
sdkName = versionLine.Split(':')[1];
}
}
}
finally
{
Environment.CurrentDirectory = orgDirectory;
}
#else
// So, we're just going to use the latest known SDK from our catalog instead.
// This isn't ideal but should work fine for the vast majority of people.
var targetSdk = (Sdk)null;
var targetSdkVersion = (SemanticVersion)null;
foreach (var sdkItem in PackageHelper.SdkCatalog.Items
.Where(item => item.IsStandalone && item.Architecture == SdkArchitecture.ARM32))
{
var sdkVersion = SemanticVersion.Parse(sdkItem.Version);
if (sdkVersion.Major != netVersion.Major || sdkVersion.Minor != netVersion.Minor)
{
continue;
}
if (targetSdkVersion == null || sdkVersion > targetSdkVersion)
{
targetSdkVersion = sdkVersion;
targetSdk = new Sdk(sdkItem.Name, sdkItem.Version);;
}
}
if (targetSdk == null)
{
// I don't believe we'll ever see this because the project shouldn't build
// when the required SDK isn't present.
Log.Error($"Unable to locate an SDK for [{targetFrameworkMonikers}].");
Covenant.Assert(false, $"Unable to locate an SDK for [{targetFrameworkMonikers}].");
}
sdkName = targetSdk.Name;
#endif
// Load [Properties/launchSettings.json] if present to obtain the command line
// arguments and environment variables as well as the target connection. Note
// that we're going to use the profile named for the project and ignore any others.
//
// The launch settings for Console vs. WebApps are a bit different. WebApps include
// a top-level "iisSettings"" property and two profiles: "IIS Express" and the
// profile with the project name. We're going to use the presence of the "iisSettings"
// property to determine that we're dealing with a WebApp and we'll do some additional
// processing based off of the project profile:
//
// 1. Launch the browser if [launchBrowser=true]
// 2. Extract the site port number from [applicationUrl]
// 3. Have the app listen on all IP addresses by adding this environment
// variable when we :
//
// ASPNETCORE_SERVER.URLS=http://0.0.0.0:<port>
var launchSettingsPath = Path.Combine(projectFolder, "Properties", "launchSettings.json");
var commandLineArgs = new List <string>();
var environmentVariables = new Dictionary <string, string>();
var isAspNet = false;
var aspPort = 0;
var aspLaunchBrowser = false;
var aspRelativeBrowserUri = "/";
if (File.Exists(launchSettingsPath))
{
var settings = JObject.Parse(File.ReadAllText(launchSettingsPath));
var profiles = settings.Property("profiles");
if (profiles != null)
{
foreach (var profile in ((JObject)profiles.Value).Properties())
{
if (profile.Name == project.Name)
{
var profileObject = (JObject)profile.Value;
var environmentVariablesObject = (JObject)profileObject.Property("environmentVariables")?.Value;
commandLineArgs = ParseArgs((string)profileObject.Property("commandLineArgs")?.Value);
if (environmentVariablesObject != null)
{
foreach (var variable in environmentVariablesObject.Properties())
{
environmentVariables[variable.Name] = (string)variable.Value;
}
}
// Extract additional settings for ASPNET projects.
if (settings.Property("iisSettings") != null)
{
isAspNet = true;
// Note that we're going to fall back to port 5000 if there are any
// issues parsing the application URL.
const int fallbackPort = 5000;
var jProperty = profileObject.Property("applicationUrl");
if (jProperty != null && jProperty.Value.Type == JTokenType.String)
{
try
{
var uri = new Uri((string)jProperty.Value);
aspPort = uri.Port;
if (!NetHelper.IsValidPort(aspPort))
{
aspPort = fallbackPort;
}
}
catch
{
aspPort = fallbackPort;
}
}
else
{
aspPort = fallbackPort;
}
jProperty = profileObject.Property("launchBrowser");
if (jProperty != null && jProperty.Value.Type == JTokenType.Boolean)
{
aspLaunchBrowser = (bool)jProperty.Value;
}
}
}
else if (profile.Name == "IIS Express")
{
// For ASPNET apps, this profile may include a "launchUrl" which
// specifies the absolute or relative URI to display in a debug
// browser launched during debugging.
//
// We're going to normalize this as a relative URI and save it
// so we'll be able to launch the browser on the correct page.
var profileObject = (JObject)profile.Value;
var jProperty = profileObject.Property("launchUrl");
if (jProperty != null && jProperty.Value.Type == JTokenType.String)
{
var launchUri = (string)jProperty.Value;
if (!string.IsNullOrEmpty(launchUri))
{
try
{
var uri = new Uri(launchUri, UriKind.RelativeOrAbsolute);
if (uri.IsAbsoluteUri)
{
aspRelativeBrowserUri = uri.PathAndQuery;
}
else
{
aspRelativeBrowserUri = launchUri;
}
}
catch
{
// We'll fall back to "/" for any URI parsing errors.
aspRelativeBrowserUri = "/";
}
}
}
}
}
}
}
// Get the target Raspberry from the debug settings.
var projects = PackageHelper.ReadRaspberryProjects(solution);
var projectSettings = projects[project.UniqueName];
var debugEnabled = projectSettings.EnableRemoteDebugging;
var debugConnectionName = projectSettings.RemoteDebugTarget;
// Determine whether the referenced .NET Core SDK is currently supported.
var sdk = PackageHelper.SdkCatalog.Items.SingleOrDefault(item => SemanticVersion.Parse(item.Name) == SemanticVersion.Parse(sdkName) && item.Architecture == SdkArchitecture.ARM32);
var isSupportedSdkVersion = sdk != null;
// Determine whether the project is Raspberry compatible.
var isRaspberryCompatible = isNetCore &&
outputType == 1 && // 1=EXE
isSupportedSdkVersion;
// Return the properties.
return(new ProjectProperties()
{
Name = project.Name,
FullPath = project.FullName,
Configuration = project.ConfigurationManager.ActiveConfiguration.ConfigurationName,
IsNetCore = isNetCore,
SdkVersion = sdk?.Version,
OutputFolder = Path.Combine(projectFolder, project.ConfigurationManager.ActiveConfiguration.Properties.Item("OutputPath").Value.ToString()),
OutputFileName = (string)project.Properties.Item("OutputFileName").Value,
IsExecutable = outputType == 1, // 1=EXE
AssemblyName = project.Properties.Item("AssemblyName").Value.ToString(),
DebugEnabled = debugEnabled,
DebugConnectionName = debugConnectionName,
CommandLineArgs = commandLineArgs,
EnvironmentVariables = environmentVariables,
IsSupportedSdkVersion = isSupportedSdkVersion,
IsRaspberryCompatible = isRaspberryCompatible,
IsAspNet = isAspNet,
AspPort = aspPort,
AspLaunchBrowser = aspLaunchBrowser,
AspRelativeBrowserUri = aspRelativeBrowserUri
});
}
