public void ArrayEquals_Byte()
{
Assert.True(NeonHelper.ArrayEquals(null, null));
Assert.True(NeonHelper.ArrayEquals(new byte[0], new byte[0]));
Assert.True(NeonHelper.ArrayEquals(new byte[] { 0, 1, 2, 3, 4 }, new byte[] { 0, 1, 2, 3, 4 }));
Assert.False(NeonHelper.ArrayEquals(new byte[0], null));
Assert.False(NeonHelper.ArrayEquals(null, new byte[0]));
Assert.False(NeonHelper.ArrayEquals(new byte[] { 0, 1, 2, 3, 100 }, new byte[] { 0, 1, 2, 3, 4 }));
Assert.False(NeonHelper.ArrayEquals(new byte[] { 0, 1, 2, 3 }, new byte[] { 0, 1, 2, 3, 4 }));
}
/// <summary>
/// Implements the <b>download</b> command.
/// </summary>
/// <param name="commandLine">The command line.</param>
public static void Download(CommandLine commandLine)
{
commandLine = commandLine.Shift(1);
if (commandLine.Arguments.Length != 2)
{
Console.WriteLine(usage);
Program.Exit(1);
}
// We're going to download the file as a byte array and then compare this
// against any existing file and only update the target file when the source
// and target differ. This will avoid writing to disk unnecessarily.
var sourceUri = commandLine.Arguments[0];
var targetPath = commandLine.Arguments[1];
try
{
using (var httpClient = new HttpClient())
{
var downloadBytes = httpClient.GetByteArrayAsync(sourceUri).Result;
if (File.Exists(targetPath))
{
var existingBytes = File.ReadAllBytes(targetPath);
if (!NeonHelper.ArrayEquals(downloadBytes, existingBytes))
{
File.WriteAllBytes(targetPath, downloadBytes);
}
}
else
{
File.WriteAllBytes(targetPath, downloadBytes);
}
}
}
catch (IOException e)
{
Console.Error.WriteLine($"*** ERROR: {NeonHelper.ExceptionError(e)}");
Program.Exit(1);
}
catch (Exception e)
{
// Note that we're not returning non-zero exit codes for non-I/O errors
// so that developers will be able to build when offline.
Console.Error.WriteLine($"*** ERROR: {NeonHelper.ExceptionError(e)}");
}
}
public async Task <bool> RunAsync(ReplayTest test)
{
var success = false;
if (test != ReplayTest.Nop)
{
// This ensures that the workflow has some history so that when
// Temporal restarts the workflow it will be treated as a replay
// instead of an initial execution.
await DecisionAsync();
}
switch (test)
{
case ReplayTest.Nop:
if (firstPass)
{
firstPass = false;
await Workflow.ForceReplayAsync();
}
else
{
// NOTE:
//
// The other Cadence clients (GOLANG, Java,...) always report
// IsReplaying=FALSE when a workflow with no history is restarted,
// which is what's happening in this case. This is a bit weird
// but is BY DESIGN but will probably be very rare in real life.
//
// https://github.com/uber-go/cadence-client/issues/821
success = !Workflow.IsReplaying;
}
break;
case ReplayTest.GetVersion:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.GetVersionAsync("change", Workflow.DefaultVersion, 1);
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.GetVersionAsync("change", Workflow.DefaultVersion, 1));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.WorkflowExecution:
var helloStub = Workflow.Client.NewWorkflowStub <IWorkflowReplayHello>();
if (firstPass)
{
firstPass = false;
originalValue = await helloStub.HelloAsync("Jeff");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await helloStub.HelloAsync("Jeff"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.MutableSideEffect:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.MutableSideEffectAsync(typeof(string), "value", () => "my-value");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.MutableSideEffectAsync(typeof(string), "value", () => "my-value"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.MutableSideEffectGeneric:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.MutableSideEffectAsync <string>("value", () => "my-value");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.MutableSideEffectAsync <string>("value", () => "my-value"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.SideEffect:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.SideEffectAsync(typeof(string), () => "my-value");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.SideEffectAsync(typeof(string), () => "my-value"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.SideEffectGeneric:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.SideEffectAsync <string>(() => "my-value");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.SideEffectAsync <string>(() => "my-value"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.NewGuid:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.NewGuidAsync();
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.NewGuidAsync());
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.NextRandomDouble:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.NextRandomDoubleAsync();
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.NextRandomDoubleAsync());
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.NextRandom:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.NextRandomAsync();
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.NextRandomAsync());
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.NextRandomMax:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.NextRandomAsync(10000);
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.NextRandomAsync(10000));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.NextRandomMinMax:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.NextRandomAsync(100, 1000);
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.NextRandomAsync(100, 1000));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.NextRandomBytes:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.NextRandomBytesAsync(32);
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = NeonHelper.ArrayEquals((byte[])originalValue, await Workflow.NextRandomBytesAsync(32));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.GetLastCompletionResult:
// $todo(jefflill):
//
// This case is a bit tricker to test. We'd need to schedule the
// workflow with a CRON schedule, let it run once and then perform
// this test for the second run.
//
// I'm going just fail the test here and skip the actual test case.
// Testing this case is not worth the trouble right now.
#if TODO
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.GetLastCompletionResultAsync <object>();
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.GetLastCompletionResultAsync <object>());
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
#else
success = false;
#endif
break;
case ReplayTest.GetIsSetLastCompletionResult:
if (firstPass)
{
firstPass = false;
originalValue = await Workflow.HasLastCompletionResultAsync();
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await Workflow.HasLastCompletionResultAsync());
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.ChildWorkflow:
var childStub = Workflow.NewChildWorkflowStub <IWorkflowReplayHello>();
if (firstPass)
{
firstPass = false;
originalValue = await childStub.HelloAsync("Jeff");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await childStub.HelloAsync("Jeff"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.Activity:
var activityStub = Workflow.NewActivityStub <IReplayActivity>();
if (firstPass)
{
firstPass = false;
originalValue = await activityStub.RunAsync("Hello World!");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await activityStub.RunAsync("Hello World!"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
case ReplayTest.LocalActivity:
var localActivityStub = Workflow.NewLocalActivityStub <IReplayActivity, ReplayActivity>();
if (firstPass)
{
firstPass = false;
originalValue = await localActivityStub.RunAsync("Hello World!");
await DecisionAsync();
await Workflow.ForceReplayAsync();
}
else
{
success = originalValue.Equals(await localActivityStub.RunAsync("Hello World!"));
success = success && Workflow.IsReplaying;
await DecisionAsync();
}
break;
}
return(await Task.FromResult(success));
}
/// <summary>
/// Decrypts one stream to another.
/// </summary>
/// <param name="encrypted">The encrypted input stream.</param>
/// <param name="decrypted">The decrypted output stream.</param>
public void DecryptStream(Stream encrypted, Stream decrypted)
{
Covenant.Requires <ArgumentNullException>(encrypted != null, nameof(encrypted));
Covenant.Requires <ArgumentNullException>(decrypted != null, nameof(decrypted));
// Wrap the input and output streams with [RelayStream] instances
// so that we can prevent the [CryptoStream] instances from disposing
// them (since these don't implement [leaveOpen]).
using (var hmac = new HMACSHA512(aes.Key))
{
byte[] persistedHMAC;
using (var encryptedRelay = new RelayStream(encrypted, leaveOpen: true))
{
using (var decryptedRelay = new RelayStream(decrypted, leaveOpen: true))
{
// Process the encrypted header.
using (var reader = new BinaryReader(encryptedRelay, Encoding.UTF8, leaveOpen: true))
{
// Read and verify the unencrypted magic number:
try
{
if (reader.ReadInt32() != Magic)
{
throw new FormatException($"The encrypted data was not generated by [{nameof(AesCipher)}].");
}
}
catch (IOException e)
{
throw new FormatException($"The encrypted data has been truncated or was not generated by [{nameof(AesCipher)}].", e);
}
// Read IV:
var ivLength = reader.ReadInt16();
if (ivLength < 0 || ivLength > 1024)
{
throw new FormatException("Invalid IV length.");
}
aes.IV = reader.ReadBytes(ivLength);
// Read the HMAC:
var hmacLength = reader.ReadInt16();
if (hmacLength < 0 || hmacLength > 1024)
{
throw new FormatException("Invalid HMAC length.");
}
persistedHMAC = reader.ReadBytes(hmacLength);
}
// Decrypt the input stream to the output while also
// computing the HMAC.
using (var hmacStream = new CryptoStream(encryptedRelay, hmac, CryptoStreamMode.Read))
{
using (var decryptor = aes.CreateDecryptor())
{
using (var decryptorStream = new CryptoStream(hmacStream, decryptor, CryptoStreamMode.Read))
{
// Decrypt the random padding.
using (var reader = new BinaryReader(decryptorStream, Encoding.UTF8, leaveOpen: true))
{
var paddingLength = reader.ReadInt16();
if (paddingLength < 0 || paddingLength > short.MaxValue)
{
throw new CryptographicException("The encrypted data has been tampered with or is corrupt: Invalid padding size.");
}
reader.ReadBytes(paddingLength);
}
// Decrypt the user data:
decryptorStream.CopyTo(decryptedRelay);
}
}
}
}
}
// Ensure that the encrypted data hasn't been tampered with by
// comparing the peristed and computed HMAC values.
if (!NeonHelper.ArrayEquals(persistedHMAC, hmac.Hash))
{
throw new CryptographicException("The encrypted data has been tampered with or is corrupt: The persisted and computed HMAC hashes don't match. ");
}
}
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
var localMD5 = string.Empty;
var remoteMD5 = "[unknown]";
var verifyTimer = new PolledTimer(verifyInterval, autoReset: true);
var periodicTask =
new AsyncPeriodicTask(
pollInterval,
onTaskAsync:
async() =>
{
log.LogDebug(() => "Starting poll");
log.LogDebug(() => "Fetching DNS answers MD5 from Consul.");
remoteMD5 = await consul.KV.GetStringOrDefault(HiveConst.ConsulDnsHostsMd5Key, terminator.CancellationToken);
if (remoteMD5 == null)
{
remoteMD5 = "[unknown]";
}
var verify = verifyTimer.HasFired;
if (verify)
{
// Under normal circumstances, we should never see the reload signal file
// here because the [neon-dns-loader] service should have deleted it after
// handling the last change signal.
//
// This probably means that [neon-dns-loader] is not running or if this service
// is configured with POLL_INTERVAL being so short that [neon-dns-loader]
// hasn't had a chance to handle the previous signal.
if (File.Exists(reloadSignalPath))
{
log.LogWarn("[neon-dns-loader] service doesn't appear to be running because the reload signal file is present.");
}
}
if (!verify && localMD5 == remoteMD5)
{
log.LogDebug(() => "DNS answers are unchanged.");
}
else
{
if (localMD5 == remoteMD5)
{
log.LogDebug(() => "DNS answers have not changed but we're going to verify that we have the correct hosts anyway.");
}
else
{
log.LogDebug(() => "DNS answers have changed.");
}
log.LogDebug(() => "Fetching DNS answers.");
var hostsTxt = await consul.KV.GetStringOrDefault(HiveConst.ConsulDnsHostsKey, terminator.CancellationToken);
if (hostsTxt == null)
{
log.LogWarn(() => "DNS answers do not exist on Consul. Is [neon-dns-mon] functioning properly?");
}
else
{
var marker = "# -------- NEON-DNS --------";
// We have the host entries from Consul. We need to add these onto the
// end [/etc/powserdns/hosts], replacing any host entries written during
// a previous run.
//
// We're going to use the special marker line:
//
// # ---DYNAMIC-HOSTS---
//
// to separate the built-in hosts (above the line) from the dynamic hosts
// we're generating here (which will be below the line). Note that this
// line won't exist the first time this service runs, so we'll just add it.
//
// Note that it's possible that the PowerDNS Recursor might be reading this
// file while we're trying to write it. We're going to treat these as a
// transient errors and retry.
var retry = new LinearRetryPolicy(typeof(IOException), maxAttempts: 5, retryInterval: TimeSpan.FromSeconds(1));
await retry.InvokeAsync(
async() =>
{
using (var stream = new FileStream(powerDnsHostsPath, FileMode.Open, FileAccess.ReadWrite))
{
// Read a copy of the hosts file as bytes so we can compare
// the old version with the new one generated below for changes.
var orgHostBytes = stream.ReadToEnd();
stream.Position = 0;
// Generate the new hosts file.
var sbHosts = new StringBuilder();
// Read the hosts file up to but not including the special marker
// line (if it's present).
using (var reader = new StreamReader(stream, Encoding.UTF8, true, 32 * 1024, leaveOpen: true))
{
foreach (var line in reader.Lines())
{
if (line.StartsWith(marker))
{
break;
}
sbHosts.AppendLine(line);
}
}
// Strip any trailing whitespace from the hosts file so we'll
// be able to leave a nice blank line between the end of the
// original file and the special marker line.
var text = sbHosts.ToString().TrimEnd();
sbHosts.Clear();
sbHosts.AppendLine(text);
// Append the marker line, followed by dynamic host
// entries we downloaded from Consul.
sbHosts.AppendLine();
sbHosts.AppendLine(marker);
sbHosts.AppendLine();
sbHosts.Append(hostsTxt);
// Generate the new host file bytes, taking care to ensure that
// we're using Linux style line endings and then update the
// hosts file if anything changed.
var hostsText = NeonHelper.ToLinuxLineEndings(sbHosts.ToString());
var newHostBytes = Encoding.UTF8.GetBytes(hostsText);
if (NeonHelper.ArrayEquals(orgHostBytes, newHostBytes))
{
log.LogDebug(() => $"[{powerDnsHostsPath}] file is up-to-date.");
}
else
{
log.LogDebug(() => $"[{powerDnsHostsPath}] is being updated.");
stream.Position = 0;
stream.SetLength(0);
stream.Write(newHostBytes);
// Signal to the local [neon-dns-loader] systemd service that it needs
// to have PowerDNS Recursor reload the hosts file.
File.WriteAllText(reloadSignalPath, "reload now");
}
}
log.LogDebug(() => "Finished poll");
await Task.CompletedTask;
});
// We've successfully synchronized the local hosts file with
// the Consul DNS settings.
localMD5 = remoteMD5;
}
}
return(await Task.FromResult(false));
},
onExceptionAsync:
async e =>
{
log.LogError(e);
return(await Task.FromResult(false));
},
onTerminateAsync:
async() =>
{
log.LogInfo(() => "Terminating");
await Task.CompletedTask;
});
terminator.AddDisposable(periodicTask);
await periodicTask.Run();
}
/// <summary>
/// Implements the <b>download-const-uri</b> command.
/// </summary>
/// <param name="commandLine">The command line.</param>
public static void DownloadConstUri(CommandLine commandLine)
{
commandLine = commandLine.Shift(1);
if (commandLine.Arguments.Length != 4)
{
Console.WriteLine(usage);
Program.Exit(1);
}
// We're going to download the file as a byte array and then compare this
// against any existing file and only update the target file when the source
// and target differ. This will avoid writing to disk unnecessarily.
var assemblyPath = commandLine.Arguments[0];
var typeName = commandLine.Arguments[1];
var constName = commandLine.Arguments[2];
var targetPath = commandLine.Arguments[3];
Console.WriteLine("-------------------------------------------------------------------------------");
Console.WriteLine(commandLine);
Console.WriteLine();
Console.WriteLine($"AssemblyPath: {assemblyPath}");
Console.WriteLine($"TypeName: {typeName}");
Console.WriteLine($"ConstName {constName}");
Console.WriteLine($"TargetPath: {targetPath}");
// Load the referenced assembly and locate the type and extract the constant URI.
var sourceUri = (string)null;
var assembly = Assembly.LoadFrom(assemblyPath);
var type = assembly.GetType(typeName);
if (type == null)
{
throw new Exception($"Cannot locate type [{typeName}] within the [{assemblyPath}] assembly.");
}
var field = type.GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)
.Where(field => field.Name == constName && field.IsLiteral)
.FirstOrDefault();
if (field == null)
{
throw new Exception($"Cannot locate constant [{constName}] within the [{typeName}] type.");
}
if (field.FieldType != typeof(string))
{
throw new Exception($"Cannot constant [{typeName}.{constName}] is not a [string].");
}
sourceUri = (string)field.GetValue(null);
Console.WriteLine($"SourceUri: {sourceUri}");
try
{
using (var httpClient = new HttpClient())
{
var downloadBytes = httpClient.GetByteArrayAsync(sourceUri).Result;
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
if (File.Exists(targetPath))
{
var existingBytes = File.ReadAllBytes(targetPath);
if (!NeonHelper.ArrayEquals(downloadBytes, existingBytes))
{
File.WriteAllBytes(targetPath, downloadBytes);
}
}
else
{
File.WriteAllBytes(targetPath, downloadBytes);
}
}
Console.WriteLine();
Console.WriteLine("Manifest downloaded successfully.");
Console.WriteLine("-------------------------------------------------------------------------------");
}
catch (AggregateException e)
{
var innerException = e.InnerExceptions.First();
var innerExceptionType = innerException.GetType();
if (innerExceptionType == typeof(SocketException) ||
innerExceptionType == typeof(HttpRequestException))
{
Console.Error.WriteLine();
Console.Error.WriteLine($"*** WARNING: Unable to download [{sourceUri}]: {NeonHelper.ExceptionError(innerException)}");
Console.Error.WriteLine();
}
else
{
Console.Error.WriteLine($"*** ERROR: {NeonHelper.ExceptionError(innerException)}");
Program.Exit(1);
}
}
catch (SocketException e)
{
Console.Error.WriteLine($"*** WARNING: Unable to download [{targetPath}]: {NeonHelper.ExceptionError(e)}");
}
catch (HttpRequestException e)
{
Console.Error.WriteLine($"*** WARNING: Unable to download [{targetPath}]: {NeonHelper.ExceptionError(e)}");
}
catch (IOException e)
{
Console.Error.WriteLine($"*** ERROR: {NeonHelper.ExceptionError(e)}");
Program.Exit(1);
}
catch (Exception e)
{
// Note that we're not returning non-zero exit codes for non-I/O errors
// so that developers will be able to build when offline.
Console.Error.WriteLine($"*** ERROR: {NeonHelper.ExceptionError(e)}");
}
}