public void MapTest()
{
var configStore = new MockConfigStore();
var configSection = new ConfigSection(TraceType, configStore, SectionName);
string rootPrefix = "Test.";
var policy = new RoleInstanceStatusMapHealthPolicy(configSection, rootPrefix);
var ri = new RoleInstance("1", RoleInstanceState.Unhealthy, DateTime.UtcNow);
string key = BaseRoleInstanceHealthPolicy.GetFullKeyName(
rootPrefix,
policy.Name,
RoleInstanceState.Unhealthy);
var defaultValue = policy.GetConfigValue(key);
HealthState output = policy.Apply(ri, HealthState.Ok);
Assert.AreEqual(output.ToString(), defaultValue, "Health state with default mapping");
const HealthState NewHealth = HealthState.Warning;
configStore.AddKeyValue(SectionName, key, NewHealth.ToString());
output = policy.Apply(ri, HealthState.Ok);
Assert.AreEqual(output, NewHealth, "New health state with updated mapping");
}
private static IConfigSection GetConfigSection()
{
var store = new MockConfigStore();
store.AddKeyValue(ConfigSectionName, WindowsAzureInfrastructureCoordinator.ConfigEnableLearningModeForJobTypeFormatKeyName.ToString(JobType.PlatformUpdateJob), true.ToString());
store.AddKeyValue(ConfigSectionName, WindowsAzureInfrastructureCoordinator.ConfigEnableLearningModeForJobTypeFormatKeyName.ToString(JobType.DeploymentUpdateJob), true.ToString());
var cs = new ConfigSection(traceType, store, ConfigSectionName);
return(cs);
}
public void MaxAllowedValueTest()
{
var configStore = new MockConfigStore();
var configSection = new ConfigSection(TraceType, configStore, SectionName);
string rootPrefix = "Test.";
var policy = new RoleInstanceStatusMaxAllowedHealthPolicy(configSection, rootPrefix);
var ri = new RoleInstance("1", RoleInstanceState.Unhealthy, DateTime.UtcNow);
string maxAllowedKeyName = BaseRoleInstanceHealthPolicy.GetFullKeyName(
rootPrefix,
policy.Name,
"HealthState");
configStore.UpdateKey(SectionName, maxAllowedKeyName, HealthState.Error.ToString());
var inputHealthState = HealthState.Unknown;
HealthState output = policy.Apply(ri, inputHealthState);
Assert.AreEqual(output, HealthState.Error, string.Format(CultureInfo.CurrentCulture, "Output health state clipped down from '{0}' to max allowed state '{1}'", inputHealthState, output));
inputHealthState = HealthState.Warning;
output = policy.Apply(ri, inputHealthState);
Assert.AreEqual(output, inputHealthState, "Output health state not clipped since it is below max allowed state");
inputHealthState = HealthState.Unknown;
string oldValue = configStore.GetValue(SectionName, maxAllowedKeyName);
configStore.UpdateKey(SectionName, maxAllowedKeyName, HealthState.Warning.ToString());
output = policy.Apply(ri, inputHealthState);
Assert.AreEqual(output, HealthState.Warning, string.Format(CultureInfo.CurrentCulture, "Output health state clipped down from '{0}' to max allowed state '{1}'", inputHealthState, output));
// revert to old value
configStore.UpdateKey(SectionName, maxAllowedKeyName, oldValue);
output = policy.Apply(ri, inputHealthState);
Assert.AreEqual(output.ToString(), oldValue, string.Format(CultureInfo.CurrentCulture, "Output health state clipped down from '{0}' to max allowed state '{1}'", inputHealthState, output));
}
public IInfrastructureCoordinator Create()
{
string configSectionName = typeof(WindowsAzureInfrastructureCoordinatorTest).Name + "ConfigSection";
const string ConfigJobPollingIntervalInSecondsKeyName = "WindowsAzure.JobPollingIntervalInSeconds";
var configStore = new MockConfigStore();
// loop faster in the ProcessManagementNotification loop of WindowsAzureInfrastructureCoordinator
configStore.AddKeyValue(configSectionName, ConfigJobPollingIntervalInSecondsKeyName, "1");
foreach (var pair in configSettings)
{
configStore.AddKeyValue(configSectionName, pair.Key, pair.Value);
}
var partitionId = Guid.NewGuid();
var jobBlockingPolicyManager = new MockJobBlockingPolicyManager();
var jobImpactManager = new JobImpactManager(new ConfigSection(traceType, configStore, configSectionName), queryClient);
var coordinator = new WindowsAzureInfrastructureCoordinator(
"mytenant",
managementClient,
this.agent,
configStore,
configSectionName,
partitionId,
0,
new MockHealthClient(),
jobBlockingPolicyManager,
jobImpactManager,
null);
return(coordinator);
}
public void TestInitialize()
{
this.configStore = new MockConfigStore();
this.translator = new ImpactTranslator(new ConfigSection(TraceType, this.configStore, ConfigSectionName));
}
public void InvalidKeysTest()
{
var configStore = new MockConfigStore();
var configSection = new ConfigSection(TraceType, configStore, SectionName);
string rootPrefix = "Test.";
var policy = new RoleInstanceStatusMaxAllowedHealthPolicy(configSection, rootPrefix);
var ri = new RoleInstance("1", RoleInstanceState.Unhealthy, DateTime.UtcNow);
string maxAllowedKeyName = BaseRoleInstanceHealthPolicy.GetFullKeyName(
rootPrefix,
policy.Name,
"HealthState");
#region typo in key name
string nonExistentKeyName = BaseRoleInstanceHealthPolicy.GetFullKeyName(
rootPrefix,
policy.Name,
"HealthStatt"); // note the type from "HealthState". user has typed this in his clustermanifest.xml
// set a default value to config store
configStore.UpdateKey(SectionName, maxAllowedKeyName, HealthState.Error.ToString());
// now update config store with an invalid key
configStore.UpdateKey(SectionName, nonExistentKeyName, HealthState.Unknown.ToString());
// fetch the correct key
var maxAllowedHealthState = configStore.GetValue(SectionName, maxAllowedKeyName);
const HealthState InputHealthState = HealthState.Unknown;
HealthState output = policy.Apply(ri, InputHealthState);
Assert.AreEqual(
output,
HealthState.Error,
string.Format(CultureInfo.CurrentCulture, "Output health state clipped down from '{0}' to default max allowed state '{1}'", InputHealthState, output));
// now user realizes that he has made a typo in his clustermanifest.xml
configStore.UpdateKey(SectionName, maxAllowedKeyName, HealthState.Unknown.ToString());
output = policy.Apply(ri, InputHealthState);
Assert.AreEqual(output, HealthState.Unknown, string.Format(CultureInfo.CurrentCulture, "Output health state not clipped down from '{0}' to max allowed state '{1}'", InputHealthState, output));
#endregion typo in key name
#region typo in key value
configStore.ClearKeys(); // revert to defaults
string defaultMaxHealthState = policy.GetConfigValue(maxAllowedKeyName);
Assert.AreEqual(defaultMaxHealthState, HealthState.Error.ToString());
// In this case, we know the default value as 'Error'.
// Ideally, we need to parse the enum and get the next higher enum to update the key with
configStore.UpdateKey(SectionName, maxAllowedKeyName, "Wurning");
output = policy.Apply(ri, InputHealthState);
// verify that Wurning didn't apply due to the typo. Since input was higher (Unknown) than the default, the output is also Unknown
// i.e. no clipping applied
Assert.AreEqual(
output,
InputHealthState,
string.Format(
CultureInfo.CurrentCulture,
"Output health state not altered from '{0}' to default max allowed state '{1}' as it may not be safe to apply default since input was already higher than default",
InputHealthState,
defaultMaxHealthState));
// provide a lower health state. Since there is an error in parsing, a warning should be returned
output = policy.Apply(ri, HealthState.Ok);
// verify that input of Ok is converted to Warning so that user observes that something is wrong and fixes his typo
Assert.AreEqual(
output,
HealthState.Warning,
string.Format(
CultureInfo.CurrentCulture,
"Output health state altered from '{0}' to '{1}' as there was an error applying policy",
HealthState.Ok,
output));
// user realizes typo and fixes it
configStore.UpdateKey(SectionName, maxAllowedKeyName, HealthState.Warning.ToString());
output = policy.Apply(ri, InputHealthState);
// verify that the new max allowed policy is in effect
Assert.AreEqual(
output,
HealthState.Warning,
string.Format(CultureInfo.CurrentCulture, "Output health state clipped down from '{0}' to max allowed state '{1}'", InputHealthState, output));
#endregion typo in key value
}
public async Task BasicTestAsync()
{
var configStore = new MockConfigStore();
var configSection = new ConfigSection(traceType, configStore, "Test");
CoordinatorFactoryArgs args = new CoordinatorFactoryArgs()
{
Agent = new MockInfrastructureAgentWrapper(),
ConfigSection = configSection,
PartitionId = Guid.NewGuid(),
ReplicaId = 42L,
ServiceName = new Uri("fabric:/System/InfrastructureService"),
};
configStore.AddKeyValue(args.ConfigSectionName, "DelayLoad.RetryDelayInSeconds", "1");
bool runAsyncCalled = false;
MockCoordinator mockCoordinator = new MockCoordinator();
mockCoordinator.RunAsyncHandler = (e, t) =>
{
runAsyncCalled = true;
return(Task.Delay(Timeout.InfiniteTimeSpan, t));
};
const string expectedCommandResult = "Hello";
mockCoordinator.RunCommandAsyncHandler = (admin, cmd, t, ct) =>
{
return(Task.FromResult(expectedCommandResult));
};
int failureCount = 2;
Func <CoordinatorFactoryArgs, IInfrastructureCoordinator> factory = a =>
{
if (failureCount-- > 0)
{
throw new Exception("Intentional failure");
}
return(mockCoordinator);
};
var coordinator = new DelayLoadCoordinator(
factory: factory,
factoryArgs: args,
healthClient: new MockHealthClient(),
configSection: configSection);
var cts = new CancellationTokenSource();
Task runAsyncTask = coordinator.RunAsync(0, cts.Token);
while (!runAsyncCalled)
{
await Task.Delay(100);
}
Verify.AreEqual(expectedCommandResult, await coordinator.RunCommandAsync(false, "", TimeSpan.FromSeconds(1), CancellationToken.None));
cts.Cancel();
try
{
await runAsyncTask;
}
catch (TaskCanceledException)
{
}
}
public async Task ServiceReplicaLifecycleTest()
{
var configStore = new MockConfigStore();
configStore.AddKeyValue("InfrastructureService", "CrashOnRunAsyncUnexpectedCompletion", "false");
var coordinator = new RunAsyncTestCoordinator();
var replica = new ServiceReplica(
new NullInfrastructureAgent(),
coordinator,
null,
false,
new ConfigSection(new TraceType("ServiceReplicaTest"), configStore, "InfrastructureService"));
replica.Initialize(new StatefulServiceInitializationParameters()
{
PartitionId = Guid.NewGuid(),
ReplicaId = Stopwatch.GetTimestamp(),
});
await replica.OpenAsync(ReplicaOpenMode.New, new MockStatefulServicePartition(), CancellationToken.None);
// Good coordinator: runs until cancelled
coordinator.RunAsyncHandler = async(epoch, token) => { await Task.Delay(Timeout.InfiniteTimeSpan, token); };
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Bad coordinator: returns immediately
coordinator.RunAsyncHandler = (epoch, token) => Task.FromResult(0);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Bad coordinator: synchronous unhandled exception
configStore.AddKeyValue("InfrastructureService", "CrashOnRunAsyncUnhandledException", "false");
coordinator.RunAsyncHandler = null;
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Bad coordinator: delayed (async) unhandled exception
coordinator.RunAsyncHandler = async(epoch, token) =>
{
await Task.Delay(100, token);
throw new InvalidOperationException();
};
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Old code - Bad coordinator: delayed (async) unhandled exception
configStore.AddKeyValue("InfrastructureService", "WrapRunAsync", "false");
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Old code - Good coordinator: runs until cancelled
coordinator.RunAsyncHandler = async(epoch, token) => { await Task.Delay(Timeout.InfiniteTimeSpan, token); };
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
}
