private static IConfigSection GetConfigSection()
{
var store = new MockConfigStore();
store.AddKeyValue(ConfigSectionName, WindowsAzureInfrastructureCoordinator.ConfigEnableLearningModeForJobTypeFormatKeyName.ToString(JobType.PlatformUpdateJob), true.ToString());
store.AddKeyValue(ConfigSectionName, WindowsAzureInfrastructureCoordinator.ConfigEnableLearningModeForJobTypeFormatKeyName.ToString(JobType.DeploymentUpdateJob), true.ToString());
var cs = new ConfigSection(traceType, store, ConfigSectionName);
return(cs);
}
public void MapTest()
{
var configStore = new MockConfigStore();
var configSection = new ConfigSection(TraceType, configStore, SectionName);
string rootPrefix = "Test.";
var policy = new RoleInstanceStatusMapHealthPolicy(configSection, rootPrefix);
var ri = new RoleInstance("1", RoleInstanceState.Unhealthy, DateTime.UtcNow);
string key = BaseRoleInstanceHealthPolicy.GetFullKeyName(
rootPrefix,
policy.Name,
RoleInstanceState.Unhealthy);
var defaultValue = policy.GetConfigValue(key);
HealthState output = policy.Apply(ri, HealthState.Ok);
Assert.AreEqual(output.ToString(), defaultValue, "Health state with default mapping");
const HealthState NewHealth = HealthState.Warning;
configStore.AddKeyValue(SectionName, key, NewHealth.ToString());
output = policy.Apply(ri, HealthState.Ok);
Assert.AreEqual(output, NewHealth, "New health state with updated mapping");
}
public IInfrastructureCoordinator Create()
{
string configSectionName = typeof(WindowsAzureInfrastructureCoordinatorTest).Name + "ConfigSection";
const string ConfigJobPollingIntervalInSecondsKeyName = "WindowsAzure.JobPollingIntervalInSeconds";
var configStore = new MockConfigStore();
// loop faster in the ProcessManagementNotification loop of WindowsAzureInfrastructureCoordinator
configStore.AddKeyValue(configSectionName, ConfigJobPollingIntervalInSecondsKeyName, "1");
foreach (var pair in configSettings)
{
configStore.AddKeyValue(configSectionName, pair.Key, pair.Value);
}
var partitionId = Guid.NewGuid();
var jobBlockingPolicyManager = new MockJobBlockingPolicyManager();
var jobImpactManager = new JobImpactManager(new ConfigSection(traceType, configStore, configSectionName), queryClient);
var coordinator = new WindowsAzureInfrastructureCoordinator(
"mytenant",
managementClient,
this.agent,
configStore,
configSectionName,
partitionId,
0,
new MockHealthClient(),
jobBlockingPolicyManager,
jobImpactManager,
null);
return(coordinator);
}
public async Task BasicTestAsync()
{
var configStore = new MockConfigStore();
var configSection = new ConfigSection(traceType, configStore, "Test");
CoordinatorFactoryArgs args = new CoordinatorFactoryArgs()
{
Agent = new MockInfrastructureAgentWrapper(),
ConfigSection = configSection,
PartitionId = Guid.NewGuid(),
ReplicaId = 42L,
ServiceName = new Uri("fabric:/System/InfrastructureService"),
};
configStore.AddKeyValue(args.ConfigSectionName, "DelayLoad.RetryDelayInSeconds", "1");
bool runAsyncCalled = false;
MockCoordinator mockCoordinator = new MockCoordinator();
mockCoordinator.RunAsyncHandler = (e, t) =>
{
runAsyncCalled = true;
return(Task.Delay(Timeout.InfiniteTimeSpan, t));
};
const string expectedCommandResult = "Hello";
mockCoordinator.RunCommandAsyncHandler = (admin, cmd, t, ct) =>
{
return(Task.FromResult(expectedCommandResult));
};
int failureCount = 2;
Func <CoordinatorFactoryArgs, IInfrastructureCoordinator> factory = a =>
{
if (failureCount-- > 0)
{
throw new Exception("Intentional failure");
}
return(mockCoordinator);
};
var coordinator = new DelayLoadCoordinator(
factory: factory,
factoryArgs: args,
healthClient: new MockHealthClient(),
configSection: configSection);
var cts = new CancellationTokenSource();
Task runAsyncTask = coordinator.RunAsync(0, cts.Token);
while (!runAsyncCalled)
{
await Task.Delay(100);
}
Verify.AreEqual(expectedCommandResult, await coordinator.RunCommandAsync(false, "", TimeSpan.FromSeconds(1), CancellationToken.None));
cts.Cancel();
try
{
await runAsyncTask;
}
catch (TaskCanceledException)
{
}
}
public async Task ServiceReplicaLifecycleTest()
{
var configStore = new MockConfigStore();
configStore.AddKeyValue("InfrastructureService", "CrashOnRunAsyncUnexpectedCompletion", "false");
var coordinator = new RunAsyncTestCoordinator();
var replica = new ServiceReplica(
new NullInfrastructureAgent(),
coordinator,
null,
false,
new ConfigSection(new TraceType("ServiceReplicaTest"), configStore, "InfrastructureService"));
replica.Initialize(new StatefulServiceInitializationParameters()
{
PartitionId = Guid.NewGuid(),
ReplicaId = Stopwatch.GetTimestamp(),
});
await replica.OpenAsync(ReplicaOpenMode.New, new MockStatefulServicePartition(), CancellationToken.None);
// Good coordinator: runs until cancelled
coordinator.RunAsyncHandler = async(epoch, token) => { await Task.Delay(Timeout.InfiniteTimeSpan, token); };
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Bad coordinator: returns immediately
coordinator.RunAsyncHandler = (epoch, token) => Task.FromResult(0);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Bad coordinator: synchronous unhandled exception
configStore.AddKeyValue("InfrastructureService", "CrashOnRunAsyncUnhandledException", "false");
coordinator.RunAsyncHandler = null;
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Bad coordinator: delayed (async) unhandled exception
coordinator.RunAsyncHandler = async(epoch, token) =>
{
await Task.Delay(100, token);
throw new InvalidOperationException();
};
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Old code - Bad coordinator: delayed (async) unhandled exception
configStore.AddKeyValue("InfrastructureService", "WrapRunAsync", "false");
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
// Old code - Good coordinator: runs until cancelled
coordinator.RunAsyncHandler = async(epoch, token) => { await Task.Delay(Timeout.InfiniteTimeSpan, token); };
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.ChangeRoleAsync(ReplicaRole.ActiveSecondary, CancellationToken.None);
await replica.ChangeRoleAsync(ReplicaRole.Primary, CancellationToken.None);
await Task.Delay(500);
await replica.CloseAsync(CancellationToken.None);
}