/// <summary>
/// Processes a cluster that is in the "Deleting" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task <Cluster> ProcessDeletingClusterAsync(Cluster cluster)
{
ClusterOperationStatus deleteStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (deleteStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
// hopefully shouldn't ever get here
return(new Cluster(ClusterStatus.Remove, cluster));
case ClusterOperationStatus.Deleting:
// still deleting, no updates necessary
return(cluster);
case ClusterOperationStatus.ClusterNotFound:
// If the cluster can't be found, it's been deleted.
ServiceEventSource.Current.ServiceMessage(this, "Cluster successfully deleted: {0}.", cluster.Address);
return(new Cluster(ClusterStatus.Deleted, cluster));
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
// Failed to delete, set its status to "remove" to try again.
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to delete: {0}.", cluster.Address);
return(new Cluster(ClusterStatus.Remove, cluster));
}
return(cluster);
}
/// <summary>
/// Processes a cluster in the "Remove" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task <Cluster> ProcessRemoveClusterAsync(Cluster cluster)
{
ClusterOperationStatus removeStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (removeStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
// In any of these cases, instruct the operator to delete the cluster.
ServiceEventSource.Current.ServiceMessage(this, "Deleting cluster {0}.", cluster.Address);
await this.clusterOperator.DeleteClusterAsync(cluster.InternalName);
return(new Cluster(ClusterStatus.Deleting, cluster));
case ClusterOperationStatus.Deleting:
// If the cluster is now being deleted, update the status accordingly.
return(new Cluster(ClusterStatus.Deleting, cluster));
case ClusterOperationStatus.ClusterNotFound:
// Cluster was already deleted.
return(new Cluster(ClusterStatus.Deleted, cluster));
}
return(cluster);
}
/// <summary>
/// Processes clusters in the "Ready" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task <Cluster> ProcessReadyClusterAsync(Cluster cluster)
{
// Check for expiration. If the cluster has expired, mark it for removal.
if (DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime() >= this.config.MaximumClusterUptime)
{
ServiceEventSource.Current.ServiceMessage(this, "Cluster expired: {0}", cluster.Address);
return(new Cluster(ClusterStatus.Remove, cluster));
}
ClusterOperationStatus readyStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (readyStatus)
{
case ClusterOperationStatus.Deleting:
// If the cluster was deleted, mark the state accordingly
return(new Cluster(ClusterStatus.Deleting, cluster));
case ClusterOperationStatus.ClusterNotFound:
// Cluster was already deleted.
return(new Cluster(ClusterStatus.Deleted, cluster));
}
try
{
// Update the application and service count for the cluster.
int deployedApplications = await this.applicationDeployService.GetApplicationCountAsync(cluster.Address, ClusterConnectionPort);
int deployedServices = await this.applicationDeployService.GetServiceCountAsync(cluster.Address, ClusterConnectionPort);
return(new Cluster(
cluster.InternalName,
cluster.Status,
deployedApplications,
deployedServices,
cluster.Address,
cluster.Ports,
cluster.Users,
cluster.CreatedOn));
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Unable to determine application and service count. Cluster: {0}. Error: {1}",
cluster.Address,
e.GetActualMessage());
}
return(cluster);
}
public async Task <ClusterOperationStatus> GetClusterStatusAsync(string name)
{
IReliableDictionary <string, ClusterOperationStatus> clusters =
await this.stateManager.GetOrAddAsync <IReliableDictionary <string, ClusterOperationStatus> >(new Uri("fakeclusterops:/clusters"));
IReliableDictionary <string, DateTimeOffset> clusterCreateDelay =
await this.stateManager.GetOrAddAsync <IReliableDictionary <string, DateTimeOffset> >(new Uri("fakeclusterops:/clusterCreateDelay"));
using (ITransaction tx = this.stateManager.CreateTransaction())
{
ClusterOperationStatus status = (await clusters.TryGetValueAsync(tx, name)).Value;
DateTimeOffset clusterDelay = (await clusterCreateDelay.TryGetValueAsync(tx, name)).Value;
ClusterOperationStatus newStatus = ClusterOperationStatus.Ready;
switch (status)
{
case ClusterOperationStatus.Creating:
if (DateTimeOffset.UtcNow > clusterDelay)
{
newStatus = ClusterOperationStatus.Ready;
}
break;
case ClusterOperationStatus.Ready:
newStatus = ClusterOperationStatus.Ready;
break;
case ClusterOperationStatus.Deleting:
newStatus = ClusterOperationStatus.ClusterNotFound;
break;
}
await clusters.SetAsync(tx, name, newStatus);
await tx.CommitAsync();
return(newStatus);
}
}
private async Task <IOperationStatus> BuildStatusAsync(ClusterOperationDescription description, IOperationContext context, ClusterErrorDetails errorDetails)
{
var upgradeProgressTask = this.fabricClientWrapper.GetFabricUpgradeProgressAsync(Constants.MaxOperationTimeout, context.CancellationToken);
var nodeListQueryTask = this.fabricClientWrapper.GetNodeListAsync(Constants.MaxOperationTimeout, context.CancellationToken);
// Only query for system services when WRP needs to adjust the replica set size
Task <Dictionary <string, ServiceRuntimeDescription> > systemServiceSizeQueryTask = null;
if (description != null &&
description.SystemServiceDescriptionsToSet != null &&
description.SystemServiceDescriptionsToSet.Any())
{
systemServiceSizeQueryTask =
this.fabricClientWrapper.GetSystemServiceRuntimeDescriptionsAsync(Constants.MaxOperationTimeout, context.CancellationToken);
}
await Task.WhenAll(
nodeListQueryTask,
upgradeProgressTask,
systemServiceSizeQueryTask ?? Task.FromResult <Dictionary <string, ServiceRuntimeDescription> >(null));
FabricUpgradeProgress currentUpgradeProgress = GetResultFromTask(upgradeProgressTask);
NodeList nodeList = FilterPrimaryNodeTypesStatus(GetResultFromTask(nodeListQueryTask), description?.PrimaryNodeTypes);
Dictionary <string, ServiceRuntimeDescription> systemServicesRuntimeDescriptions = GetResultFromTask(systemServiceSizeQueryTask);
List <PaasNodeStatusInfo> nodesDisabled = null;
List <PaasNodeStatusInfo> nodesEnabled = null;
if (description != null &&
nodeList != null)
{
Trace.WriteNoise(TraceType, "BuildStatusAsync: Begin this.nodeStatusManager.ProcessNodeQuery.");
await this.nodeStatusManager.ProcessNodeQuery(nodeList, Constants.KvsCommitTimeout, context.CancellationToken);
Trace.WriteNoise(TraceType, "BuildStatusAsync: End this.nodeStatusManager.ProcessNodeQuery.");
// Send back the list of nodes that are disabled
if (description.NodesToDisabled != null)
{
nodesDisabled = new List <PaasNodeStatusInfo>();
// Send back the Instance# for the requested disabled nodes
foreach (var nodeToDisable in description.NodesToDisabled)
{
var matchingNodeStatus =
nodeList.FirstOrDefault(
node =>
string.Equals(node.NodeName, nodeToDisable.NodeName,
StringComparison.OrdinalIgnoreCase));
if (matchingNodeStatus != null && matchingNodeStatus.NodeStatus == NodeStatus.Disabled)
{
var nodeDisabled = new PaasNodeStatusInfo(nodeToDisable)
{
NodeState = NodeState.Disabled
};
nodesDisabled.Add(nodeDisabled);
Trace.WriteInfo(TraceType, "BuildStatusAsync: Node has been successfully disabled. {0}", nodeDisabled);
}
}
}
// Send back the list of nodes that are Enabled
if (description.NodesToEnabled != null)
{
nodesEnabled = new List <PaasNodeStatusInfo>();
// Send back the Instance# for the requested up nodes
foreach (var nodeToEnable in description.NodesToEnabled)
{
var matchingNodeStatus =
nodeList.FirstOrDefault(
node =>
string.Equals(node.NodeName, nodeToEnable.NodeName,
StringComparison.OrdinalIgnoreCase));
// Since a node can be enabled and can still be down, we infer enabled status instead.
if (matchingNodeStatus != null &&
(matchingNodeStatus.NodeStatus != NodeStatus.Disabling) &&
(matchingNodeStatus.NodeStatus != NodeStatus.Disabled) &&
(matchingNodeStatus.NodeStatus != NodeStatus.Enabling))
{
var nodeEnabled = new PaasNodeStatusInfo(nodeToEnable);
nodeEnabled.NodeState = NodeState.Enabled;
nodesEnabled.Add(nodeEnabled);
Trace.WriteInfo(TraceType, "BuildStatusAsync: Node has been successfully enabled. {0}", nodeEnabled);
}
}
}
}
Trace.WriteNoise(TraceType, "BuildStatusAsync: Begin this.nodeStatusManager.GetNodeStates.");
var nodesStatus = await this.nodeStatusManager.GetNodeStates(Constants.KvsCommitTimeout, context.CancellationToken);
Trace.WriteNoise(TraceType, "BuildStatusAsync: End this.nodeStatusManager.GetNodeStates.");
var status = new ClusterOperationStatus(description)
{
DisabledNodes = nodesDisabled,
EnabledNodes = nodesEnabled,
NodesStatus = nodesStatus,
SystemServiceDescriptions = systemServicesRuntimeDescriptions,
};
if (currentUpgradeProgress != null)
{
status.Progress = JObject.FromObject(currentUpgradeProgress, this.serializer);
}
if (errorDetails != null)
{
status.ErrorDetails = JObject.FromObject(errorDetails, this.serializer);
}
return(status);
}
/// <summary>
/// Processes a cluster in the "Creating" stage.
/// </summary>
/// <param name="cluster"></param>
/// <returns></returns>
private async Task <Cluster> ProcessCreatingClusterAsync(Cluster cluster)
{
ClusterOperationStatus creatingStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.InternalName);
switch (creatingStatus)
{
case ClusterOperationStatus.Creating:
// Still creating, no updates necessary.
return(cluster);
case ClusterOperationStatus.Ready:
// Cluster is ready to go.
try
{
// Queue up sample application deployment
await this.applicationDeployService.QueueApplicationDeploymentAsync(cluster.Address, ClusterConnectionPort);
}
catch (Exception e)
{
// couldn't queue samples for deployment, but that shouldn't prevent starting the cluster.
ServiceEventSource.Current.ServiceMessage(
this,
"Failed to queue sample deployment. Cluster: {0} Error: {1}",
cluster.Address,
e.GetActualMessage());
}
ServiceEventSource.Current.ServiceMessage(
this,
"Cluster is ready: {0}",
cluster.Address);
return(new Cluster(
cluster.InternalName,
ClusterStatus.Ready,
cluster.AppCount,
cluster.ServiceCount,
cluster.Address,
cluster.Ports,
new List <ClusterUser>(cluster.Users),
DateTimeOffset.UtcNow));
case ClusterOperationStatus.CreateFailed:
// Failed to create the cluster, so remove it.
// Processing will add a new one in the next iteration if we need more.
ServiceEventSource.Current.ServiceMessage(this, "Cluster failed to create: {0}", cluster.Address);
return(new Cluster(ClusterStatus.Remove, cluster));
case ClusterOperationStatus.Deleting:
// Cluster is being deleted.
return(new Cluster(ClusterStatus.Deleting, cluster));
case ClusterOperationStatus.ClusterNotFound:
// Cluster was deleted before it finished being created.
return(new Cluster(ClusterStatus.Deleted, cluster));
default:
return(cluster);
}
}
/// <summary>
/// Processes a cluster based on its current state.
/// </summary>
/// <returns></returns>
internal async Task ProcessClusterStatusAsync(Cluster cluster)
{
switch (cluster.Status)
{
case ClusterStatus.New:
Random random = new Random();
cluster.Address = await this.clusterOperator.CreateClusterAsync(random.Next().ToString());
cluster.Status = ClusterStatus.Creating;
break;
case ClusterStatus.Creating:
ClusterOperationStatus creatingStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (creatingStatus)
{
case ClusterOperationStatus.Creating:
// still creating
break;
case ClusterOperationStatus.Ready:
cluster.Ports = await this.clusterOperator.GetClusterPortsAsync(cluster.Address);
cluster.CreatedOn = DateTimeOffset.UtcNow;
cluster.Status = ClusterStatus.Ready;
break;
case ClusterOperationStatus.CreateFailed:
cluster.Status = ClusterStatus.New;
break;
case ClusterOperationStatus.Deleting:
cluster.Status = ClusterStatus.Deleting;
break;
}
break;
case ClusterStatus.Ready:
if (DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime() >= this.Config.MaxClusterUptime)
{
await this.clusterOperator.DeleteClusterAsync(cluster.Address);
cluster.Status = ClusterStatus.Deleting;
}
ClusterOperationStatus readyStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (readyStatus)
{
case ClusterOperationStatus.Deleting:
cluster.Status = ClusterStatus.Deleting;
break;
}
//TODO: update application and service count
break;
case ClusterStatus.Remove:
ClusterOperationStatus removeStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (removeStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
await this.clusterOperator.DeleteClusterAsync(cluster.Address);
cluster.Status = ClusterStatus.Deleting;
break;
case ClusterOperationStatus.Deleting:
cluster.Status = ClusterStatus.Deleting;
break;
}
break;
case ClusterStatus.Deleting:
ClusterOperationStatus deleteStatus = await this.clusterOperator.GetClusterStatusAsync(cluster.Address);
switch (deleteStatus)
{
case ClusterOperationStatus.Creating:
case ClusterOperationStatus.Ready:
await this.clusterOperator.DeleteClusterAsync(cluster.Address);
break;
case ClusterOperationStatus.Deleting:
break; // still in progress
case ClusterOperationStatus.ClusterNotFound:
cluster.Status = ClusterStatus.Deleted;
break;
case ClusterOperationStatus.CreateFailed:
case ClusterOperationStatus.DeleteFailed:
cluster.Status = ClusterStatus.Remove;
break;
}
break;
}
}
