public override async Task <IOperationStatus> CreateOperationStatusAsync(IOperationDescription operationDescription, IOperationContext context)
{
context.ThrowIfNull(nameof(context));
ClusterOperationDescription description = operationDescription == null ? null : operationDescription as ClusterOperationDescription;
Trace.WriteInfo(TraceType, "CreateOperationStatusAsync: begin");
IOperationStatus status = null;
try
{
Trace.WriteInfo(TraceType, "CreateOperationStatusAsync: performing operation");
var errorDetails = await DoOperation(description, context);
Trace.WriteInfo(TraceType, "CreateOperationStatusAsync: building status");
status = await BuildStatusAsync(description, context, errorDetails);
}
catch (Exception ex)
{
Trace.WriteError(this.TraceType, $"CreateOperationStatusAsync: Exception encountered: {ex}");
}
Trace.WriteInfo(TraceType, "CreateOperationStatusAsync: end");
return(status);
}
private async Task <ClusterErrorDetails> DoOperation(ClusterOperationDescription description, IOperationContext context)
{
if (description == null)
{
Trace.WriteInfo(TraceType, "CreateOperationStatusAsync: operation description is null. No operation to perform.");
return(null);
}
var upgradeTask = ClusterTask(
() => this.ProcessClusterUpgradeAsync(description.UpgradeDescription, context.CancellationToken),
ClusterOperation.ProcessClusterUpgrade);
var enableNodesTask = ClusterTask(
() => this.fabricClientWrapper.EnableNodesAsync(description.NodesToEnabled, Constants.MaxOperationTimeout, context.CancellationToken),
ClusterOperation.EnableNodes);
var disableNodesTask = ClusterTask(
() => this.fabricClientWrapper.DisableNodesAsync(description.NodesToDisabled, Constants.MaxOperationTimeout, context.CancellationToken),
ClusterOperation.DisableNodes);
var updateSystemServiceSizeTask = ClusterTask(
() => this.fabricClientWrapper.UpdateSystemServicesDescriptionAsync(
description.SystemServiceDescriptionsToSet,
Constants.MaxOperationTimeout,
context.CancellationToken),
ClusterOperation.UpdateSystemServicesReplicaSetSize);
var nodeListForAutoScaleTask = ClusterTask(
() => this.nodeStatusManager.ProcessWRPResponse(
description.ProcessedNodesStatus,
Constants.KvsCommitTimeout,
context.CancellationToken),
ClusterOperation.ProcessNodesStatus);
var errors = await Task.WhenAll(
upgradeTask,
enableNodesTask,
disableNodesTask,
nodeListForAutoScaleTask,
updateSystemServiceSizeTask);
var taskErrors = errors.Where(item => item != null).ToList();
if (!taskErrors.Any())
{
return(null);
}
return(new ClusterErrorDetails()
{
Errors = new List <ClusterOperationError>(taskErrors)
});
}
private async Task <IOperationStatus> BuildStatusAsync(ClusterOperationDescription description, IOperationContext context, ClusterErrorDetails errorDetails)
{
var upgradeProgressTask = this.fabricClientWrapper.GetFabricUpgradeProgressAsync(Constants.MaxOperationTimeout, context.CancellationToken);
var nodeListQueryTask = this.fabricClientWrapper.GetNodeListAsync(Constants.MaxOperationTimeout, context.CancellationToken);
// Only query for system services when WRP needs to adjust the replica set size
Task <Dictionary <string, ServiceRuntimeDescription> > systemServiceSizeQueryTask = null;
if (description != null &&
description.SystemServiceDescriptionsToSet != null &&
description.SystemServiceDescriptionsToSet.Any())
{
systemServiceSizeQueryTask =
this.fabricClientWrapper.GetSystemServiceRuntimeDescriptionsAsync(Constants.MaxOperationTimeout, context.CancellationToken);
}
await Task.WhenAll(
nodeListQueryTask,
upgradeProgressTask,
systemServiceSizeQueryTask ?? Task.FromResult <Dictionary <string, ServiceRuntimeDescription> >(null));
FabricUpgradeProgress currentUpgradeProgress = GetResultFromTask(upgradeProgressTask);
NodeList nodeList = FilterPrimaryNodeTypesStatus(GetResultFromTask(nodeListQueryTask), description?.PrimaryNodeTypes);
Dictionary <string, ServiceRuntimeDescription> systemServicesRuntimeDescriptions = GetResultFromTask(systemServiceSizeQueryTask);
List <PaasNodeStatusInfo> nodesDisabled = null;
List <PaasNodeStatusInfo> nodesEnabled = null;
if (description != null &&
nodeList != null)
{
Trace.WriteNoise(TraceType, "BuildStatusAsync: Begin this.nodeStatusManager.ProcessNodeQuery.");
await this.nodeStatusManager.ProcessNodeQuery(nodeList, Constants.KvsCommitTimeout, context.CancellationToken);
Trace.WriteNoise(TraceType, "BuildStatusAsync: End this.nodeStatusManager.ProcessNodeQuery.");
// Send back the list of nodes that are disabled
if (description.NodesToDisabled != null)
{
nodesDisabled = new List <PaasNodeStatusInfo>();
// Send back the Instance# for the requested disabled nodes
foreach (var nodeToDisable in description.NodesToDisabled)
{
var matchingNodeStatus =
nodeList.FirstOrDefault(
node =>
string.Equals(node.NodeName, nodeToDisable.NodeName,
StringComparison.OrdinalIgnoreCase));
if (matchingNodeStatus != null && matchingNodeStatus.NodeStatus == NodeStatus.Disabled)
{
var nodeDisabled = new PaasNodeStatusInfo(nodeToDisable)
{
NodeState = NodeState.Disabled
};
nodesDisabled.Add(nodeDisabled);
Trace.WriteInfo(TraceType, "BuildStatusAsync: Node has been successfully disabled. {0}", nodeDisabled);
}
}
}
// Send back the list of nodes that are Enabled
if (description.NodesToEnabled != null)
{
nodesEnabled = new List <PaasNodeStatusInfo>();
// Send back the Instance# for the requested up nodes
foreach (var nodeToEnable in description.NodesToEnabled)
{
var matchingNodeStatus =
nodeList.FirstOrDefault(
node =>
string.Equals(node.NodeName, nodeToEnable.NodeName,
StringComparison.OrdinalIgnoreCase));
// Since a node can be enabled and can still be down, we infer enabled status instead.
if (matchingNodeStatus != null &&
(matchingNodeStatus.NodeStatus != NodeStatus.Disabling) &&
(matchingNodeStatus.NodeStatus != NodeStatus.Disabled) &&
(matchingNodeStatus.NodeStatus != NodeStatus.Enabling))
{
var nodeEnabled = new PaasNodeStatusInfo(nodeToEnable);
nodeEnabled.NodeState = NodeState.Enabled;
nodesEnabled.Add(nodeEnabled);
Trace.WriteInfo(TraceType, "BuildStatusAsync: Node has been successfully enabled. {0}", nodeEnabled);
}
}
}
}
Trace.WriteNoise(TraceType, "BuildStatusAsync: Begin this.nodeStatusManager.GetNodeStates.");
var nodesStatus = await this.nodeStatusManager.GetNodeStates(Constants.KvsCommitTimeout, context.CancellationToken);
Trace.WriteNoise(TraceType, "BuildStatusAsync: End this.nodeStatusManager.GetNodeStates.");
var status = new ClusterOperationStatus(description)
{
DisabledNodes = nodesDisabled,
EnabledNodes = nodesEnabled,
NodesStatus = nodesStatus,
SystemServiceDescriptions = systemServicesRuntimeDescriptions,
};
if (currentUpgradeProgress != null)
{
status.Progress = JObject.FromObject(currentUpgradeProgress, this.serializer);
}
if (errorDetails != null)
{
status.ErrorDetails = JObject.FromObject(errorDetails, this.serializer);
}
return(status);
}