// Running multiple times should be safe. private async Task LoadPartitionAndReplicaCountAsync(CancellationToken ct) { // Optimization. No need for more than one thread to enter this method. await this.asyncMutex.WaitAsync(); try { if (this.isStateful.HasValue && this.partitionCount.HasValue && this.targetReplicaSetSize.HasValue) { // values already loaded return; } ServicePartitionList servicePartitions = await this.GetPartitionsAsync(ct).ConfigureAwait(false); // Make sure servicePartitions has at least one item. ThrowIf.NullOrEmpty(servicePartitions, "servicePartitions"); // set PartitionCount ReleaseAssert.AssertIfNot(this.partitionCount.TrySetValue(servicePartitions.Count), "partitionCount has already been set to a different value"); // set isStateful field Partition partition = servicePartitions[0]; bool stateful = partition is StatefulServicePartition; ReleaseAssert.AssertIfNot(this.isStateful.TrySetValue(stateful), "isStateful has already been set to a different value"); // retrieve replicaCount if (stateful) { var statefulServicePartition = partition as StatefulServicePartition; ReleaseAssert.AssertIfNot(this.targetReplicaSetSize.TrySetValue((int)statefulServicePartition.TargetReplicaSetSize), "targetReplicaSetSize has already been set to a different value"); } else { var statelessServicePartition = partition as StatelessServicePartition; ReleaseAssert.AssertIfNot(this.targetReplicaSetSize.TrySetValue((int)statelessServicePartition.InstanceCount), "targetReplicaSetSize has already been set to a different value"); } } catch (Exception e) { TestabilityTrace.TraceSource.WriteError(TraceSource, "Error while getting partitions for service {0}. Exception: {1}", this.serviceName, e.Message); throw; } finally { this.asyncMutex.Release(); } }
private bool ValidatePartitionCount(int totalPartitionsFound) { int expectedPartitionCount; // Assert ReleaseAssert.AssertIfNot(this.partitionCount.TryGetValue(out expectedPartitionCount), "ParitionCount is null."); if (expectedPartitionCount != totalPartitionsFound) { TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Found only {0}/{1} Partitions for service {2}", totalPartitionsFound, expectedPartitionCount, this.serviceName); return(false); } // Success TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Validated that service '{0}' has {1} stable partitions.", this.serviceName, totalPartitionsFound); return(true); }
private int GetExpectedReplicaSetSize(IEnumerable <NodeInfo> nodes) { int replicaCount; // Assert ReleaseAssert.AssertIfNot(this.targetReplicaSetSize.TryGetValue(out replicaCount), "TargetReplicaSetSize is null"); int upNodeCount = nodes.Count(n => n.IsNodeUp); if (replicaCount == -1) { // This can only be the case for stateless service and this means we want to place on all nodes replicaCount = upNodeCount; } // Return min of nodes or replica count i.e. if we have 3 nodes and replica count of 5 we will only be able // to place 3/5 replicas and the check below will handle this case return(nodes.Any() ? Math.Min(upNodeCount, replicaCount) : replicaCount); }
public async Task <ValidationReport> EnsureStabilityWithReportAsync(TimeSpan maximumStabilizationTimeout, TimeSpan retryWait, CancellationToken ct) { TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Ensuring that '{0}' is online with timeout '{1}'.", this.serviceName, maximumStabilizationTimeout); bool checkQuorumLoss = (this.checkFlags & ValidationCheckFlag.CheckQuorumLoss) != 0; // Load basic information about this service. TestabilityTrace.TraceSource.WriteNoise(TraceSource, "Querying basic information for {0}.", this.serviceName); await this.LoadPartitionAndReplicaCountAsync(ct); DateTime startTime = DateTime.Now; TimeoutHelper timer = new TimeoutHelper(maximumStabilizationTimeout); bool success = false; List <Guid> partitionsInQuorumLoss = new List <Guid>(); StringBuilder errorString = new StringBuilder(); int retryCount = 1; while (!success && timer.GetRemainingTime() > TimeSpan.Zero) { TestabilityTrace.TraceSource.WriteInfo(TraceSource, "EnsureStabilityWithReportAsync(): retryCount='{0}', timer.GetRemainingTime()='{1}'", retryCount, timer.GetRemainingTime()); var nodes = await this.TestContext.FabricCluster.GetLatestNodeInfoAsync(this.requestTimeout, this.operationTimeout, ct); // Empty error string and list of partitions in quorum loss partitionsInQuorumLoss.Clear(); errorString.Clear(); success = true; int totalPartitionsFound = 0; bool stateful; ReleaseAssert.AssertIfNot(this.isStateful.TryGetValue(out stateful), "isStateful flag is not available"); bool checkTarget = (this.checkFlags & ValidationCheckFlag.CheckTargetReplicaSetSize) != 0; bool checkInBuild = (this.checkFlags & ValidationCheckFlag.CheckInBuildReplica) != 0; if (stateful) { var partitionDictionary = await this.QueryPartitionAndReplicaResultAsyncStateful(ct); totalPartitionsFound = partitionDictionary.Count(); foreach (KeyValuePair <Partition, StatefulServiceReplica[]> partition in partitionDictionary) { bool partitionIsReady = partition.Key.PartitionStatus == ServicePartitionStatus.Ready; if (!partitionIsReady) { var message = StringHelper.Format("Partition '{0}' is not Ready", partition.Key.PartitionId()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); } if (partition.Key.PartitionStatus != ServicePartitionStatus.InQuorumLoss) { int validCount = 0; int inBuildReplicas = 0; foreach (StatefulServiceReplica replica in partition.Value) { if (replica.ReplicaStatus == ServiceReplicaStatus.Ready && (replica.ReplicaRole == ReplicaRole.Primary || replica.ReplicaRole == ReplicaRole.ActiveSecondary)) { ++validCount; } if (replica.ReplicaStatus == ServiceReplicaStatus.InBuild) { ++inBuildReplicas; var message = StringHelper.Format("Replica {0} for partition '{1}' is InBuild", replica.Id, partition.Key.PartitionId()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); } } bool targetAchieved = this.CheckReplicaSetSize(partition.Key.PartitionInformation.Id, validCount, startTime, nodes, errorString); if (!partitionIsReady || (checkInBuild && inBuildReplicas > 0) || (checkTarget && !targetAchieved)) { success = false; } } else { partitionsInQuorumLoss.Add(partition.Key.PartitionInformation.Id); } } } else { int targetInstanceCount = 0; ReleaseAssert.AssertIf(!this.targetReplicaSetSize.TryGetValue(out targetInstanceCount), "targetReplicaSetSize for service: {0} should have been populated at this point.", this.serviceName); bool placementConstraintsDefined = false; try { // Get the service description to find out if there are placement constraints on the service ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.TestContext.FabricClient.ServiceManager.GetServiceDescriptionAsync( this.serviceName, this.requestTimeout, ct), this.operationTimeout, ct).ConfigureAwait(false); ThrowIf.IsTrue(result == null, "A description must be associated with the service: {0}", this.serviceName); placementConstraintsDefined = !string.IsNullOrEmpty(result.PlacementConstraints); } catch (UnauthorizedAccessException) { ServiceGroupDescription groupDescription = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.TestContext.FabricClient.ServiceGroupManager.GetServiceGroupDescriptionAsync( this.serviceName, this.requestTimeout, ct), this.operationTimeout, ct).ConfigureAwait(false); ThrowIf.IsTrue(groupDescription == null, "A description must be associated with the service group: {0}", this.serviceName); placementConstraintsDefined = !string.IsNullOrEmpty(groupDescription.ServiceDescription.PlacementConstraints); } // If a stateless service has instance count == -1 and it has placement constraints such // that the possible number of instances cannot match the total number of nodes, // we need to find out the number of eligible nodes for the service which is tracked by RDBug 8993319. // Until RDBug 8993319 is fixed, we take the presence of placement constraints into consideration to make the // validation more accurate. if (targetInstanceCount == -1 && placementConstraintsDefined) { checkTarget = false; } var partitionDictionary = await this.QueryPartitionAndReplicaResultAsyncStateless(timer.GetRemainingTime(), ct); totalPartitionsFound = partitionDictionary.Count(); foreach (KeyValuePair <Partition, StatelessServiceInstance[]> partition in partitionDictionary) { bool partitionIsReady = partition.Key.PartitionStatus == ServicePartitionStatus.Ready; if (!partitionIsReady) { var message = StringHelper.Format("Partition '{0}' is not Ready", partition.Key.PartitionId()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); } int validCount = 0; foreach (StatelessServiceInstance instance in partition.Value) { if (instance.ReplicaStatus == ServiceReplicaStatus.Ready) { ++validCount; } } bool targetAchieved = this.CheckReplicaSetSize(partition.Key.PartitionInformation.Id, validCount, startTime, nodes, errorString); if (!partitionIsReady || (checkTarget && !targetAchieved)) { success = false; } } } if (!this.ValidatePartitionCount(totalPartitionsFound)) { success = false; } if (partitionsInQuorumLoss.Count > 0 && checkQuorumLoss) { string paritionIds = string.Join(",", partitionsInQuorumLoss.ToArray()); var message = StringHelper.Format("Partitions '{0}' in quorum loss for service {1}", paritionIds, this.serviceName); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); success = false; } if (!success) { if (retryCount % 10 == 0) { TestabilityTrace.TraceSource.WriteWarning(TraceSource, "Service {0} validation failed due to issues below, will retry: \n{1}", this.serviceName, errorString); } // Delay before querying again so we allow some time for state to change - don't spam the node await AsyncWaiter.WaitAsync(retryWait, ct).ConfigureAwait(false); } retryCount++; } if (partitionsInQuorumLoss.Count > 0) { string partitionIds = string.Join(",", partitionsInQuorumLoss.ToArray()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Partitions in quorum loss for service {0} are '{1}'", this.serviceName, partitionIds); if (checkQuorumLoss) { throw new FabricValidationException(StringHelper.Format(StringResources.Error_PartitionsInQuorumLoss, partitionIds, this.serviceName)); } } if (!success) { return(new ValidationReport( true, StringHelper.Format(StringResources.Error_ServiceNotStable, this.serviceName, maximumStabilizationTimeout, errorString))); } else { return(ValidationReport.Default); } }