public RunspaceGroup.Token Take()
{
Runspace runspace = null;
bool createNew = false;
lock (this.thisLock)
{
if (this.runspaces.IsEmpty && this.currentStackSize < GroupSizeMax)
{
createNew = true;
this.currentStackSize++;
}
}
if (createNew)
{
runspace = this.CreateRunspace();
}
TimeoutHelper helper = new TimeoutHelper(MaxRunspaceWaitTime);
while (runspace == null)
{
if (!this.runspaces.TryPop(out runspace))
{
this.runSpaceAvailableEvent.WaitOne(TimeSpan.FromSeconds(5));
}
ThrowIf.IsTrue(helper.GetRemainingTime() == TimeSpan.Zero, "Could not get a RunSpace in {0} minutes. Check if something is stuck", MaxRunspaceWaitTime);
}
return(new Token(this, runspace));
}
public ValidationReport(bool failed, string reason)
{
ThrowIf.IsTrue(failed && string.IsNullOrEmpty(reason), "If validation fails, you must provide a reason for the failure.");
this.validationFailed = failed;
this.failureReason = reason;
}
public async Task <ValidationReport> EnsureStabilityWithReportAsync(TimeSpan maximumStabilizationTimeout, TimeSpan retryWait, CancellationToken ct)
{
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Ensuring that '{0}' is online with timeout '{1}'.", this.serviceName, maximumStabilizationTimeout);
bool checkQuorumLoss = (this.checkFlags & ValidationCheckFlag.CheckQuorumLoss) != 0;
// Load basic information about this service.
TestabilityTrace.TraceSource.WriteNoise(TraceSource, "Querying basic information for {0}.", this.serviceName);
await this.LoadPartitionAndReplicaCountAsync(ct);
DateTime startTime = DateTime.Now;
TimeoutHelper timer = new TimeoutHelper(maximumStabilizationTimeout);
bool success = false;
List <Guid> partitionsInQuorumLoss = new List <Guid>();
StringBuilder errorString = new StringBuilder();
int retryCount = 1;
while (!success && timer.GetRemainingTime() > TimeSpan.Zero)
{
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "EnsureStabilityWithReportAsync(): retryCount='{0}', timer.GetRemainingTime()='{1}'", retryCount, timer.GetRemainingTime());
var nodes = await this.TestContext.FabricCluster.GetLatestNodeInfoAsync(this.requestTimeout, this.operationTimeout, ct);
// Empty error string and list of partitions in quorum loss
partitionsInQuorumLoss.Clear();
errorString.Clear();
success = true;
int totalPartitionsFound = 0;
bool stateful;
ReleaseAssert.AssertIfNot(this.isStateful.TryGetValue(out stateful), "isStateful flag is not available");
bool checkTarget = (this.checkFlags & ValidationCheckFlag.CheckTargetReplicaSetSize) != 0;
bool checkInBuild = (this.checkFlags & ValidationCheckFlag.CheckInBuildReplica) != 0;
if (stateful)
{
var partitionDictionary = await this.QueryPartitionAndReplicaResultAsyncStateful(ct);
totalPartitionsFound = partitionDictionary.Count();
foreach (KeyValuePair <Partition, StatefulServiceReplica[]> partition in partitionDictionary)
{
bool partitionIsReady = partition.Key.PartitionStatus == ServicePartitionStatus.Ready;
if (!partitionIsReady)
{
var message = StringHelper.Format("Partition '{0}' is not Ready", partition.Key.PartitionId());
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message);
errorString.AppendLine(message);
}
if (partition.Key.PartitionStatus != ServicePartitionStatus.InQuorumLoss)
{
int validCount = 0;
int inBuildReplicas = 0;
foreach (StatefulServiceReplica replica in partition.Value)
{
if (replica.ReplicaStatus == ServiceReplicaStatus.Ready &&
(replica.ReplicaRole == ReplicaRole.Primary || replica.ReplicaRole == ReplicaRole.ActiveSecondary))
{
++validCount;
}
if (replica.ReplicaStatus == ServiceReplicaStatus.InBuild)
{
++inBuildReplicas;
var message = StringHelper.Format("Replica {0} for partition '{1}' is InBuild", replica.Id, partition.Key.PartitionId());
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message);
errorString.AppendLine(message);
}
}
bool targetAchieved = this.CheckReplicaSetSize(partition.Key.PartitionInformation.Id, validCount, startTime, nodes, errorString);
if (!partitionIsReady ||
(checkInBuild && inBuildReplicas > 0) ||
(checkTarget && !targetAchieved))
{
success = false;
}
}
else
{
partitionsInQuorumLoss.Add(partition.Key.PartitionInformation.Id);
}
}
}
else
{
int targetInstanceCount = 0;
ReleaseAssert.AssertIf(!this.targetReplicaSetSize.TryGetValue(out targetInstanceCount), "targetReplicaSetSize for service: {0} should have been populated at this point.", this.serviceName);
bool placementConstraintsDefined = false;
try
{
// Get the service description to find out if there are placement constraints on the service
ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.TestContext.FabricClient.ServiceManager.GetServiceDescriptionAsync(
this.serviceName,
this.requestTimeout,
ct),
this.operationTimeout,
ct).ConfigureAwait(false);
ThrowIf.IsTrue(result == null, "A description must be associated with the service: {0}", this.serviceName);
placementConstraintsDefined = !string.IsNullOrEmpty(result.PlacementConstraints);
}
catch (UnauthorizedAccessException)
{
ServiceGroupDescription groupDescription = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.TestContext.FabricClient.ServiceGroupManager.GetServiceGroupDescriptionAsync(
this.serviceName,
this.requestTimeout,
ct),
this.operationTimeout,
ct).ConfigureAwait(false);
ThrowIf.IsTrue(groupDescription == null, "A description must be associated with the service group: {0}", this.serviceName);
placementConstraintsDefined = !string.IsNullOrEmpty(groupDescription.ServiceDescription.PlacementConstraints);
}
// If a stateless service has instance count == -1 and it has placement constraints such
// that the possible number of instances cannot match the total number of nodes,
// we need to find out the number of eligible nodes for the service which is tracked by RDBug 8993319.
// Until RDBug 8993319 is fixed, we take the presence of placement constraints into consideration to make the
// validation more accurate.
if (targetInstanceCount == -1 && placementConstraintsDefined)
{
checkTarget = false;
}
var partitionDictionary = await this.QueryPartitionAndReplicaResultAsyncStateless(timer.GetRemainingTime(), ct);
totalPartitionsFound = partitionDictionary.Count();
foreach (KeyValuePair <Partition, StatelessServiceInstance[]> partition in partitionDictionary)
{
bool partitionIsReady = partition.Key.PartitionStatus == ServicePartitionStatus.Ready;
if (!partitionIsReady)
{
var message = StringHelper.Format("Partition '{0}' is not Ready", partition.Key.PartitionId());
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message);
errorString.AppendLine(message);
}
int validCount = 0;
foreach (StatelessServiceInstance instance in partition.Value)
{
if (instance.ReplicaStatus == ServiceReplicaStatus.Ready)
{
++validCount;
}
}
bool targetAchieved = this.CheckReplicaSetSize(partition.Key.PartitionInformation.Id, validCount, startTime, nodes, errorString);
if (!partitionIsReady ||
(checkTarget && !targetAchieved))
{
success = false;
}
}
}
if (!this.ValidatePartitionCount(totalPartitionsFound))
{
success = false;
}
if (partitionsInQuorumLoss.Count > 0 && checkQuorumLoss)
{
string paritionIds = string.Join(",", partitionsInQuorumLoss.ToArray());
var message = StringHelper.Format("Partitions '{0}' in quorum loss for service {1}", paritionIds, this.serviceName);
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message);
errorString.AppendLine(message);
success = false;
}
if (!success)
{
if (retryCount % 10 == 0)
{
TestabilityTrace.TraceSource.WriteWarning(TraceSource, "Service {0} validation failed due to issues below, will retry: \n{1}", this.serviceName, errorString);
}
// Delay before querying again so we allow some time for state to change - don't spam the node
await AsyncWaiter.WaitAsync(retryWait, ct).ConfigureAwait(false);
}
retryCount++;
}
if (partitionsInQuorumLoss.Count > 0)
{
string partitionIds = string.Join(",", partitionsInQuorumLoss.ToArray());
TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Partitions in quorum loss for service {0} are '{1}'", this.serviceName, partitionIds);
if (checkQuorumLoss)
{
throw new FabricValidationException(StringHelper.Format(StringResources.Error_PartitionsInQuorumLoss, partitionIds, this.serviceName));
}
}
if (!success)
{
return(new ValidationReport(
true,
StringHelper.Format(StringResources.Error_ServiceNotStable, this.serviceName, maximumStabilizationTimeout, errorString)));
}
else
{
return(ValidationReport.Default);
}
}
