private void PopulateCodepackageWithDeployedPartitions(
DeployedServiceReplica deployedReplica,
CodePackageEntity codePackage,
ApplicationEntity applicationEntity)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Inside PopulateCodepackageWithDeployedPartitions");
var serviceEntity = applicationEntity.ServiceList.FirstOrDefault(s => s.Service.ServiceName == deployedReplica.ServiceName);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "ServiceEntity: {0}", serviceEntity);
if (serviceEntity != null)
{
var partitionEntity =
serviceEntity.PartitionList.FirstOrDefault(
p => p.Guid == deployedReplica.Partitionid);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "PartitionEntity: {0}", partitionEntity);
if (partitionEntity != null)
{
codePackage.DeployedPartitions.Add(partitionEntity);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Added partitionEntity: {0} in codepack: {1}", partitionEntity, codePackage);
}
else
{
// This would mean that the service for this partition has been deleted and recreated hence the old partition does not exist
}
}
else
{
// This could mean that the service for this replica has been deleted but the replica is still alive
}
}
internal static bool IsSystemServiceReplica(DeployedServiceReplica replica)
{
// Except for fabric:/System/InfrastructureService, for all other System service replicas, replica.ServiceName matches one of the
// System service names in Constants.SystemServiceList. For fabric:/System/InfrastructureService the name may have a suffix
// like fabric:/System/InfrastructureService/ABC, so we are doing StartsWith here.
return(replica.ServiceName.OriginalString.StartsWith(Constants.SystemApplicationName) &&
Constants.SystemServiceList.Any(s => replica.ServiceName.OriginalString.StartsWith(s.OriginalString, StringComparison.Ordinal)));
}
private static void FormatDeployedStatefulServiceReplica(DeployedServiceReplica deployedServiceReplica, PSObject deployedServiceReplicaDetailPSObj)
{
var deployedServiceReplicaPSObj = new PSObject(deployedServiceReplica);
deployedServiceReplicaPSObj.Members.Add(
new PSCodeMethod(
Constants.ToStringMethodName,
typeof(OutputFormatter).GetMethod(Constants.FormatObjectMethodName)));
deployedServiceReplicaDetailPSObj.Properties.Add(
new PSNoteProperty(
Constants.DeployedServiceReplica,
deployedServiceReplicaPSObj));
}
private bool MatchReplicaWithCodepackage(
DeployedServiceReplica deployedReplica,
DeployedCodePackage deployedCodePackage)
{
string deployedCodePackageName = deployedCodePackage.CodePackageName;
string deployedCodePackageVersion = deployedCodePackage.CodePackageVersion;
string deployedServiceManifestName = deployedCodePackage.ServiceManifestName;
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"Inside MatchReplicaWithCodepackage: {0} vs {1}",
deployedReplica.Address,
deployedCodePackageName + deployedCodePackageVersion + deployedServiceManifestName);
return(deployedReplica.CodePackageName.Equals(deployedCodePackageName, StringComparison.OrdinalIgnoreCase) &&
deployedReplica.ServiceManifestName.Equals(deployedServiceManifestName, StringComparison.OrdinalIgnoreCase));
}
private bool MatchFmCmNsReplicaWithDummyCodepackage(
DeployedServiceReplica deployedReplica,
DeployedCodePackage codePackage)
{
var isFmReplica =
deployedReplica.ServiceName.OriginalString.Equals(Constants.FailoverManagerServiceName, StringComparison.OrdinalIgnoreCase) &&
codePackage.CodePackageVersion.Equals(ChaosConstants.DummyFMCodePackageVersion, StringComparison.OrdinalIgnoreCase);
var isCmReplica =
deployedReplica.ServiceName.OriginalString.Equals(Constants.ClusterManagerServiceName, StringComparison.OrdinalIgnoreCase) &&
codePackage.CodePackageVersion.Equals(ChaosConstants.DummyCMCodePackageVersion, StringComparison.OrdinalIgnoreCase);
var isNsReplica =
deployedReplica.ServiceName.OriginalString.Equals(Constants.NamingServiceName, StringComparison.OrdinalIgnoreCase) &&
codePackage.CodePackageVersion.Equals(ChaosConstants.DummyNSCodePackageVersion, StringComparison.OrdinalIgnoreCase);
return(isFmReplica || isCmReplica || isNsReplica);
}
private async Task <bool> TryAssociateDeployedReplicaWithDeployedCodepackageAsync(
NodeInfo node1,
ApplicationEntity applicationEntity,
CancellationToken cancellationToken)
{
// Handle the case where application is deployed but code package is not
var retryableErrorsForGetDeployedCodePackageList = new FabricClientRetryErrors();
// RDBug 5408739: GetDeployedApplicationList found the application;
// but, by the time GetDeployedCodePackageList was issued, the application is no longer deployed on the node
retryableErrorsForGetDeployedCodePackageList.RetrySuccessFabricErrorCodes.Add(FabricErrorCode.ApplicationNotFound);
// RDBug 5420064 : GetDeployed(Application/CodePackage)ListRequest fails with FABRIC_E_INVALID_ADDRESS
retryableErrorsForGetDeployedCodePackageList.RetryableFabricErrorCodes.Add(FabricErrorCode.InvalidAddress);
var deployedCodePackageList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.testContext.FabricClient.QueryManager.GetDeployedCodePackageListAsync(
node1.NodeName,
applicationEntity.Application.ApplicationName,
string.Empty,
string.Empty,
this.requestTimeOut,
cancellationToken),
retryableErrorsForGetDeployedCodePackageList,
this.timer.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// If FM, CM, or NS replicas are on the node, we add corresponding dummy codepackages in the deployedCodePackageList
// This is required, because when we want to fault a node, we want to reach the deployed FM/CM/NS replicas through these
// deployed dummy codepackages
var listOfDeployedCodePackages = await this.GetSystemServiceDummyCodepackagesAsync(
node1,
applicationEntity,
cancellationToken).ConfigureAwait(false);
listOfDeployedCodePackages.AddRangeNullSafe(deployedCodePackageList);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed codepacks for app entity: {0}...", applicationEntity.Application.ApplicationName.OriginalString);
if (listOfDeployedCodePackages != null && listOfDeployedCodePackages.Any())
{
foreach (var dcp in listOfDeployedCodePackages)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed codepack: {0}", dcp.CodePackageName + dcp.ServiceManifestName + dcp.CodePackageVersion);
}
var deployedReplicaList =
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
this.testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync(
node1.NodeName,
applicationEntity.Application.ApplicationName,
string.Empty,
Guid.Empty,
this.requestTimeOut,
cancellationToken),
FabricClientRetryErrors.GetDeployedClusterEntityErrors.Value,
this.timer.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replicas...");
foreach (var dr in deployedReplicaList)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replica: {0}", dr.ServiceName.OriginalString + dr.Partitionid);
}
foreach (var dc in listOfDeployedCodePackages)
{
DeployedCodePackage deployedCodePackage = dc;
DeployedServicePackageHealth spHealth;
if (!string.IsNullOrEmpty(deployedCodePackage.ServiceManifestName))
{
DeployedServicePackageHealthQueryDescription desc = new DeployedServicePackageHealthQueryDescription(applicationEntity.Application.ApplicationName, node1.NodeName, deployedCodePackage.ServiceManifestName, deployedCodePackage.ServicePackageActivationId);
spHealth = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServicePackageHealth>(
() =>
this.testContext.FabricClient.HealthManager.GetDeployedServicePackageHealthAsync(desc, this.requestTimeOut, cancellationToken),
FabricClientRetryErrors.GetDeployedClusterEntityErrors.Value,
this.timer.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
else
{
spHealth = DefaultDeployedServicePackageHealth;
}
var codePackage = applicationEntity.AddCodePackage(
deployedCodePackage,
node1.NodeName,
spHealth);
foreach (var replica in deployedReplicaList)
{
string partitionIdentifier = string.Format(ReplicaViewFormat, replica.Partitionid, node1.NodeName, replica.ReplicaStatus);
this.PartitionMapFromNodes.Add(partitionIdentifier);
DeployedServiceReplica deployedReplica = replica;
var isSystemServiceReplicaWithoutCodepackage =
string.IsNullOrEmpty(deployedReplica.CodePackageName) &&
string.IsNullOrEmpty(deployedReplica.ServiceManifestName);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replica: {0} isSystemServiceReplicaWithoutCodepackage={1}", deployedReplica.ServiceName.OriginalString + deployedReplica.Partitionid, isSystemServiceReplicaWithoutCodepackage);
// Handle FM/CM/NS replica separately, because these
// services do not have an actual codepackage
if ((isSystemServiceReplicaWithoutCodepackage && this.MatchFmCmNsReplicaWithDummyCodepackage(deployedReplica, deployedCodePackage)) ||
(!isSystemServiceReplicaWithoutCodepackage && this.MatchReplicaWithCodepackage(deployedReplica, deployedCodePackage)))
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replica:{0} is in codepack: {1} so need to add deployed partition to codepack", deployedReplica.ServiceName.OriginalString + deployedReplica.Partitionid, deployedCodePackage.CodePackageName + deployedCodePackage.ServiceManifestName + deployedCodePackage.CodePackageVersion);
this.PopulateCodepackageWithDeployedPartitions(deployedReplica, codePackage, applicationEntity);
}
} // iterate through deployed replicas
} // iterate through deployed codepacks
} // if there are deployed codepacks
return(true);
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartDeployedCodePackageAction action, CancellationToken cancellationToken)
{
this.helper = new TimeoutHelper(action.ActionTimeout);
string nodeName = action.NodeName;
Uri applicationName = action.ApplicationName;
string serviceManifestName = action.ServiceManifestName;
string servicePackageActivationId = action.ServicePackageActivationId;
string codePackageName = action.CodePackageName;
SelectedReplica replicaSelectorResult = SelectedReplica.None;
ThrowIf.Null(applicationName, "ApplicationName");
if (string.IsNullOrEmpty(nodeName) ||
string.IsNullOrEmpty(serviceManifestName) ||
string.IsNullOrEmpty(codePackageName))
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = this.helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false);
var replicaStateActionResult = getReplicaStateAction.Result;
ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null");
replicaSelectorResult = replicaStateActionResult.Item1;
ReleaseAssert.AssertIf(replicaSelectorResult == null || replicaSelectorResult.SelectedPartition == null,
"replicaSelectorResult cannot be null or for a non-null replicaSelectorResult, the selected partition must be non-null");
Guid partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;
Replica replicaStateResult = replicaStateActionResult.Item2;
ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null");
nodeName = replicaStateResult.NodeName;
var deployedReplicaListResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServiceReplicaList>(
() => testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync(
nodeName,
applicationName,
null,
partitionId,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
DeployedServiceReplica selectedReplica = deployedReplicaListResult.FirstOrDefault(r => r.Partitionid == partitionId);
if (selectedReplica == null)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_DidNotFindDeployedReplicaOnNode, partitionId, nodeName),
FabricErrorCode.ReplicaDoesNotExist);
}
serviceManifestName = selectedReplica.ServiceManifestName;
servicePackageActivationId = selectedReplica.ServicePackageActivationId;
codePackageName = selectedReplica.CodePackageName;
}
ActionTraceSource.WriteInfo(TraceSource, "SelectedReplica: serviceManifestName: {0}, servicePackageActivationId: {1}, codePackageName: {2}", serviceManifestName, servicePackageActivationId, codePackageName);
DeployedCodePackage deployedCodePackageListResult = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false);
var codepackageEntrypointToRestart = GetCodepackageEntrypointToRestart(action, deployedCodePackageListResult);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RestartDeployedCodePackageUsingNodeNameAsync(
nodeName,
applicationName,
serviceManifestName,
servicePackageActivationId,
codePackageName,
codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (action.CompletionMode == CompletionMode.Verify)
{
bool success = false;
while (this.helper.GetRemainingTime() > TimeSpan.Zero)
{
var deployedCodePackageListResultAfterRestart = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false);
if (deployedCodePackageListResultAfterRestart != null)
{
var entryPointAfterRestart = codepackageEntrypointToRestart.EntryPointType == EntryPointType.Main ? deployedCodePackageListResultAfterRestart.EntryPoint : deployedCodePackageListResultAfterRestart.SetupEntryPoint;
if (entryPointAfterRestart != null && entryPointAfterRestart.CodePackageInstanceId > codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId && entryPointAfterRestart.EntryPointStatus == EntryPointStatus.Started)
{
success = true;
break;
}
}
ActionTraceSource.WriteInfo(TraceSource, "CodePackage = {0}:{1}:{2} not yet restarted. Retrying...", nodeName, applicationName, codePackageName);
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
}
if (!success)
{
throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout,
"RestartDeployedCodePackage",
applicationName));
}
}
action.Result = new RestartDeployedCodePackageResult(
nodeName,
applicationName,
serviceManifestName,
servicePackageActivationId,
codePackageName,
codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId,
replicaSelectorResult);
ResultTraceString = StringHelper.Format("RestartCodePackageAction succeeded for {0}:{1}:{2} with CompletionMode = {3}", nodeName, applicationName, codePackageName, action.CompletionMode);
}