private DeployedCodePackage GetDummyDeployedServiceCodepackage(string dummyCodePackageVersion)
{
var dummyEntryPoint = new CodePackageEntryPoint(
string.Empty, // entry-point location
0, // process id
string.Empty, // runas username
EntryPointStatus.Started,
DateTime.UtcNow, // next activation utc
null, // codepack statistics
0 /*codepack instance id*/);
var dummyCodePack = new DeployedCodePackage(
string.Empty, // codepack name
dummyCodePackageVersion, // codepack version
dummyEntryPoint, // setup entrypoint
string.Empty, // servicemanifest name
string.Empty, // Default ServicePackageActivationId
0, // run frequency interval
HostType.ExeHost,
HostIsolationMode.None,
DeploymentStatus.Active,
dummyEntryPoint /*entrypoint*/);
return(dummyCodePack);
}
public CodePackageEntity AddCodePackage(DeployedCodePackage codePackage, string nodeName, DeployedServicePackageHealth health)
{
var codePackageTraceStr = new StringBuilder();
codePackageTraceStr.Append(codePackage.CodePackageName);
codePackageTraceStr.Append("_");
codePackageTraceStr.Append(codePackage.CodePackageVersion);
codePackageTraceStr.Append("_");
codePackageTraceStr.Append(codePackage.ServiceManifestName);
if (string.IsNullOrWhiteSpace(codePackage.ServicePackageActivationId))
{
codePackageTraceStr.Append("_");
codePackageTraceStr.Append(codePackage.ServicePackageActivationId);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Inside of AddCodePackage: app={0}, cp={1}, nodename={2}",
this.Application.ApplicationName.OriginalString,
codePackageTraceStr.ToString(),
nodeName);
CodePackageEntity codePackageEntity = new CodePackageEntity(codePackage, nodeName, this, health);
this.CodePackages.Add(codePackageEntity);
return(codePackageEntity);
}
public CodePackageEntity(DeployedCodePackage codePackageResult, string nodeName, ApplicationEntity applicationEntity, DeployedServicePackageHealth health)
{
this.CodePackageResult = codePackageResult;
this.NodeName = nodeName;
this.ParentApplicationEntity = applicationEntity;
this.DeployedPartitions = new List <PartitionEntity>();
this.CodePackageFlags = ClusterEntityFlags.Excluded;
this.health = health;
}
public DeployedCodePackageEntity(DeployedCodePackage dcp, string nodeName, ApplicationEntity applicationEntity)
{
Assert.IsNotNull(dcp);
Assert.IsNotEmptyOrNull(nodeName, "Node Name can't be null/empty");
Assert.IsNotNull(applicationEntity);
this.CodePackageName = dcp.CodePackageName;
this.CodePackageVersion = dcp.CodePackageVersion;
this.ServiceManifestName = dcp.ServiceManifestName;
this.NodeName = nodeName;
this.EntryPointLocation = dcp.EntryPoint.EntryPointLocation;
this.SetupPointLocation = dcp.SetupEntryPoint != null ? dcp.SetupEntryPoint.EntryPointLocation : null;
this.Application = applicationEntity;
}
private bool MatchReplicaWithCodepackage(
DeployedServiceReplica deployedReplica,
DeployedCodePackage deployedCodePackage)
{
string deployedCodePackageName = deployedCodePackage.CodePackageName;
string deployedCodePackageVersion = deployedCodePackage.CodePackageVersion;
string deployedServiceManifestName = deployedCodePackage.ServiceManifestName;
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"Inside MatchReplicaWithCodepackage: {0} vs {1}",
deployedReplica.Address,
deployedCodePackageName + deployedCodePackageVersion + deployedServiceManifestName);
return(deployedReplica.CodePackageName.Equals(deployedCodePackageName, StringComparison.OrdinalIgnoreCase) &&
deployedReplica.ServiceManifestName.Equals(deployedServiceManifestName, StringComparison.OrdinalIgnoreCase));
}
private bool MatchFmCmNsReplicaWithDummyCodepackage(
DeployedServiceReplica deployedReplica,
DeployedCodePackage codePackage)
{
var isFmReplica =
deployedReplica.ServiceName.OriginalString.Equals(Constants.FailoverManagerServiceName, StringComparison.OrdinalIgnoreCase) &&
codePackage.CodePackageVersion.Equals(ChaosConstants.DummyFMCodePackageVersion, StringComparison.OrdinalIgnoreCase);
var isCmReplica =
deployedReplica.ServiceName.OriginalString.Equals(Constants.ClusterManagerServiceName, StringComparison.OrdinalIgnoreCase) &&
codePackage.CodePackageVersion.Equals(ChaosConstants.DummyCMCodePackageVersion, StringComparison.OrdinalIgnoreCase);
var isNsReplica =
deployedReplica.ServiceName.OriginalString.Equals(Constants.NamingServiceName, StringComparison.OrdinalIgnoreCase) &&
codePackage.CodePackageVersion.Equals(ChaosConstants.DummyNSCodePackageVersion, StringComparison.OrdinalIgnoreCase);
return(isFmReplica || isCmReplica || isNsReplica);
}
private async Task <bool> TryAssociateDeployedReplicaWithDeployedCodepackageAsync(
NodeInfo node1,
ApplicationEntity applicationEntity,
CancellationToken cancellationToken)
{
// Handle the case where application is deployed but code package is not
var retryableErrorsForGetDeployedCodePackageList = new FabricClientRetryErrors();
// RDBug 5408739: GetDeployedApplicationList found the application;
// but, by the time GetDeployedCodePackageList was issued, the application is no longer deployed on the node
retryableErrorsForGetDeployedCodePackageList.RetrySuccessFabricErrorCodes.Add(FabricErrorCode.ApplicationNotFound);
// RDBug 5420064 : GetDeployed(Application/CodePackage)ListRequest fails with FABRIC_E_INVALID_ADDRESS
retryableErrorsForGetDeployedCodePackageList.RetryableFabricErrorCodes.Add(FabricErrorCode.InvalidAddress);
var deployedCodePackageList = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.testContext.FabricClient.QueryManager.GetDeployedCodePackageListAsync(
node1.NodeName,
applicationEntity.Application.ApplicationName,
string.Empty,
string.Empty,
this.requestTimeOut,
cancellationToken),
retryableErrorsForGetDeployedCodePackageList,
this.timer.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
// If FM, CM, or NS replicas are on the node, we add corresponding dummy codepackages in the deployedCodePackageList
// This is required, because when we want to fault a node, we want to reach the deployed FM/CM/NS replicas through these
// deployed dummy codepackages
var listOfDeployedCodePackages = await this.GetSystemServiceDummyCodepackagesAsync(
node1,
applicationEntity,
cancellationToken).ConfigureAwait(false);
listOfDeployedCodePackages.AddRangeNullSafe(deployedCodePackageList);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed codepacks for app entity: {0}...", applicationEntity.Application.ApplicationName.OriginalString);
if (listOfDeployedCodePackages != null && listOfDeployedCodePackages.Any())
{
foreach (var dcp in listOfDeployedCodePackages)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed codepack: {0}", dcp.CodePackageName + dcp.ServiceManifestName + dcp.CodePackageVersion);
}
var deployedReplicaList =
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() =>
this.testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync(
node1.NodeName,
applicationEntity.Application.ApplicationName,
string.Empty,
Guid.Empty,
this.requestTimeOut,
cancellationToken),
FabricClientRetryErrors.GetDeployedClusterEntityErrors.Value,
this.timer.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replicas...");
foreach (var dr in deployedReplicaList)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replica: {0}", dr.ServiceName.OriginalString + dr.Partitionid);
}
foreach (var dc in listOfDeployedCodePackages)
{
DeployedCodePackage deployedCodePackage = dc;
DeployedServicePackageHealth spHealth;
if (!string.IsNullOrEmpty(deployedCodePackage.ServiceManifestName))
{
DeployedServicePackageHealthQueryDescription desc = new DeployedServicePackageHealthQueryDescription(applicationEntity.Application.ApplicationName, node1.NodeName, deployedCodePackage.ServiceManifestName, deployedCodePackage.ServicePackageActivationId);
spHealth = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServicePackageHealth>(
() =>
this.testContext.FabricClient.HealthManager.GetDeployedServicePackageHealthAsync(desc, this.requestTimeOut, cancellationToken),
FabricClientRetryErrors.GetDeployedClusterEntityErrors.Value,
this.timer.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
}
else
{
spHealth = DefaultDeployedServicePackageHealth;
}
var codePackage = applicationEntity.AddCodePackage(
deployedCodePackage,
node1.NodeName,
spHealth);
foreach (var replica in deployedReplicaList)
{
string partitionIdentifier = string.Format(ReplicaViewFormat, replica.Partitionid, node1.NodeName, replica.ReplicaStatus);
this.PartitionMapFromNodes.Add(partitionIdentifier);
DeployedServiceReplica deployedReplica = replica;
var isSystemServiceReplicaWithoutCodepackage =
string.IsNullOrEmpty(deployedReplica.CodePackageName) &&
string.IsNullOrEmpty(deployedReplica.ServiceManifestName);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replica: {0} isSystemServiceReplicaWithoutCodepackage={1}", deployedReplica.ServiceName.OriginalString + deployedReplica.Partitionid, isSystemServiceReplicaWithoutCodepackage);
// Handle FM/CM/NS replica separately, because these
// services do not have an actual codepackage
if ((isSystemServiceReplicaWithoutCodepackage && this.MatchFmCmNsReplicaWithDummyCodepackage(deployedReplica, deployedCodePackage)) ||
(!isSystemServiceReplicaWithoutCodepackage && this.MatchReplicaWithCodepackage(deployedReplica, deployedCodePackage)))
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Deployed replica:{0} is in codepack: {1} so need to add deployed partition to codepack", deployedReplica.ServiceName.OriginalString + deployedReplica.Partitionid, deployedCodePackage.CodePackageName + deployedCodePackage.ServiceManifestName + deployedCodePackage.CodePackageVersion);
this.PopulateCodepackageWithDeployedPartitions(deployedReplica, codePackage, applicationEntity);
}
} // iterate through deployed replicas
} // iterate through deployed codepacks
} // if there are deployed codepacks
return(true);
}
private static EntrypointAndType GetCodepackageEntrypointToRestart(RestartDeployedCodePackageAction action, DeployedCodePackage codepackage)
{
if (codepackage == null ||
(codepackage.EntryPoint == null &&
codepackage.SetupEntryPoint == null))
{
throw new FabricException(
StringHelper.Format(StringResources.Error_CodePackageNotDeployedOnNode, action.ApplicationName, action.CodePackageName, action.NodeName),
FabricErrorCode.CodePackageNotFound);
}
CodePackageEntryPoint restartedEntryPoint = null;
EntryPointType restartedEntryPointType = EntryPointType.None;
if (codepackage.EntryPoint != null && codepackage.EntryPoint.EntryPointStatus == EntryPointStatus.Started)
{
restartedEntryPoint = codepackage.EntryPoint;
restartedEntryPointType = EntryPointType.Main;
}
else if (codepackage.SetupEntryPoint != null && codepackage.SetupEntryPoint.EntryPointStatus == EntryPointStatus.Started)
{
restartedEntryPoint = codepackage.SetupEntryPoint;
restartedEntryPointType = EntryPointType.Setup;
}
if (restartedEntryPoint == null || restartedEntryPointType == EntryPointType.None)
{
throw new InvalidOperationException(StringResources.Error_CodePackageCannotBeRestarted);
}
if (action.CodePackageInstanceId != 0 && restartedEntryPoint.CodePackageInstanceId != action.CodePackageInstanceId)
{
throw new FabricException(StringResources.Error_InstanceIdMismatch, FabricErrorCode.InstanceIdMismatch);
}
return(new EntrypointAndType(restartedEntryPoint, restartedEntryPointType));
}
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartDeployedCodePackageAction action, CancellationToken cancellationToken)
{
this.helper = new TimeoutHelper(action.ActionTimeout);
string nodeName = action.NodeName;
Uri applicationName = action.ApplicationName;
string serviceManifestName = action.ServiceManifestName;
string servicePackageActivationId = action.ServicePackageActivationId;
string codePackageName = action.CodePackageName;
SelectedReplica replicaSelectorResult = SelectedReplica.None;
ThrowIf.Null(applicationName, "ApplicationName");
if (string.IsNullOrEmpty(nodeName) ||
string.IsNullOrEmpty(serviceManifestName) ||
string.IsNullOrEmpty(codePackageName))
{
ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector");
var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector)
{
RequestTimeout = action.RequestTimeout,
ActionTimeout = this.helper.GetRemainingTime()
};
await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false);
var replicaStateActionResult = getReplicaStateAction.Result;
ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null");
replicaSelectorResult = replicaStateActionResult.Item1;
ReleaseAssert.AssertIf(replicaSelectorResult == null || replicaSelectorResult.SelectedPartition == null,
"replicaSelectorResult cannot be null or for a non-null replicaSelectorResult, the selected partition must be non-null");
Guid partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId;
Replica replicaStateResult = replicaStateActionResult.Item2;
ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null");
nodeName = replicaStateResult.NodeName;
var deployedReplicaListResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServiceReplicaList>(
() => testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync(
nodeName,
applicationName,
null,
partitionId,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
DeployedServiceReplica selectedReplica = deployedReplicaListResult.FirstOrDefault(r => r.Partitionid == partitionId);
if (selectedReplica == null)
{
throw new FabricException(
StringHelper.Format(StringResources.Error_DidNotFindDeployedReplicaOnNode, partitionId, nodeName),
FabricErrorCode.ReplicaDoesNotExist);
}
serviceManifestName = selectedReplica.ServiceManifestName;
servicePackageActivationId = selectedReplica.ServicePackageActivationId;
codePackageName = selectedReplica.CodePackageName;
}
ActionTraceSource.WriteInfo(TraceSource, "SelectedReplica: serviceManifestName: {0}, servicePackageActivationId: {1}, codePackageName: {2}", serviceManifestName, servicePackageActivationId, codePackageName);
DeployedCodePackage deployedCodePackageListResult = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false);
var codepackageEntrypointToRestart = GetCodepackageEntrypointToRestart(action, deployedCodePackageListResult);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => testContext.FabricClient.FaultManager.RestartDeployedCodePackageUsingNodeNameAsync(
nodeName,
applicationName,
serviceManifestName,
servicePackageActivationId,
codePackageName,
codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId,
action.RequestTimeout,
cancellationToken),
this.helper.GetRemainingTime(),
cancellationToken).ConfigureAwait(false);
if (action.CompletionMode == CompletionMode.Verify)
{
bool success = false;
while (this.helper.GetRemainingTime() > TimeSpan.Zero)
{
var deployedCodePackageListResultAfterRestart = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false);
if (deployedCodePackageListResultAfterRestart != null)
{
var entryPointAfterRestart = codepackageEntrypointToRestart.EntryPointType == EntryPointType.Main ? deployedCodePackageListResultAfterRestart.EntryPoint : deployedCodePackageListResultAfterRestart.SetupEntryPoint;
if (entryPointAfterRestart != null && entryPointAfterRestart.CodePackageInstanceId > codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId && entryPointAfterRestart.EntryPointStatus == EntryPointStatus.Started)
{
success = true;
break;
}
}
ActionTraceSource.WriteInfo(TraceSource, "CodePackage = {0}:{1}:{2} not yet restarted. Retrying...", nodeName, applicationName, codePackageName);
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
}
if (!success)
{
throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout,
"RestartDeployedCodePackage",
applicationName));
}
}
action.Result = new RestartDeployedCodePackageResult(
nodeName,
applicationName,
serviceManifestName,
servicePackageActivationId,
codePackageName,
codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId,
replicaSelectorResult);
ResultTraceString = StringHelper.Format("RestartCodePackageAction succeeded for {0}:{1}:{2} with CompletionMode = {3}", nodeName, applicationName, codePackageName, action.CompletionMode);
}