private void InternalTest()
{
TestabilityTrace.TraceSource.WriteInfo(FaultAnalysisServiceTraceType, "Running internal tests");
ActionTest actionTest = ActionTest.CreateAsync(this.messageProcessor, this.actionStore, this.StateManager).GetAwaiter().GetResult();
actionTest.Test();
}
public StateTransition(ActionTest shouldExecute, ActionExecution execute, T nextState, TransitionType transitionType)
{
ShouldExecuteAction = shouldExecute;
ExecuteAction = execute;
NextState = nextState;
TransitionType = transitionType;
}
private void Awake()
{
if (Instance == null)
{
Instance = this;
}
}
private async Task <RollbackState> CheckUserCancellationAndUpdateIfNeededInnerAsync(ActionStateBase actionState, CancellationToken cancellationToken, string location)
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "CheckUserCancellationAndUpdateIfNeededAsync() {0} ENTER location='{1}'", actionState.OperationId, location);
RollbackState rollbackState;
rollbackState = await this.actionStore.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, location).ConfigureAwait(false);
if (rollbackState == RollbackState.RollingBackDueToUserCancel || rollbackState == RollbackState.RollingBackForce)
{
actionState.RollbackState = rollbackState;
}
else
{
// This is an internal test hook used in ActionTest.cs and TSInternal.test. This condition will never be true outside of that.
if (actionState.ServiceInternalFaultInfo != null)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "CheckUserCancellationAndUpdateIfNeededAsync() {0}/{1} - test cancel case, checking...", actionState.OperationId, location);
rollbackState = ActionTest.PerformTestUserCancelIfRequested(actionState.OperationId, location, actionState.ServiceInternalFaultInfo, actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "CheckUserCancellationAndUpdateIfNeededAsync() {0}/{1} - returned rollbackState is {2}", actionState.OperationId, location, rollbackState, rollbackState);
actionState.RollbackState = rollbackState;
}
}
return(rollbackState);
}
static void Main(string[] args)
{
/*Calculate calc = (a, b) => a + b;
*
* Console.WriteLine($"{3} + {4} : {calc(3, 4)}");*/
//----------------------------------------
Concatenate concat =
(arr) =>
{
string result = "";
foreach (string s in arr)
{
result += s;
}
return(result);
};
Console.WriteLine(concat(args)); //속성에 애플리케이션 인수 추가해야 결과값 나옴.
FuncTest funcTest = new FuncTest();
funcTest.exam();
ActionTest actionTest = new ActionTest();
actionTest.exam();
}
// MonoBehaviour's interface
void Awake()
{
m_ActionTest = GetComponent <ActionTest>();
int bufferSize = 10;
m_Buffer = new TimedBuffer <bool>(bufferSize);
}
static void Main(string[] args)
{
ActionTest target = new ActionTest();
MethodTest instance = new MethodTest();
target.DoExpressionAction(() => instance.Method1().Method2());
Console.ReadLine();
}
static void Main(string[] args)
{
ActionTest target = new ActionTest();
target.DoAction(() => Console.WriteLine("Action"));
target.DoExpressionAction(() => Console.WriteLine("ExpressionAction"));
Console.ReadLine();
}
public void Update()
{
if (Input.GetKeyDown(KeyCode.A))
{
ActionTest testing = GameObject.FindObjectOfType <ActionTest>();
testing.m_Action += MyAction;
}
if (Input.GetKeyDown(KeyCode.C))
{
Destroy(gameObject);
}
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - Inside TestRetryStepStepTwo - this should retry w/o rollback when exception is thrown", this.State.OperationId);
TestRetryStepState castedState = Convert(this.State);
// Simulate work
await Task.Delay(TimeSpan.FromMilliseconds(500)).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - Inside TestRetryStepStepTwo - before PerformInternalServiceFaultIfRequested", this.State.OperationId);
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - Inside TestRetryStepStepTwo - after PerformInternalServiceFaultIfRequested", this.State.OperationId);
this.State.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(this.State);
}
//Abrupt camera change
void SwitchAngle90(ActionTest t)
{
var rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x + 0.25f) * (xMax - xMin) + xMin, 0);
t.targetPosition = rotation * new Vector3(0, 0, -(z * (zMax - zMin) + zMin)) + lookPosition;
if (Terrain.activeTerrain != null)
{
t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition) + 1, 10000);
}
t.chosen = () => { x += 0.25f; transform.position = t.targetPosition; };
t.Test();
if (!t.success)
{
rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x - 0.25f) * (xMax - xMin) + xMin, 0);
t.targetPosition = rotation * new Vector3(0, 0, -(z * (zMax - zMin) + zMin)) + lookPosition;
if (Terrain.activeTerrain != null)
{
t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition) + 1, 10000);
}
t.chosen = () => { x -= 0.25f; transform.position = t.targetPosition; };
t.Test();
}
if (!t.success)
{
rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x + 0.35f) * (xMax - xMin) + xMin, 0);
t.targetPosition = rotation * new Vector3(0, 0, -(z * (zMax - zMin) + zMin)) + lookPosition;
if (Terrain.activeTerrain != null)
{
t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition) + 1, 10000);
}
t.chosen = () => { x += 0.35f; transform.position = t.targetPosition; };
t.Test();
}
if (!t.success)
{
rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x - 0.35f) * (xMax - xMin) + xMin, 0);
t.targetPosition = rotation * new Vector3(0, 0, -(z * (zMax - zMin) + zMin)) + lookPosition;
if (Terrain.activeTerrain != null)
{
t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition) + 1, 10000);
}
t.chosen = () => { x -= 0.35f; transform.position = t.targetPosition; };
t.Test();
}
t.distanceMoved = (transform.position - t.targetPosition).magnitude * cutWeighting;
}
//Get closer
void MoveCloser(ActionTest t)
{
var rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, x * (xMax - xMin) + xMin, 0);
var relative = requestedPosition - lookPosition;
RaycastHit hit;
Physics.Raycast(lookPosition, relative, out hit, relative.magnitude + 0.5f);
var distance = Mathf.Clamp01((hit.distance - zMin) / (zMax - zMin) - 0.01f);
t.targetPosition = rotation * new Vector3(0, 0, -(distance * (zMax - zMin) + zMin)) + lookPosition;
if (Terrain.activeTerrain != null)
{
t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition) + 1, 10000);
}
t.chosen = () => { transform.position = t.targetPosition; z = z - 0.01f; };
t.Test();
t.distanceMoved = (transform.position - t.targetPosition).magnitude * zoomWeighting;
}
private void Update()
{
if (Input.GetKeyDown(KeyCode.Space))
{
ActionTest test = GameObject.FindObjectOfType <ActionTest>();
test.m_Action += MyAction;
}
if (Input.GetKeyDown(KeyCode.Q))
{
Destroy(gameObject);
}
if (Input.GetKeyDown(KeyCode.A))
{
gameObject.AddComponent <ActionTest>();
}
}
//Rise above an object and look downwards
void RiseUp(ActionTest t)
{
t.targetPosition = requestedPosition;
t.success = false;
var offset = 0.002f;
while (!t.success && y + offset < 1)
{
var rotation = Quaternion.Euler(y + offset * (yMax - yMin) + yMin, x * (xMax - xMin) + xMin, 0);
t.targetPosition = rotation * new Vector3(0, 0, -(z * (zMax - zMin) + zMin)) + lookPosition;
if (Terrain.activeTerrain != null)
{
t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition) + 1, 10000);
}
t.chosen = () => { transform.position = t.targetPosition; y = y + offset; };
t.Test();
t.distanceMoved = (transform.position - t.targetPosition).magnitude * riseWeighting;
offset = offset + 0.02f;
}
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
// The return value is ignored, this is just being used to check if the RemoveNodeState was called.
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - calling StartNodeUsingNodeNameAsync, ApiInputNodeInstanceId={1}", this.State.OperationId, state.Info.InputNodeInstanceId);
Exception exception = null;
try
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.FaultManager.StartNodeUsingNodeNameAsync(
state.Info.NodeName,
state.Info.InputNodeInstanceId,
null,
0,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.StartNodeErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw {1}", this.State.OperationId, e);
exception = e;
}
cancellationToken.ThrowIfCancellationRequested();
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
if (exception != null)
{
FabricException fe = exception as FabricException;
if (fe != null)
{
status = this.HandleFabricException(fe, state);
}
else
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw non-FabricException with ErrorCode={1}", this.State.OperationId, exception);
status = SuccessRetryOrFail.RetryStep;
}
}
else
{
// success
status = SuccessRetryOrFail.Success;
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
false,
cancellationToken).ConfigureAwait(false);
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
if (status == SuccessRetryOrFail.RetryStep)
{
throw new RetrySameStepException("retrystep", exception);
}
else if (status == SuccessRetryOrFail.Fail)
{
throw new FatalException("fatal", exception);
}
else if (status == SuccessRetryOrFail.Success)
{
// no-op
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "This condition should not have been hit. OperationId: {0}", this.State.OperationId));
}
await this.ValidateAsync(this.FabricClient, state, cancellationToken).ConfigureAwait(false);
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
static void Main(string[] args)
{
// System Object
A a = new A();
A a3 = a;
var b = a.GetHashCode();
B b5 = new B();
var g5 = b5.GetHashCode();
// Extention Method
a.GlobelExtention();
var b1 = a.GetType();
var b2 = a.ToString();
var flag = a.Equals(a3);
// Palindrom
Palindrome.IsPalindrom("MALAYALAM");
// Number and Count
FindNumberCount();
//Shallow Copy
Person obj = new Person();
obj.age = 30;
Clone.Employee objEmpSalary = new Clone.Employee(1000);
obj.objEmp = objEmpSalary;
//ShallowCopy obj2 = obj.ShallowCopyClass(obj);
Person obj2 = obj.CreateDeepCopy(obj);
objEmpSalary.Salary = 3000;
Console.WriteLine(obj.objEmp.Salary);
Console.WriteLine(obj2.objEmp.Salary);
// Maximum Employee Salry based on department
EmployeeMaster();
// Null for String and Date
string abc = null;
DateTime dob = new DateTime();
Console.WriteLine(abc == null ? "Mahesh" : abc);
Console.WriteLine(dob == null ? "Mahesh" : dob.ToString());
// Expando Object
dynamic user = new System.Dynamic.ExpandoObject();
user.Name = "John Doe";
user.Age = 42;
user.HomeTown = "New York";
Console.WriteLine(user.Name + " is " + user.Age + " years old and lives in " + user.HomeTown);
/// <summary>
/// Action
/// </summary>
/// <returns></returns>
ActionTest objAction = new ActionTest();
objAction.FindSalary();
Console.WriteLine("Delegate");
Print printDel = PrintNumber;
printDel += PrintMoney;
printDel(2000);
//................................................................
Console.WriteLine("Action");
Action <int> action = PrintNumber;
action += PrintMoney;
action(3000);
//................................................................
Action <int> action1 = (num) => {
Console.WriteLine("Number: {0,-12:N0}", num);
};
action1(2500);
Console.WriteLine("Func");
Func <int, double> func = new Func <int, double>(CalculateHra);
Console.WriteLine(func(2500).ToString());
Print print = delegate(int val) {
Console.WriteLine("Inside Anonymous method. Value: {0}", val);
};
Print printn = (val) => {
Console.WriteLine("Inside Anonymous method. Value: {0}", val);
};
}
private async Task RunStateMachineAsync(
FabricClient fabricClient,
FabricTestAction action,
ActionStateBase actionState,
ServiceInternalFaultInfo serviceInternalFaultInfo,
CancellationToken cancellationToken)
{
if (actionState.StateProgress == null || actionState.StateProgress.Count == 0)
{
ReleaseAssert.AssertIf(actionState.StateProgress == null || actionState.StateProgress.Count == 0, "ActionProgress should not be null or empty");
}
Exception actionError = null;
if (actionState.RollbackState == RollbackState.NotRollingBack ||
(actionState.RollbackState != RollbackState.RollingBackForce && actionState.RetryStepWithoutRollingBackOnFailure))
{
// TODO: should also include Error
while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully)
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.ForwardLoop).ConfigureAwait(false);
if ((readRollbackState == RollbackState.RollingBackForce) ||
((readRollbackState == RollbackState.RollingBackDueToUserCancel) && !actionState.RetryStepWithoutRollingBackOnFailure))
{
break;
}
try
{
await this.RunStepAsync(fabricClient, action, actionState, cancellationToken, serviceInternalFaultInfo).ConfigureAwait(false);
ActionTest.PerformInternalServiceFaultIfRequested(actionState.OperationId, serviceInternalFaultInfo, actionState, cancellationToken);
if (actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - completed successfully, clearing ErrorCausingRollback", actionState.OperationId);
actionState.ErrorCausingRollback = 0;
}
actionState.TimeStopped = DateTime.UtcNow;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
catch (RetrySameStepException)
{
// Retry the command in the same step - do not rollback or go forward, and do not call ActionStore.UpdateActionStateAsync().
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - threw RetrySameStepException, retrying state {1} ", actionState.StateProgress.Peek());
}
catch (FabricNotPrimaryException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - RunStateMachineAsync caught: {1}", actionState.OperationId, e.ToString());
readRollbackState = this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.ForwardLoopExceptionBlock).GetAwaiter().GetResult();
// 1st line: if this is a force rollback (RollingBackForce), just exit
// 2nd line: if !RetryStepWithoutRollingBackOnFailure and there was a graceful cancel then exit this block and proceed to the rollback code block below.
// If RetryStepWithoutRollingBackOnFailure is true, which it is only for the node steps today, then first call HandleRollback to translate the exception.
if ((readRollbackState == RollbackState.RollingBackForce) ||
((readRollbackState == RollbackState.RollingBackDueToUserCancel) && !actionState.RetryStepWithoutRollingBackOnFailure))
{
break;
}
else
{
bool isRetryable = this.HandleRollback(actionState.OperationId, e);
if (isRetryable)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - observed retryable exception, will retry action. Exception: {1}", actionState.OperationId, e.ToString());
actionState.RollbackState = RollbackState.RollingBackAndWillRetryAction;
}
else
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - observed non-retryable exception. Exception: {1}", actionState.OperationId, e.ToString());
actionState.RollbackState = RollbackState.RollingBackAndWillFailAction;
}
}
actionError = e;
break;
}
}
}
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction ||
actionState.RollbackState == RollbackState.RollingBackAndWillFailAction ||
(actionState.RollbackState == RollbackState.RollingBackDueToUserCancel && (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully)))
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Rollingback type={1}", actionState.OperationId, actionState.ActionType);
if (!this.isTestMode && actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - state should not be CompletedSuccessfully", actionState.OperationId);
TestabilityTrace.TraceSource.WriteError(TraceType, error);
ReleaseAssert.Failfast(error);
}
// If actionError is not null it means we are currently running a resumed rollback. In that case the ErrorCausingRollback must have
// already been set.
if (actionError != null)
{
actionState.ErrorCausingRollback = TranslateRollbackError(actionError.HResult);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Translated ErrorCausingRollback ={1}", actionState.OperationId, actionState.ErrorCausingRollback);
}
if (this.isTestMode && actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
// In test mode it's intentionally possible to fault an action after it's completed its work, but before the state name has been updated.
actionState.StateProgress.Pop();
}
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
try
{
while (actionState.StateProgress.Peek() != StepStateNames.IntentSaved &&
actionState.StateProgress.Peek() != StepStateNames.Failed)
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterCleanupLoop).ConfigureAwait(false);
if (readRollbackState == RollbackState.RollingBackDueToUserCancel)
{
// Do nothing, already rolling back - debug only
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Read RollingBackDueToUserCancel in outer rollback loop", actionState.OperationId);
}
else if (readRollbackState == RollbackState.RollingBackForce)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Read RollingBackForce in outer rollback loop", actionState.OperationId);
break;
}
StepStateNames currentStateName = actionState.StateProgress.Peek();
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - DEBUG - Rollback path loop, current state {1}", actionState.OperationId, actionState.StateProgress.Peek());
try
{
await this.CleanupStepAsync(fabricClient, action, actionState, cancellationToken, serviceInternalFaultInfo).ConfigureAwait(false);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
catch (FabricNotPrimaryException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
ReleaseAssert.Failfast("Unexpected exception, RunStateAsync for cleanup should have handled {0}", e);
}
}
// If this is true rollback is finished. If it is retryable set the state to LookingUpState
if (actionState.StateProgress.Peek() == StepStateNames.IntentSaved)
{
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
actionState.ClearInfo();
}
else if (actionState.RollbackState == RollbackState.RollingBackAndWillFailAction)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.RollbackState = RollbackState.NotRollingBack;
actionState.TimeStopped = DateTime.UtcNow;
}
else if (actionState.RollbackState == RollbackState.RollingBackDueToUserCancel)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
}
else if (actionState.RollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - RollbackState == NotRollingBack not expected", actionState.OperationId);
ReleaseAssert.Failfast(error);
}
}
else if (actionState.RollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
}
}
catch (OperationCanceledException)
{
// This means the cancellation token is set, not that an api call observed an E_ABORT
throw;
}
catch (FabricNotPrimaryException)
{
throw;
}
catch (FabricObjectClosedException)
{
throw;
}
catch (Exception e)
{
ReleaseAssert.Failfast("Unexpected exception, RunStateAsync for cleanup should have handled {0}", e);
}
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - Action failed, type='{1}', will retry={2}, RollbackState={3}",
actionState.OperationId,
actionState.ActionType,
actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction ? "true" : "false",
actionState.RollbackState);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
else if (actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
// user cancelled, but action/command completed anyways before cancellation was checked.
TestabilityTrace.TraceSource.WriteInfo(TraceType, "DEBUG {0} - Action type '{1}' completed successfully, not updating again ", actionState.OperationId, actionState.ActionType);
}
else if ((actionState.StateProgress.Peek() == StepStateNames.IntentSaved) &&
(actionState.RollbackState == RollbackState.RollingBackDueToUserCancel))
{
actionState.StateProgress.Push(StepStateNames.Failed);
actionState.TimeStopped = DateTime.UtcNow;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
else if (actionState.RollbackState == RollbackState.RollingBackForce)
{
// Note: unlike the case above this does not have a state of IntentSaved as a requirement since a force rollback is an abort and does run the steps in reverse.
// It is possible for the StateProgress to be CompletedSuccessfully here, since we want to exit as quickly as possible. In that case, the block 2 blocks above handles it -
// we do nothing extra, and the command finishes executing. If the user calls an api for information on this command, we translate the state to ForceCancelled if state is a terminal state
// and RollbackState is RollingBackForce. See ActionStore.MatchesStateFilter().
actionState.TimeStopped = DateTime.UtcNow;
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Bottom of Engine.RunAsync() - state is={0}, rollbackState={1}", actionState.StateProgress.Peek().ToString(), actionState.RollbackState.ToString());
actionState.StateProgress.Push(StepStateNames.Failed);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
}
else
{
string unexpectedError = string.Format(CultureInfo.InvariantCulture, "Unexpected case reached, state is={0}, rollbackState={1}", actionState.StateProgress.Peek().ToString(), actionState.RollbackState.ToString());
TestabilityTrace.TraceSource.WriteError(TraceType, "{0}", unexpectedError);
ReleaseAssert.Failfast(unexpectedError);
}
}
private void OnDestroy()
{
ActionTest test = GameObject.FindObjectOfType <ActionTest>();
test.m_Action -= MyAction;
}
//Abrupt camera change
void SwitchAngle90(ActionTest t)
{
var rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x+0.25f) * (xMax - xMin) + xMin,0);
t.targetPosition = rotation * new Vector3(0,0,-( z * (zMax - zMin) + zMin )) + lookPosition;
if(Terrain.activeTerrain != null) t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition)+1, 10000);
t.chosen = ()=>{ x += 0.25f; transform.position = t.targetPosition;};
t.Test();
if(!t.success)
{
rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x-0.25f) * (xMax - xMin) + xMin,0);
t.targetPosition = rotation * new Vector3(0,0,-( z * (zMax - zMin) + zMin )) + lookPosition;
if(Terrain.activeTerrain != null) t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition)+1, 10000);
t.chosen = ()=> { x -= 0.25f; transform.position = t.targetPosition;};
t.Test();
}
if(!t.success)
{
rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x+0.35f) * (xMax - xMin) + xMin,0);
t.targetPosition = rotation * new Vector3(0,0,-( z * (zMax - zMin) + zMin )) + lookPosition;
if(Terrain.activeTerrain != null) t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition)+1, 10000);
t.chosen = ()=> { x += 0.35f; transform.position = t.targetPosition;};
t.Test();
}
if(!t.success)
{
rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, (x-0.35f) * (xMax - xMin) + xMin,0);
t.targetPosition = rotation * new Vector3(0,0,-( z * (zMax - zMin) + zMin )) + lookPosition;
if(Terrain.activeTerrain != null) t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition)+1, 10000);
t.chosen = ()=> { x -= 0.35f; transform.position = t.targetPosition;};
t.Test();
}
t.distanceMoved = (transform.position - t.targetPosition).magnitude * cutWeighting;
}
public Task InsertCommandAsync(Command command)
{
Uri failoverManagerUri = new Uri("fabric:/System/FailoverManagerService");
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "****Adding command: " + command);
Task task = null;
if (command == Command.FailoverManagerDataLoss)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLoss];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, null);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeDataLossMidActionTestFatal)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFatal];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.InvokeDataLossMidActionTestTransient)
{
// rollback and retry then success
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestTransient];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeDataLossMidActionTestFailover)
{
// failover then success
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFailover];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatal)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatal];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossFailoverFaultAnalysisService)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossFailoverFaultAnalysisService];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeQuorumLossMidActionTestFatal)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFatal];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.InvokeQuorumLossMidActionTestFailover)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFailover];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeQuorumLossMidActionTestTransient)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestTransient];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.RestartPartitionMidActionTestFatal)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFatal];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.RestartPartitionMidActionTestFailover)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFailover];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.RestartPartitionMidActionTestTransient)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestTransient];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.StuckAction)
{
Guid id = MockClientCommandInfo[Command.StuckAction];
task = this.messageProcessor.ProcessStuckCommandAsync(id, null);
}
else if (command == Command.RestartPartitionCancelOuterLoopNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.None, // for this case this value does not matter
ServiceInternalFaultType.None,
RollbackState.RollingBackDueToUserCancel,
FASConstants.OuterLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelForwardNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.None,
ServiceInternalFaultType.None,
RollbackState.RollingBackDueToUserCancel,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelForwardExceptionNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.ForwardLoopExceptionBlock);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelOuterCleanupNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.OuterCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelCleanupInnerNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.InnerCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelOuterLoopForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions, // for this case, this value does not matter
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelForwardForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelForwardExceptionForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoopExceptionBlock);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelOuterCleanupForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.OuterCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelCleanupInnerForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.InnerCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.TestRetryStepWithSuccessAfterRetries)
{
// Intentionally fail the step corresponding to StepStateNames.PerformingActions step a few times, then run it normally (pass). It should succeed.
Guid id = MockClientCommandInfo[Command.TestRetryStepWithSuccessAfterRetries];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.ThrowThreeTimes);
task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
}
else if (command == Command.TestRetryStepWithForceCancel)
{
// Force cancel a command with ActionStateBase.RetryStepWithoutRollingBackOnFailure set to true
Guid id = MockClientCommandInfo[Command.TestRetryStepWithForceCancel];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.CompletedSuccessfully, // this just has to be a late step so an earlier fault is not used before we reach the situation we want.
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce, // note, the graceful one should not cause cancellation since for this type we only allow user cancellation when force is true
FASConstants.InnerForwardLoop,
StepStateNames.PerformingActions);
task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.StopNodeWithUnknownException)
{
Guid id = MockClientCommandInfo[Command.StopNodeWithUnknownException];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.RollbackAction); // In this case, since start and stop node do not rollback like other commands, this exception should cause the step to retry.
Node target = ActionTest.GetNodeWithFASSecondary().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId);
task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo);
// Let the command make progress
Task.Delay(TimeSpan.FromSeconds(30)).Wait();
this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.NotRollingBack);
// This should not result in cancellation, since start and stop node have different rollback policies than the other commands.
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==false. This should not cancel the command", id);
this.messageProcessor.CancelTestCommandAsync(id, false);
this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.RollingBackDueToUserCancel);
// Now force cancel. This should cancel.
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==true. This should cancel the command", id);
this.messageProcessor.CancelTestCommandAsync(id, true);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
NodeList nodes = ActionTest.GetNodeListAsync().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - node info:", id);
foreach (Node n in nodes)
{
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, " OperationId:{0} - NodeName{1}, NodeStatus:{2}, IsStopped:{3}", id, n.NodeName, n.NodeStatus, n.IsStopped);
}
Node targetNodeAfterTest = nodes.Where(n => n.NodeName == target.NodeName).FirstOrDefault();
if (targetNodeAfterTest == null)
{
throw new InvalidOperationException("target node was not found in query after test");
}
if (targetNodeAfterTest.IsStopped == false)
{
throw new InvalidOperationException("target node should have IsStopped true, was false");
}
}
else if (command == Command.StopNodeWithExceptionAndSuccessAfterRetries)
{
Guid id = MockClientCommandInfo[Command.StopNodeWithExceptionAndSuccessAfterRetries];
// Inject a fault during the operation so that step "StepStateNames.MidPerformingActions" has to retry 3 times before succeeding
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.ThrowThreeTimes);
Node target = ActionTest.GetNodeWithFASSecondary().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId);
task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
// Start the stopped node
task = this.messageProcessor.ProcessStartNodeCommandAsync(Guid.NewGuid(), target.NodeName, target.NodeInstanceId, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
}
else
{
ReleaseAssert.Failfast("Unexpected command");
}
return(task);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
InvokeDataLossState state = Convert(this.State);
PartitionSelector partitionSelector = state.Info.PartitionSelector;
DataLossMode dataLossMode = state.Info.DataLossMode;
long preDataLossNumber = state.Info.DataLossNumber;
string failoverManagerPrimaryNodeName = state.Info.NodeName;
Guid partitionId = state.Info.PartitionId;
string behaviorName = state.Info.UnreliableTransportInfo.First().Item2;
int targetReplicaSetSize = state.Info.TargetReplicaSetSize;
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - applying UT, partitionId={1}", this.State.OperationId, partitionId);
System.Fabric.Common.UnreliableTransportBehavior behavior = new System.Fabric.Common.UnreliableTransportBehavior("*", "DoReconfiguration");
behavior.AddFilterForPartitionId(partitionId);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
failoverManagerPrimaryNodeName,
behaviorName,
behavior,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
List <StatefulServiceReplica> replicaList = new List <StatefulServiceReplica>();
foreach (var replica in replicasResult)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Service Replica is not of stateful type even though service is stateful");
replicaList.Add(statefulReplica);
}
// Select target replicas based on the DataLosMode
List <StatefulServiceReplica> targets = null;
if (dataLossMode == DataLossMode.FullDataLoss)
{
targets = GetReplicasForFullDataLoss(replicaList);
}
else if (dataLossMode == DataLossMode.PartialDataLoss)
{
targets = FaultAnalysisServiceUtility.GetReplicasForPartialLoss(state.OperationId, replicaList);
}
else
{
throw FaultAnalysisServiceUtility.CreateException(StepBase.TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_INVALIDARG, Strings.StringResources.Error_UnsupportedDataLossMode);
}
if (targets == null)
{
// This will cause the command to rollback and retry
throw new FabricTransientException("The operation could not be performed, please retry", FabricErrorCode.NotReady);
}
foreach (var replica in targets)
{
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - Removing replica {1} in partition {2} with role {3} and status {4} to induce data loss",
this.State.OperationId,
replica.Id,
partitionId,
replica.ReplicaRole,
replica.ReplicaStatus);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.RemoveReplicaAsync(
replica.NodeName,
partitionId,
replica.Id,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveReplicaErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
await this.WaitForAllTargetReplicasToGetDroppedAsync(partitionId, targets, cancellationToken).ConfigureAwait(false);
await RemoveUnreliableTransportAsync(this.FabricClient, failoverManagerPrimaryNodeName, behaviorName, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false);
bool dataLossWasSuccessful = false;
TimeoutHelper timeoutHelper = new TimeoutHelper(TimeSpan.FromSeconds(30));
do
{
ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetPartitionListAsync(
this.partitionSelector.ServiceName,
null,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
bool partitionFound = false;
long postDataLossNumber = 0;
foreach (StatefulServicePartition partition in partitionsResult)
{
if (partition.PartitionInformation.Id == partitionId)
{
postDataLossNumber = partition.PrimaryEpoch.DataLossNumber;
partitionFound = true;
break;
}
}
if (!partitionFound)
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionNotFound), FabricErrorCode.PartitionNotFound);
}
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - Checking data loss numbers for partition {1} with remaining time {2}. Current numbers {3}:{4}",
this.State.OperationId,
partitionId,
timeoutHelper.GetRemainingTime(),
preDataLossNumber,
postDataLossNumber);
if (postDataLossNumber != preDataLossNumber)
{
dataLossWasSuccessful = true;
break;
}
await System.Fabric.Common.AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(this.dataLossCheckPollIntervalInSeconds), cancellationToken).ConfigureAwait(false);
}while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero);
if (!dataLossWasSuccessful)
{
// This is only viewable internally for debug. This will cause a retry of the whole flow.
string error = string.Format(
CultureInfo.InvariantCulture,
"{0} - Service could not induce data loss for service '{1}' partition '{2}' in '{3}' Please retry",
this.State.OperationId,
partitionSelector.ServiceName,
partitionId,
this.dataLossCheckWaitDurationInSeconds);
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, error);
throw new FabricTransientException("The operation could not be performed, please retry", FabricErrorCode.NotReady);
}
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
//Get closer
void MoveCloser(ActionTest t)
{
var rotation = Quaternion.Euler(y * (yMax - yMin) + yMin, x * (xMax - xMin) + xMin,0);
var relative = requestedPosition - lookPosition;
RaycastHit hit;
Physics.Raycast (lookPosition, relative, out hit, relative.magnitude + 0.5f);
var distance = Mathf.Clamp01((hit.distance - zMin)/(zMax - zMin) - 0.01f);
t.targetPosition = rotation * new Vector3(0,0,-( distance * (zMax - zMin) + zMin )) + lookPosition;
if(Terrain.activeTerrain != null) t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition)+1, 10000);
t.chosen = () => {transform.position = t.targetPosition; z = z - 0.01f;};
t.Test();
t.distanceMoved = (transform.position - t.targetPosition).magnitude * zoomWeighting;
}
//Rise above an object and look downwards
void RiseUp(ActionTest t)
{
t.targetPosition = requestedPosition;
t.success = false;
var offset = 0.002f;
while(!t.success && y+offset < 1)
{
var rotation = Quaternion.Euler(y+offset * (yMax - yMin) + yMin, x * (xMax - xMin) + xMin,0);
t.targetPosition = rotation * new Vector3(0,0,-( z * (zMax - zMin) + zMin )) + lookPosition;
if(Terrain.activeTerrain != null) t.targetPosition.y = Mathf.Clamp(t.targetPosition.y, Terrain.activeTerrain.SampleHeight(t.targetPosition)+1, 10000);
t.chosen = () => {transform.position = t.targetPosition; y = y + offset; };
t.Test();
t.distanceMoved = (transform.position - t.targetPosition).magnitude*riseWeighting;
offset = offset + 0.02f;
}
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
RestartPartitionState state = Convert(this.State);
Guid partitionId = state.Info.PartitionId;
bool hasPersistedState = state.Info.HasPersistedState;
string failoverManagerPrimaryNodeName = state.Info.NodeName;
string behaviorName = state.Info.UnreliableTransportInfo.First().Item2;
System.Fabric.Common.UnreliableTransportBehavior behavior = new System.Fabric.Common.UnreliableTransportBehavior("*", "DoReconfiguration");
behavior.AddFilterForPartitionId(partitionId);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
failoverManagerPrimaryNodeName,
behaviorName,
behavior,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - applied UT on partitionId {1}, node={2}", this.State.OperationId, partitionId, failoverManagerPrimaryNodeName);
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
var stableReplicasToRestart = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
foreach (var replica in stableReplicasToRestart)
{
if (this.restartPartitionMode == RestartPartitionMode.OnlyActiveSecondaries)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Stateful service replica is not StatefulServiceReplica");
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
continue;
}
}
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - restarting replica partition={1}, node={2}, replica id={3}", this.State.OperationId, partitionId, replica.NodeName, replica.Id);
if (hasPersistedState)
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.RestartReplicaAsync(
replica.NodeName,
partitionId,
replica.Id,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RestartReplicaErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
else
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.RemoveReplicaAsync(
replica.NodeName,
partitionId,
replica.Id,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveReplicaErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
await RemoveUnreliableTransportAsync(this.State, this.FabricClient, this.RequestTimeout, this.OperationTimeout, cancellationToken);
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
InvokeQuorumLossState state = Convert(this.State);
Guid partitionId = state.Info.PartitionId;
List <Tuple <string, string> > unreliableTransportInfo = state.Info.UnreliableTransportInfo;
List <long> targetReplicas = state.Info.ReplicaIds;
var unreliableTransportTaskList = new List <Task>();
List <Tuple <string, string> > unreliableTransportInfoList = new List <Tuple <string, string> >();
foreach (Tuple <string, string> ut in unreliableTransportInfo)
{
string nodeName = ut.Item1;
string behaviorName = ut.Item2;
System.Fabric.Common.UnreliableTransportBehavior behavior = new System.Fabric.Common.UnreliableTransportBehavior("*", "StatefulServiceReopen");
behavior.AddFilterForPartitionId(partitionId);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - applying '{1}'", this.State.OperationId, behaviorName);
unreliableTransportTaskList.Add(FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
nodeName,
behaviorName,
behavior,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken));
}
await Task.WhenAll(unreliableTransportTaskList).ConfigureAwait(false);
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
List <Task> tasks = new List <Task>();
foreach (long replicaId in targetReplicas)
{
ReplicaSelector replicaSelector = ReplicaSelector.ReplicaIdOf(PartitionSelector.PartitionIdOf(this.partitionSelector.ServiceName, partitionId), replicaId);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - faulting replica with id={1}", this.State.OperationId, replicaId);
Task task = FaultAnalysisServiceUtility.RestartReplicaAsync(this.FabricClient, replicaSelector, CompletionMode.DoNotVerify, this.RequestTimeout, this.OperationTimeout, cancellationToken);
tasks.Add(task);
}
await Task.WhenAll(tasks).ConfigureAwait(false);
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - keeping partition in quorum loss for '{1}'", this.State.OperationId, state.Info.QuorumLossDuration);
await Task.Delay(state.Info.QuorumLossDuration, cancellationToken).ConfigureAwait(false);
TimeoutHelper timeoutHelper = new TimeoutHelper(this.OperationTimeout);
bool conditionSatisfied = false;
int quorumLossCheckRetries = FASConstants.QuorumLossCheckRetryCount;
do
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - checking PartitionStatus", this.State.OperationId);
ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetPartitionListAsync(
this.partitionSelector.ServiceName,
null,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
foreach (StatefulServicePartition partition in partitionsResult)
{
if (partition.PartitionInformation.Id == partitionId)
{
if (partition.PartitionStatus == ServicePartitionStatus.InQuorumLoss)
{
conditionSatisfied = true;
break;
}
}
}
await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
}while (!conditionSatisfied && quorumLossCheckRetries-- > 0);
if (!conditionSatisfied)
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - Service could not induce quorum loss for service '{1}', partition '{2}'. Please retry", this.State.OperationId, this.partitionSelector.ServiceName, partitionId);
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, error);
throw new FabricTransientException("The operation could not be performed, please retry", FabricErrorCode.NotReady);
}
await QuorumLossStepsFactory.RemoveUTAsync(this.FabricClient, this.State, this.RequestTimeout, this.OperationTimeout, cancellationToken);
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
public StateTransition(ActionTest shouldExecute, ActionExecution execute, T nextState) : this(shouldExecute, execute, nextState, TransitionType.AfterSpell)
{
}