public async Task <long> GetActionCountAsync(bool excludeRunningActions) { long count = 0; try { using (var tx = this.stateManager.CreateTransaction()) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}, operationId.GetHashCode={2}, tx.TranslationId={3}, tx.GetHashCode={4}", "GetActionCountAsync", Guid.Empty, 0, tx.TransactionId, tx.GetHashCode()); if (!excludeRunningActions) { count = await this.actionTable.GetCountAsync(tx).ConfigureAwait(false); } count += await this.historyTable.GetCountAsync(tx).ConfigureAwait(false); await tx.CommitAsync().ConfigureAwait(false); } TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}, operationId.GetHashCode={2}", "GetActionCountAsync", Guid.Empty, 0); } catch (FabricNotPrimaryException fnp) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp); throw; } catch (Exception e) { TestabilityTrace.TraceSource.WriteError(TraceType, "GetActionCountAsync: {0}", e.ToString()); throw; } return(count); }
public async Task RunAsync(FabricClient fc, FabricTestAction action, ActionStateBase actionState, ServiceInternalFaultInfo serviceInternalFaultInfo, CancellationToken cancellationToken) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Inside RunAsync of Engine, entering state machine", actionState.OperationId); try { do { cancellationToken.ThrowIfCancellationRequested(); RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterLoop).ConfigureAwait(false); // For the non-force case we need to cleanup, so that is why there's no break statement in that case. if (readRollbackState == RollbackState.RollingBackForce) { actionState.StateProgress.Push(StepStateNames.Failed); await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false); break; } await this.RunStateMachineAsync(fc, action, actionState, serviceInternalFaultInfo, cancellationToken).ConfigureAwait(false); if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction) { actionState.ErrorCausingRollback = 0; int pauseTime = this.random.Next(10, 60); TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Pausing for {1} seconds before retrying", actionState.OperationId, pauseTime); // Clear the rollback state so it will go forward when it resumes. actionState.RollbackState = RollbackState.NotRollingBack; await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false); await Task.Delay(TimeSpan.FromSeconds(pauseTime), cancellationToken).ConfigureAwait(false); } }while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully && actionState.StateProgress.Peek() != StepStateNames.Failed); } catch (FabricNotPrimaryException notPrimary) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary); } catch (FabricObjectClosedException objectClosed) { FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed); } catch (Exception e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} caught exception - {1}", actionState.OperationId, e); throw; } TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Exiting state machine", actionState.OperationId); }
// This method should only run one at a time (controlled by timer) public async Task TruncateCallbackInnerAsync() { bool observedException = false; TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter truncate callback"); try { long count = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure( Guid.Empty, () => this.GetActionCountAsync(true), this.partition, "GetActionCountAsync", ActionStore.MaxRetries, this.cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action store size is {0}", count); if (count > this.maxStoredActionCount) { await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure( Guid.Empty, () => this.TruncateAsync(count), this.partition, "TruncateCallbackInnerAsync", ActionStore.MaxRetries, this.cancellationToken).ConfigureAwait(false); } } catch (FabricNotPrimaryException fnp) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp); observedException = true; } catch (FabricObjectClosedException foc) { FaultAnalysisServiceUtility.TraceFabricObjectClosed(Guid.Empty, foc); observedException = true; } catch (Exception e) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Error: {0}", e.ToString()); observedException = true; throw; } if (!observedException) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Rescheduling timer for {0} seconds", this.storedActionCleanupIntervalInSeconds); this.truncateTimer.Change(TimeSpan.FromSeconds(this.storedActionCleanupIntervalInSeconds), Timeout.InfiniteTimeSpan); } }
public async Task <ActionStateBase> FindByOperationIdAsync(Guid operationId, CancellationToken cancellationToken) { ActionStateBase state = null; try { using (var tx = this.stateManager.CreateTransaction()) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "FindByOperationId", operationId); var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false); if (data.HasValue) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, actiontable has value", operationId); state = this.ReadData(data.Value); } else { data = await this.historyTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false); if (data.HasValue) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, history table has value", operationId); state = this.ReadData(data.Value); } else { string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId); TestabilityTrace.TraceSource.WriteWarning(TraceType, error); throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error); } } } TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside {0} - operationId={1}", "FindByOperationId", operationId); } catch (FabricNotPrimaryException fnp) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp); throw; } catch (Exception e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "FindOperationId: {0}", e.ToString()); throw; } return(state); }
// Called to initialize state when a new message is received. private async Task InitializeNewActionInnerAsync(ActionStateBase actionState) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter InitializeNewActionAsync"); try { using (var tx = this.stateManager.CreateTransaction()) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId); bool keyAlreadyExists = await this.actionTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false); if (!keyAlreadyExists) { keyAlreadyExists = await this.historyTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false); } if (keyAlreadyExists) { string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' already exists", actionState.OperationId); TestabilityTrace.TraceSource.WriteWarning(TraceType, error); throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TEST_COMMAND_OPERATION_ID_ALREADY_EXISTS, error); } } TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId); } catch (FabricNotPrimaryException fnp) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, fnp); throw; } catch (Exception e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "Could not add new action: {0}", e.ToString()); throw; } await this.PersistAsync(actionState, true).ConfigureAwait(false); }
public async Task CancelTestCommandAsync(Guid operationId, bool force) { ActionStateBase state = null; try { using (var tx = this.stateManager.CreateTransaction()) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "CancelTestCommandAsync", operationId); var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false); if (data.HasValue) { state = this.ReadData(data.Value); } else { string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId); TestabilityTrace.TraceSource.WriteWarning(TraceType, error); throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error); } TestabilityTrace.TraceSource.WriteInfo(TraceType, "CancelActionAsync: operationId={0}, force={1}", operationId, force); if (force) { if (state.RollbackState != RollbackState.RollingBackDueToUserCancel) { string error = string.Format(CultureInfo.InvariantCulture, "CancelActionAsync: operationId={0}, force is not allowed for something in {1}", operationId, state.RollbackState.ToString()); TestabilityTrace.TraceSource.WriteError(TraceType, error); throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_TEST_COMMAND_STATE, error); } else { state.RollbackState = RollbackState.RollingBackForce; } } else { // if it's in force, don't allow it back down to graceful if (state.RollbackState == RollbackState.RollingBackForce) { string error = string.Format( CultureInfo.InvariantCulture, "CancelActionAsync: operationId={0}, CancelTestCommandAsync has already been called with force==true. Switching to force==false is not valid.", operationId); TestabilityTrace.TraceSource.WriteWarning(TraceType, error); throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_TEST_COMMAND_STATE, error); } state.RollbackState = RollbackState.RollingBackDueToUserCancel; } byte[] stateToPersist = state.ToBytes(); await this.actionTable.AddOrUpdateAsync(tx, state.OperationId, stateToPersist, (k, v) => stateToPersist, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false); await tx.CommitAsync().ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "CancelActionAsync: operationId={0}, force={1} committed", operationId, force); } TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "CancelTestCommandAsync", operationId); } catch (FabricNotPrimaryException fnp) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp); throw; } catch (Exception e) { TestabilityTrace.TraceSource.WriteError(TraceType, "CancelActionAsync: {0}", e.ToString()); throw; } }
// Launched as a task private async Task ConsumeAndRunActionsAsync(CancellationToken cancellationToken) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter ConsumeAndRunActions"); using (SemaphoreSlim semaphore = new SemaphoreSlim(this.concurrentRequests, this.concurrentRequests)) { while (!cancellationToken.IsCancellationRequested) { try { TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Waiting for semaphore, max count={0}", this.concurrentRequests); await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Done waiting for semaphore"); } catch (OperationCanceledException e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString()); break; } ActionStateBase actionState = null; bool wasSuccessful = false; try { TestabilityTrace.TraceSource.WriteNoise(TraceType, "Trying to Dequeue"); wasSuccessful = this.queue.TryTake(out actionState); if (!wasSuccessful) { TestabilityTrace.TraceSource.WriteNoise(TraceType, "Queue was empty"); semaphore.Release(); await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); continue; } TestabilityTrace.TraceSource.WriteNoise(TraceType, "Dequeued Name={0} Key={1}", actionState.ActionType, actionState.OperationId); } catch (OperationCanceledException e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString()); break; } catch (Exception ex) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "TryDequeue failed with={0}", ex.ToString()); semaphore.Release(); continue; } Task t = null; try { System.Fabric.FaultAnalysis.Service.Actions.FabricTestAction action = await this.ConstructActionAsync(actionState.ActionType, actionState); t = this.engine.RunAsync(this.fabricClient, action, actionState, actionState.ServiceInternalFaultInfo, cancellationToken); } catch (FabricNotPrimaryException notPrimary) { FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary); break; } catch (FabricObjectClosedException objectClosed) { FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed); break; } if (!this.pendingTasks.TryAdd(actionState.OperationId, new ActionCompletionInfo(actionState.OperationId, t, semaphore))) { TestabilityTrace.TraceSource.WriteError(TraceType, "Add of key={0} failed, was it already added?", actionState.OperationId); ReleaseAssert.Failfast("Add of key={0} failed, was it already added?", actionState.OperationId); } else { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action type={0}, key={1} started", actionState.ActionType, actionState.OperationId); } Interlocked.Increment(ref this.isShuttingDown); } TestabilityTrace.TraceSource.WriteWarning(TraceType, "Cancellation was requested"); return; } }