public static async Task <Guid?> CreateCasterWorkspaceAsync(CasterApiClient casterApiClient, EventEntity eventEntity, Guid directoryId, string varsFileContent, bool useDynamicHost, CancellationToken ct) { try { // remove special characters from the user name, use lower case and replace spaces with underscores var userName = Regex.Replace(eventEntity.Username.ToLower().Replace(" ", "_"), "[@&'(\\s)<>#]", "", RegexOptions.None); // create the new workspace var workspaceCommand = new CreateWorkspaceCommand() { Name = $"{userName}-{eventEntity.UserId.ToString()}", DirectoryId = directoryId, DynamicHost = useDynamicHost }; var workspaceId = (await casterApiClient.CreateWorkspaceAsync(workspaceCommand, ct)).Id; // create the workspace variable file var createFileCommand = new CreateFileCommand() { Name = $"{workspaceCommand.Name}.auto.tfvars", DirectoryId = directoryId, WorkspaceId = workspaceId, Content = varsFileContent }; await casterApiClient.CreateFileAsync(createFileCommand, ct); return(workspaceId); } catch (Exception ex) { return(null); } }
public static async Task <Guid?> CreateRunAsync( EventEntity eventEntity, CasterApiClient casterApiClient, bool isDestroy, ILogger logger, CancellationToken ct) { var runCommand = new CreateRunCommand() { WorkspaceId = eventEntity.WorkspaceId.Value, IsDestroy = isDestroy }; try { var casterRun = await casterApiClient.CreateRunAsync(runCommand, ct); return(casterRun.Id); } catch (Exception ex) { logger.LogError(ex, "Error Creating Run"); return(null); } }
public static CasterApiClient GetCasterApiClient(IHttpClientFactory httpClientFactory, string apiUrl, TokenResponse tokenResponse) { var client = ApiClientsExtensions.GetHttpClient(httpClientFactory, apiUrl, tokenResponse); var apiClient = new CasterApiClient(client); return(apiClient); }
public static async Task <bool> IsWorkspaceEmpty( EventEntity eventEntity, CasterApiClient casterApiClient, ILogger logger, CancellationToken ct) { var isEmpty = true; if (eventEntity.WorkspaceId.HasValue) { try { var resources = await casterApiClient.GetResourcesByWorkspaceAsync(eventEntity.WorkspaceId.Value); if (resources.Count > 0) { isEmpty = false; } } catch (Exception ex) { isEmpty = false; logger.LogError(ex, $"Error checking Resources for Workspace {eventEntity.WorkspaceId} in Event {eventEntity.Id}"); } } return(isEmpty); }
public static async Task <bool> WaitForRunToBeAppliedAsync( ImplementationEntity implementationEntity, CasterApiClient casterApiClient, int loopIntervalSeconds, int maxWaitMinutes, CancellationToken ct) { if (implementationEntity.RunId == null) { return(false); } var endTime = DateTime.UtcNow.AddMinutes(maxWaitMinutes); var status = "Applying"; while (status == "Applying" && DateTime.UtcNow < endTime) { var casterRun = await casterApiClient.GetRunAsync((Guid)implementationEntity.RunId); status = casterRun.Status; // if not there yet, pause before the next check if (status == "Applying") { Thread.Sleep(TimeSpan.FromSeconds(loopIntervalSeconds)); } } if (status == "Applied") { return(true); } else { return(false); } }
public static async Task <bool> DeleteCasterWorkspaceAsync(EventEntity eventEntity, CasterApiClient casterApiClient, TokenResponse tokenResponse, CancellationToken ct) { try { await casterApiClient.DeleteWorkspaceAsync((Guid)eventEntity.WorkspaceId, ct); return(true); } catch (Caster.Api.Client.ApiException ex) { if (ex.StatusCode == (int)HttpStatusCode.NotFound || ex.StatusCode == (int)HttpStatusCode.NoContent) { return(true); } else { return(false); } } catch (Exception) { return(false); } }
public static async Task <bool> WaitForRunToBePlannedAsync( EventEntity eventEntity, CasterApiClient casterApiClient, int loopIntervalSeconds, int maxWaitMinutes, ILogger logger, CancellationToken ct) { if (eventEntity.RunId == null) { return(false); } var endTime = DateTime.UtcNow.AddMinutes(maxWaitMinutes); var status = RunStatus.Planning; while ((status == RunStatus.Queued || status == RunStatus.Planning) && DateTime.UtcNow < endTime) { var casterRun = await casterApiClient.GetRunAsync((Guid)eventEntity.RunId, false, false); status = casterRun.Status; // if not there yet, pause before the next check if (status == RunStatus.Planning || status == RunStatus.Queued) { Thread.Sleep(TimeSpan.FromSeconds(loopIntervalSeconds)); } } if (status == RunStatus.Planned) { return(true); } else { return(false); } }
private CasterApiClient RefreshClient(CasterApiClient clientObject, TokenResponse tokenResponse, CancellationToken ct) { // TODO: check for token expiration also if (clientObject == null) { clientObject = CasterApiExtensions.GetCasterApiClient(_httpClientFactory, _clientOptions.CurrentValue.urls.casterApi, tokenResponse); } return(clientObject); }
public static async Task <bool> DeleteCasterWorkspaceAsync(ImplementationEntity implementationEntity, CasterApiClient casterApiClient, TokenResponse tokenResponse, CancellationToken ct) { try { await casterApiClient.DeleteWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct); return(true); } catch (Exception ex) { return(false); } }
public static async Task <bool> ApplyRunAsync( EventEntity eventEntity, CasterApiClient casterApiClient, CancellationToken ct) { var initialInternalStatus = eventEntity.InternalStatus; // if status is Planned or Applying try { await casterApiClient.ApplyRunAsync((Guid)eventEntity.RunId, ct); return(true); } catch (Exception ex) { return(false); } }
public static void AddCasterApiClient(this IServiceCollection services) { services.AddScoped <ICasterApiClient, CasterApiClient>(p => { var httpContextAccessor = p.GetRequiredService <IHttpContextAccessor>(); var httpClientFactory = p.GetRequiredService <IHttpClientFactory>(); var clientOptions = p.GetRequiredService <ClientOptions>(); var casterUri = new Uri(clientOptions.urls.casterApi); string authHeader = httpContextAccessor.HttpContext.Request.Headers["Authorization"]; var httpClient = httpClientFactory.CreateClient(); httpClient.BaseAddress = casterUri; httpClient.DefaultRequestHeaders.Add("Authorization", authHeader); var apiClient = new CasterApiClient(httpClient); return(apiClient); }); }
public static async Task <Guid?> CreateRunAsync( ImplementationEntity implementationEntity, CasterApiClient casterApiClient, bool isDestroy, CancellationToken ct) { var runCommand = new CreateRunCommand() { WorkspaceId = implementationEntity.WorkspaceId, IsDestroy = isDestroy }; try { var casterRun = await casterApiClient.CreateRunAsync(runCommand, ct); return(casterRun.Id); } catch (Exception ex) { return(null); } }
private async void ProcessTheImplementation(Object implementationEntityAsObject) { var ct = new CancellationToken(); var implementationEntity = implementationEntityAsObject == null ? (ImplementationEntity)null : (ImplementationEntity)implementationEntityAsObject; _logger.LogInformation($"Processing Implementation {implementationEntity.Id} for status '{implementationEntity.Status}'."); try { using (var scope = _scopeFactory.CreateScope()) { using (var alloyContext = scope.ServiceProvider.GetRequiredService <AlloyContext>()) { var retryCount = 0; var resourceCount = int.MaxValue; var resourceRetryCount = 0; // get the alloy context entities required implementationEntity = alloyContext.Implementations.First(x => x.Id == implementationEntity.Id); var definitionEntity = alloyContext.Definitions.First(x => x.Id == implementationEntity.DefinitionId); // get the auth token var tokenResponse = await ApiClientsExtensions.GetToken(scope); CasterApiClient casterApiClient = null; S3PlayerApiClient playerApiClient = null; SteamfitterApiClient steamfitterApiClient = null; // LOOP until this thread's process is complete while (implementationEntity.Status == ImplementationStatus.Creating || implementationEntity.Status == ImplementationStatus.Planning || implementationEntity.Status == ImplementationStatus.Applying || implementationEntity.Status == ImplementationStatus.Ending) { // the updateTheEntity flag is used to indicate if the implementation entity state should be updated at the end of this loop var updateTheEntity = false; // each time through the loop, one state (case) is handled based on Status and InternalStatus. This allows for retries of a failed state. switch (implementationEntity.Status) { // the "Creating" status means we are creating the initial player exercise, steamfitter session and caster workspace case ImplementationStatus.Creating: { switch (implementationEntity.InternalStatus) { case InternalImplementationStatus.LaunchQueued: case InternalImplementationStatus.CreatingExercise: { if (definitionEntity.ExerciseId == null) { implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession; updateTheEntity = true; } else { try { playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct); var exerciseId = await PlayerApiExtensions.CreatePlayerExerciseAsync(playerApiClient, implementationEntity, (Guid)definitionEntity.ExerciseId, ct); if (exerciseId != null) { implementationEntity.ExerciseId = exerciseId; implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession; updateTheEntity = true; } else { retryCount++; } } catch (Exception ex) { _logger.LogError($"Error creating the player exercise for Implementation {implementationEntity.Id}.", ex); retryCount++; } } break; } case InternalImplementationStatus.CreatingSession: { if (definitionEntity.ScenarioId == null) { implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace; updateTheEntity = true; } else { steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct); var session = await SteamfitterApiExtensions.CreateSteamfitterSessionAsync(steamfitterApiClient, implementationEntity, (Guid)definitionEntity.ScenarioId, ct); if (session != null) { implementationEntity.SessionId = session.Id; implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace; updateTheEntity = true; } else { retryCount++; } } break; } case InternalImplementationStatus.CreatingWorkspace: { if (definitionEntity.DirectoryId == null) { // There is no Caster directory, so start the session var launchDate = DateTime.UtcNow; implementationEntity.Name = definitionEntity.Name; implementationEntity.Description = definitionEntity.Description; implementationEntity.LaunchDate = launchDate; implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours); implementationEntity.Status = ImplementationStatus.Applying; implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession; updateTheEntity = true; } else { var varsFileContent = ""; if (implementationEntity.ExerciseId != null) { playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct); varsFileContent = await CasterApiExtensions.GetCasterVarsFileContentAsync(implementationEntity, playerApiClient, ct); } casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); var workspaceId = await CasterApiExtensions.CreateCasterWorkspaceAsync(casterApiClient, implementationEntity, (Guid)definitionEntity.DirectoryId, varsFileContent, ct); if (workspaceId != null) { implementationEntity.WorkspaceId = workspaceId; implementationEntity.InternalStatus = InternalImplementationStatus.PlanningLaunch; implementationEntity.Status = ImplementationStatus.Planning; updateTheEntity = true; } else { retryCount++; } } break; } default: { _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}"); implementationEntity.Status = ImplementationStatus.Failed; updateTheEntity = true; break; } } break; } // the "Planning" state means that caster is planning a run case ImplementationStatus.Planning: { switch (implementationEntity.InternalStatus) { case InternalImplementationStatus.PlanningLaunch: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, false, ct); if (runId != null) { implementationEntity.RunId = runId; implementationEntity.InternalStatus = InternalImplementationStatus.PlannedLaunch; updateTheEntity = true; } else { retryCount++; } break; } case InternalImplementationStatus.PlannedLaunch: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct); if (updateTheEntity) { implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingLaunch; implementationEntity.Status = ImplementationStatus.Applying; } else { retryCount++; } break; } default: { _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}"); implementationEntity.Status = ImplementationStatus.Failed; updateTheEntity = true; break; } } break; } // the "Applying" state means caster is applying a run (deploying VM's, etc.) case ImplementationStatus.Applying: { switch (implementationEntity.InternalStatus) { case InternalImplementationStatus.ApplyingLaunch: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct); if (updateTheEntity) { implementationEntity.InternalStatus = InternalImplementationStatus.AppliedLaunch; } else { retryCount++; } break; } case InternalImplementationStatus.AppliedLaunch: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); updateTheEntity = await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDeployMaxWaitMinutes, ct); if (updateTheEntity) { implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession; } else { retryCount++; } break; } case InternalImplementationStatus.StartingSession: { // start the steamfitter session, if there is one if (implementationEntity.SessionId != null) { steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct); updateTheEntity = await SteamfitterApiExtensions.StartSteamfitterSessionAsync(steamfitterApiClient, (Guid)implementationEntity.SessionId, ct); } else { updateTheEntity = true; } // moving on means that Launch is now complete if (updateTheEntity) { var launchDate = DateTime.UtcNow; implementationEntity.Name = definitionEntity.Name; implementationEntity.Description = definitionEntity.Description; implementationEntity.LaunchDate = launchDate; implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours); implementationEntity.Status = ImplementationStatus.Active; implementationEntity.InternalStatus = InternalImplementationStatus.Launched; } else { retryCount++; } break; } default: { _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}"); implementationEntity.Status = ImplementationStatus.Failed; updateTheEntity = true; break; } } break; } // the "Ending" state means all entities are being torn down case ImplementationStatus.Ending: { switch (implementationEntity.InternalStatus) { case InternalImplementationStatus.EndQueued: case InternalImplementationStatus.DeletingExercise: { if (implementationEntity.ExerciseId != null) { playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct); updateTheEntity = await PlayerApiExtensions.DeletePlayerExerciseAsync(_clientOptions.CurrentValue.urls.playerApi, implementationEntity.ExerciseId, playerApiClient, ct); } else { updateTheEntity = true; } if (updateTheEntity) { implementationEntity.ExerciseId = null; implementationEntity.InternalStatus = InternalImplementationStatus.DeletingSession; } break; } case InternalImplementationStatus.DeletingSession: { if (implementationEntity.SessionId != null) { steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct); updateTheEntity = await SteamfitterApiExtensions.EndSteamfitterSessionAsync(_clientOptions.CurrentValue.urls.steamfitterApi, implementationEntity.SessionId, steamfitterApiClient, ct); } else { updateTheEntity = true; } if (updateTheEntity) { implementationEntity.SessionId = null; implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy; } else { retryCount++; } break; } case InternalImplementationStatus.PlanningDestroy: { if (implementationEntity.WorkspaceId != null) { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, true, ct); if (runId != null) { implementationEntity.RunId = runId; implementationEntity.InternalStatus = InternalImplementationStatus.PlannedDestroy; updateTheEntity = true; } else { retryCount++; } } else { implementationEntity.InternalStatus = InternalImplementationStatus.Ended; implementationEntity.Status = ImplementationStatus.Ended; updateTheEntity = true; } break; } case InternalImplementationStatus.PlannedDestroy: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct); if (updateTheEntity) { implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingDestroy; } else { retryCount++; } break; } case InternalImplementationStatus.ApplyingDestroy: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct); if (updateTheEntity) { implementationEntity.InternalStatus = InternalImplementationStatus.AppliedDestroy; } else { retryCount++; } break; } case InternalImplementationStatus.AppliedDestroy: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDestroyMaxWaitMinutes, ct); // all conditions in this case require an implementation entity update updateTheEntity = true; // make sure that the run successfully deleted the resources var count = (await casterApiClient.GetResourcesByWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct)).Count(); implementationEntity.RunId = null; if (count == 0) { // resources deleted, so continue to delete the workspace implementationEntity.InternalStatus = InternalImplementationStatus.DeletingWorkspace; } else { if (count < resourceCount) { // still some resources, but making progress, try the whole process again implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy; resourceRetryCount = 0; } else { // still some resources and not making progress. Check max retries. if (resourceRetryCount < _clientOptions.CurrentValue.ApiClientFailureMaxRetries) { // try the whole process again after a wait implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy; resourceRetryCount++; Thread.Sleep(TimeSpan.FromMinutes(_clientOptions.CurrentValue.CasterDestroyRetryDelayMinutes)); } else { // the caster workspace resources could not be destroyed implementationEntity.InternalStatus = InternalImplementationStatus.FailedDestroy; implementationEntity.Status = ImplementationStatus.Failed; } } } break; } case InternalImplementationStatus.DeletingWorkspace: { casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct); updateTheEntity = await CasterApiExtensions.DeleteCasterWorkspaceAsync(implementationEntity, casterApiClient, tokenResponse, ct); if (updateTheEntity) { implementationEntity.WorkspaceId = null; implementationEntity.Status = ImplementationStatus.Ended; implementationEntity.InternalStatus = InternalImplementationStatus.Ended; } else { retryCount++; } break; } default: { _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}"); implementationEntity.Status = ImplementationStatus.Failed; updateTheEntity = true; break; } } break; } } // check for exceeding the max number of retries if (!updateTheEntity) { if (retryCount >= _clientOptions.CurrentValue.ApiClientFailureMaxRetries && _clientOptions.CurrentValue.ApiClientFailureMaxRetries > 0) { _logger.LogError($"Retry count exceeded for Implementation {implementationEntity.Id}, with status of {implementationEntity.Status} - {implementationEntity.InternalStatus}"); implementationEntity.Status = ImplementationStatus.Failed; updateTheEntity = true; } else { Thread.Sleep(TimeSpan.FromSeconds(_clientOptions.CurrentValue.ApiClientRetryIntervalSeconds)); } } // update the entity in the context, if we are moving on if (updateTheEntity) { retryCount = 0; implementationEntity.StatusDate = DateTime.UtcNow; await alloyContext.SaveChangesAsync(ct); } } } } } catch (Exception ex) { _logger.LogError($"Error processing implementation {implementationEntity.Id}", ex); } }