private CasterApiClient RefreshClient(CasterApiClient clientObject, TokenResponse tokenResponse, CancellationToken ct)
{
// TODO: check for token expiration also
if (clientObject == null)
{
clientObject = CasterApiExtensions.GetCasterApiClient(_httpClientFactory, _clientOptions.CurrentValue.urls.casterApi, tokenResponse);
}
return(clientObject);
}
public async Task <Event> RedeployAsync(Guid eventId, CancellationToken ct)
{
try
{
var eventEntity = await GetTheEventAsync(eventId, true, false, ct);
if (eventEntity.Status != EventStatus.Active)
{
var msg = $"Only an Active Event can be redeployed";
_logger.LogError(msg);
throw new Exception(msg);
}
var tokenResponse = await ApiClientsExtensions.RequestTokenAsync(_resourceOwnerAuthorizationOptions, _httpClientFactory.CreateClient());
var casterApiClient = CasterApiExtensions.GetCasterApiClient(_httpClientFactory, _clientOptions.urls.casterApi, tokenResponse);
var result = await casterApiClient.TaintResourcesAsync(
eventEntity.WorkspaceId.Value,
new Caster.Api.Client.TaintResourcesCommand {
SelectAll = true
},
ct);
if (result.Resources.Any(r => r.Tainted == false))
{
var msg = $"Taint failed";
_logger.LogError(msg);
throw new Exception(msg);
}
eventEntity.Status = EventStatus.Planning;
eventEntity.InternalStatus = InternalEventStatus.PlanningRedeploy;
await _context.SaveChangesAsync(ct);
// add the event to the event queue for AlloyBackgrounsService to process the caster destroy.
_alloyEventQueue.Add(eventEntity);
}
catch (Exception ex)
{
_logger.LogError($"Error ending Event {eventId}.", ex);
throw;
}
return(await GetAsync(eventId, ct));
}
private async void ProcessTheImplementation(Object implementationEntityAsObject)
{
var ct = new CancellationToken();
var implementationEntity = implementationEntityAsObject == null ? (ImplementationEntity)null : (ImplementationEntity)implementationEntityAsObject;
_logger.LogInformation($"Processing Implementation {implementationEntity.Id} for status '{implementationEntity.Status}'.");
try
{
using (var scope = _scopeFactory.CreateScope())
{
using (var alloyContext = scope.ServiceProvider.GetRequiredService <AlloyContext>())
{
var retryCount = 0;
var resourceCount = int.MaxValue;
var resourceRetryCount = 0;
// get the alloy context entities required
implementationEntity = alloyContext.Implementations.First(x => x.Id == implementationEntity.Id);
var definitionEntity = alloyContext.Definitions.First(x => x.Id == implementationEntity.DefinitionId);
// get the auth token
var tokenResponse = await ApiClientsExtensions.GetToken(scope);
CasterApiClient casterApiClient = null;
S3PlayerApiClient playerApiClient = null;
SteamfitterApiClient steamfitterApiClient = null;
// LOOP until this thread's process is complete
while (implementationEntity.Status == ImplementationStatus.Creating ||
implementationEntity.Status == ImplementationStatus.Planning ||
implementationEntity.Status == ImplementationStatus.Applying ||
implementationEntity.Status == ImplementationStatus.Ending)
{
// the updateTheEntity flag is used to indicate if the implementation entity state should be updated at the end of this loop
var updateTheEntity = false;
// each time through the loop, one state (case) is handled based on Status and InternalStatus. This allows for retries of a failed state.
switch (implementationEntity.Status)
{
// the "Creating" status means we are creating the initial player exercise, steamfitter session and caster workspace
case ImplementationStatus.Creating:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.LaunchQueued:
case InternalImplementationStatus.CreatingExercise:
{
if (definitionEntity.ExerciseId == null)
{
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
updateTheEntity = true;
}
else
{
try
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
var exerciseId = await PlayerApiExtensions.CreatePlayerExerciseAsync(playerApiClient, implementationEntity, (Guid)definitionEntity.ExerciseId, ct);
if (exerciseId != null)
{
implementationEntity.ExerciseId = exerciseId;
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
catch (Exception ex)
{
_logger.LogError($"Error creating the player exercise for Implementation {implementationEntity.Id}.", ex);
retryCount++;
}
}
break;
}
case InternalImplementationStatus.CreatingSession:
{
if (definitionEntity.ScenarioId == null)
{
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
updateTheEntity = true;
}
else
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
var session = await SteamfitterApiExtensions.CreateSteamfitterSessionAsync(steamfitterApiClient, implementationEntity, (Guid)definitionEntity.ScenarioId, ct);
if (session != null)
{
implementationEntity.SessionId = session.Id;
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
break;
}
case InternalImplementationStatus.CreatingWorkspace:
{
if (definitionEntity.DirectoryId == null)
{
// There is no Caster directory, so start the session
var launchDate = DateTime.UtcNow;
implementationEntity.Name = definitionEntity.Name;
implementationEntity.Description = definitionEntity.Description;
implementationEntity.LaunchDate = launchDate;
implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
implementationEntity.Status = ImplementationStatus.Applying;
implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
updateTheEntity = true;
}
else
{
var varsFileContent = "";
if (implementationEntity.ExerciseId != null)
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
varsFileContent = await CasterApiExtensions.GetCasterVarsFileContentAsync(implementationEntity, playerApiClient, ct);
}
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var workspaceId = await CasterApiExtensions.CreateCasterWorkspaceAsync(casterApiClient, implementationEntity, (Guid)definitionEntity.DirectoryId, varsFileContent, ct);
if (workspaceId != null)
{
implementationEntity.WorkspaceId = workspaceId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningLaunch;
implementationEntity.Status = ImplementationStatus.Planning;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Planning" state means that caster is planning a run
case ImplementationStatus.Planning:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.PlanningLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, false, ct);
if (runId != null)
{
implementationEntity.RunId = runId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlannedLaunch;
updateTheEntity = true;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.PlannedLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingLaunch;
implementationEntity.Status = ImplementationStatus.Applying;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Applying" state means caster is applying a run (deploying VM's, etc.)
case ImplementationStatus.Applying:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.ApplyingLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.AppliedLaunch;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.AppliedLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDeployMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.StartingSession:
{
// start the steamfitter session, if there is one
if (implementationEntity.SessionId != null)
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
updateTheEntity = await SteamfitterApiExtensions.StartSteamfitterSessionAsync(steamfitterApiClient, (Guid)implementationEntity.SessionId, ct);
}
else
{
updateTheEntity = true;
}
// moving on means that Launch is now complete
if (updateTheEntity)
{
var launchDate = DateTime.UtcNow;
implementationEntity.Name = definitionEntity.Name;
implementationEntity.Description = definitionEntity.Description;
implementationEntity.LaunchDate = launchDate;
implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
implementationEntity.Status = ImplementationStatus.Active;
implementationEntity.InternalStatus = InternalImplementationStatus.Launched;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Ending" state means all entities are being torn down
case ImplementationStatus.Ending:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.EndQueued:
case InternalImplementationStatus.DeletingExercise:
{
if (implementationEntity.ExerciseId != null)
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
updateTheEntity = await PlayerApiExtensions.DeletePlayerExerciseAsync(_clientOptions.CurrentValue.urls.playerApi, implementationEntity.ExerciseId, playerApiClient, ct);
}
else
{
updateTheEntity = true;
}
if (updateTheEntity)
{
implementationEntity.ExerciseId = null;
implementationEntity.InternalStatus = InternalImplementationStatus.DeletingSession;
}
break;
}
case InternalImplementationStatus.DeletingSession:
{
if (implementationEntity.SessionId != null)
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
updateTheEntity = await SteamfitterApiExtensions.EndSteamfitterSessionAsync(_clientOptions.CurrentValue.urls.steamfitterApi, implementationEntity.SessionId, steamfitterApiClient, ct);
}
else
{
updateTheEntity = true;
}
if (updateTheEntity)
{
implementationEntity.SessionId = null;
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.PlanningDestroy:
{
if (implementationEntity.WorkspaceId != null)
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, true, ct);
if (runId != null)
{
implementationEntity.RunId = runId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlannedDestroy;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
else
{
implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
implementationEntity.Status = ImplementationStatus.Ended;
updateTheEntity = true;
}
break;
}
case InternalImplementationStatus.PlannedDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.ApplyingDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.AppliedDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.AppliedDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDestroyMaxWaitMinutes, ct);
// all conditions in this case require an implementation entity update
updateTheEntity = true;
// make sure that the run successfully deleted the resources
var count = (await casterApiClient.GetResourcesByWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct)).Count();
implementationEntity.RunId = null;
if (count == 0)
{
// resources deleted, so continue to delete the workspace
implementationEntity.InternalStatus = InternalImplementationStatus.DeletingWorkspace;
}
else
{
if (count < resourceCount)
{
// still some resources, but making progress, try the whole process again
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
resourceRetryCount = 0;
}
else
{
// still some resources and not making progress. Check max retries.
if (resourceRetryCount < _clientOptions.CurrentValue.ApiClientFailureMaxRetries)
{
// try the whole process again after a wait
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
resourceRetryCount++;
Thread.Sleep(TimeSpan.FromMinutes(_clientOptions.CurrentValue.CasterDestroyRetryDelayMinutes));
}
else
{
// the caster workspace resources could not be destroyed
implementationEntity.InternalStatus = InternalImplementationStatus.FailedDestroy;
implementationEntity.Status = ImplementationStatus.Failed;
}
}
}
break;
}
case InternalImplementationStatus.DeletingWorkspace:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.DeleteCasterWorkspaceAsync(implementationEntity, casterApiClient, tokenResponse, ct);
if (updateTheEntity)
{
implementationEntity.WorkspaceId = null;
implementationEntity.Status = ImplementationStatus.Ended;
implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
}
// check for exceeding the max number of retries
if (!updateTheEntity)
{
if (retryCount >= _clientOptions.CurrentValue.ApiClientFailureMaxRetries && _clientOptions.CurrentValue.ApiClientFailureMaxRetries > 0)
{
_logger.LogError($"Retry count exceeded for Implementation {implementationEntity.Id}, with status of {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
}
else
{
Thread.Sleep(TimeSpan.FromSeconds(_clientOptions.CurrentValue.ApiClientRetryIntervalSeconds));
}
}
// update the entity in the context, if we are moving on
if (updateTheEntity)
{
retryCount = 0;
implementationEntity.StatusDate = DateTime.UtcNow;
await alloyContext.SaveChangesAsync(ct);
}
}
}
}
}
catch (Exception ex)
{
_logger.LogError($"Error processing implementation {implementationEntity.Id}", ex);
}
}
