public static async Task <Guid?> CreateCasterWorkspaceAsync(CasterApiClient casterApiClient, EventEntity eventEntity, Guid directoryId, string varsFileContent, bool useDynamicHost, CancellationToken ct)
{
try
{
// remove special characters from the user name, use lower case and replace spaces with underscores
var userName = Regex.Replace(eventEntity.Username.ToLower().Replace(" ", "_"), "[@&'(\\s)<>#]", "", RegexOptions.None);
// create the new workspace
var workspaceCommand = new CreateWorkspaceCommand()
{
Name = $"{userName}-{eventEntity.UserId.ToString()}",
DirectoryId = directoryId,
DynamicHost = useDynamicHost
};
var workspaceId = (await casterApiClient.CreateWorkspaceAsync(workspaceCommand, ct)).Id;
// create the workspace variable file
var createFileCommand = new CreateFileCommand()
{
Name = $"{workspaceCommand.Name}.auto.tfvars",
DirectoryId = directoryId,
WorkspaceId = workspaceId,
Content = varsFileContent
};
await casterApiClient.CreateFileAsync(createFileCommand, ct);
return(workspaceId);
}
catch (Exception ex)
{
return(null);
}
}
public static async Task <Guid?> CreateRunAsync(
EventEntity eventEntity,
CasterApiClient casterApiClient,
bool isDestroy,
ILogger logger,
CancellationToken ct)
{
var runCommand = new CreateRunCommand()
{
WorkspaceId = eventEntity.WorkspaceId.Value,
IsDestroy = isDestroy
};
try
{
var casterRun = await casterApiClient.CreateRunAsync(runCommand, ct);
return(casterRun.Id);
}
catch (Exception ex)
{
logger.LogError(ex, "Error Creating Run");
return(null);
}
}
public static CasterApiClient GetCasterApiClient(IHttpClientFactory httpClientFactory, string apiUrl, TokenResponse tokenResponse)
{
var client = ApiClientsExtensions.GetHttpClient(httpClientFactory, apiUrl, tokenResponse);
var apiClient = new CasterApiClient(client);
return(apiClient);
}
public static async Task <bool> IsWorkspaceEmpty(
EventEntity eventEntity,
CasterApiClient casterApiClient,
ILogger logger,
CancellationToken ct)
{
var isEmpty = true;
if (eventEntity.WorkspaceId.HasValue)
{
try
{
var resources = await casterApiClient.GetResourcesByWorkspaceAsync(eventEntity.WorkspaceId.Value);
if (resources.Count > 0)
{
isEmpty = false;
}
}
catch (Exception ex)
{
isEmpty = false;
logger.LogError(ex, $"Error checking Resources for Workspace {eventEntity.WorkspaceId} in Event {eventEntity.Id}");
}
}
return(isEmpty);
}
public static async Task <bool> WaitForRunToBeAppliedAsync(
ImplementationEntity implementationEntity,
CasterApiClient casterApiClient,
int loopIntervalSeconds,
int maxWaitMinutes,
CancellationToken ct)
{
if (implementationEntity.RunId == null)
{
return(false);
}
var endTime = DateTime.UtcNow.AddMinutes(maxWaitMinutes);
var status = "Applying";
while (status == "Applying" && DateTime.UtcNow < endTime)
{
var casterRun = await casterApiClient.GetRunAsync((Guid)implementationEntity.RunId);
status = casterRun.Status;
// if not there yet, pause before the next check
if (status == "Applying")
{
Thread.Sleep(TimeSpan.FromSeconds(loopIntervalSeconds));
}
}
if (status == "Applied")
{
return(true);
}
else
{
return(false);
}
}
public static async Task <bool> DeleteCasterWorkspaceAsync(EventEntity eventEntity,
CasterApiClient casterApiClient, TokenResponse tokenResponse, CancellationToken ct)
{
try
{
await casterApiClient.DeleteWorkspaceAsync((Guid)eventEntity.WorkspaceId, ct);
return(true);
}
catch (Caster.Api.Client.ApiException ex)
{
if (ex.StatusCode == (int)HttpStatusCode.NotFound || ex.StatusCode == (int)HttpStatusCode.NoContent)
{
return(true);
}
else
{
return(false);
}
}
catch (Exception)
{
return(false);
}
}
public static async Task <bool> WaitForRunToBePlannedAsync(
EventEntity eventEntity,
CasterApiClient casterApiClient,
int loopIntervalSeconds,
int maxWaitMinutes,
ILogger logger,
CancellationToken ct)
{
if (eventEntity.RunId == null)
{
return(false);
}
var endTime = DateTime.UtcNow.AddMinutes(maxWaitMinutes);
var status = RunStatus.Planning;
while ((status == RunStatus.Queued || status == RunStatus.Planning) && DateTime.UtcNow < endTime)
{
var casterRun = await casterApiClient.GetRunAsync((Guid)eventEntity.RunId, false, false);
status = casterRun.Status;
// if not there yet, pause before the next check
if (status == RunStatus.Planning || status == RunStatus.Queued)
{
Thread.Sleep(TimeSpan.FromSeconds(loopIntervalSeconds));
}
}
if (status == RunStatus.Planned)
{
return(true);
}
else
{
return(false);
}
}
private CasterApiClient RefreshClient(CasterApiClient clientObject, TokenResponse tokenResponse, CancellationToken ct)
{
// TODO: check for token expiration also
if (clientObject == null)
{
clientObject = CasterApiExtensions.GetCasterApiClient(_httpClientFactory, _clientOptions.CurrentValue.urls.casterApi, tokenResponse);
}
return(clientObject);
}
public static async Task <bool> DeleteCasterWorkspaceAsync(ImplementationEntity implementationEntity,
CasterApiClient casterApiClient, TokenResponse tokenResponse, CancellationToken ct)
{
try
{
await casterApiClient.DeleteWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct);
return(true);
}
catch (Exception ex)
{
return(false);
}
}
public static async Task <bool> ApplyRunAsync(
EventEntity eventEntity,
CasterApiClient casterApiClient,
CancellationToken ct)
{
var initialInternalStatus = eventEntity.InternalStatus;
// if status is Planned or Applying
try
{
await casterApiClient.ApplyRunAsync((Guid)eventEntity.RunId, ct);
return(true);
}
catch (Exception ex)
{
return(false);
}
}
public static void AddCasterApiClient(this IServiceCollection services)
{
services.AddScoped <ICasterApiClient, CasterApiClient>(p =>
{
var httpContextAccessor = p.GetRequiredService <IHttpContextAccessor>();
var httpClientFactory = p.GetRequiredService <IHttpClientFactory>();
var clientOptions = p.GetRequiredService <ClientOptions>();
var casterUri = new Uri(clientOptions.urls.casterApi);
string authHeader = httpContextAccessor.HttpContext.Request.Headers["Authorization"];
var httpClient = httpClientFactory.CreateClient();
httpClient.BaseAddress = casterUri;
httpClient.DefaultRequestHeaders.Add("Authorization", authHeader);
var apiClient = new CasterApiClient(httpClient);
return(apiClient);
});
}
public static async Task <Guid?> CreateRunAsync(
ImplementationEntity implementationEntity,
CasterApiClient casterApiClient,
bool isDestroy,
CancellationToken ct)
{
var runCommand = new CreateRunCommand()
{
WorkspaceId = implementationEntity.WorkspaceId,
IsDestroy = isDestroy
};
try
{
var casterRun = await casterApiClient.CreateRunAsync(runCommand, ct);
return(casterRun.Id);
}
catch (Exception ex)
{
return(null);
}
}
private async void ProcessTheImplementation(Object implementationEntityAsObject)
{
var ct = new CancellationToken();
var implementationEntity = implementationEntityAsObject == null ? (ImplementationEntity)null : (ImplementationEntity)implementationEntityAsObject;
_logger.LogInformation($"Processing Implementation {implementationEntity.Id} for status '{implementationEntity.Status}'.");
try
{
using (var scope = _scopeFactory.CreateScope())
{
using (var alloyContext = scope.ServiceProvider.GetRequiredService <AlloyContext>())
{
var retryCount = 0;
var resourceCount = int.MaxValue;
var resourceRetryCount = 0;
// get the alloy context entities required
implementationEntity = alloyContext.Implementations.First(x => x.Id == implementationEntity.Id);
var definitionEntity = alloyContext.Definitions.First(x => x.Id == implementationEntity.DefinitionId);
// get the auth token
var tokenResponse = await ApiClientsExtensions.GetToken(scope);
CasterApiClient casterApiClient = null;
S3PlayerApiClient playerApiClient = null;
SteamfitterApiClient steamfitterApiClient = null;
// LOOP until this thread's process is complete
while (implementationEntity.Status == ImplementationStatus.Creating ||
implementationEntity.Status == ImplementationStatus.Planning ||
implementationEntity.Status == ImplementationStatus.Applying ||
implementationEntity.Status == ImplementationStatus.Ending)
{
// the updateTheEntity flag is used to indicate if the implementation entity state should be updated at the end of this loop
var updateTheEntity = false;
// each time through the loop, one state (case) is handled based on Status and InternalStatus. This allows for retries of a failed state.
switch (implementationEntity.Status)
{
// the "Creating" status means we are creating the initial player exercise, steamfitter session and caster workspace
case ImplementationStatus.Creating:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.LaunchQueued:
case InternalImplementationStatus.CreatingExercise:
{
if (definitionEntity.ExerciseId == null)
{
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
updateTheEntity = true;
}
else
{
try
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
var exerciseId = await PlayerApiExtensions.CreatePlayerExerciseAsync(playerApiClient, implementationEntity, (Guid)definitionEntity.ExerciseId, ct);
if (exerciseId != null)
{
implementationEntity.ExerciseId = exerciseId;
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
catch (Exception ex)
{
_logger.LogError($"Error creating the player exercise for Implementation {implementationEntity.Id}.", ex);
retryCount++;
}
}
break;
}
case InternalImplementationStatus.CreatingSession:
{
if (definitionEntity.ScenarioId == null)
{
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
updateTheEntity = true;
}
else
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
var session = await SteamfitterApiExtensions.CreateSteamfitterSessionAsync(steamfitterApiClient, implementationEntity, (Guid)definitionEntity.ScenarioId, ct);
if (session != null)
{
implementationEntity.SessionId = session.Id;
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
break;
}
case InternalImplementationStatus.CreatingWorkspace:
{
if (definitionEntity.DirectoryId == null)
{
// There is no Caster directory, so start the session
var launchDate = DateTime.UtcNow;
implementationEntity.Name = definitionEntity.Name;
implementationEntity.Description = definitionEntity.Description;
implementationEntity.LaunchDate = launchDate;
implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
implementationEntity.Status = ImplementationStatus.Applying;
implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
updateTheEntity = true;
}
else
{
var varsFileContent = "";
if (implementationEntity.ExerciseId != null)
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
varsFileContent = await CasterApiExtensions.GetCasterVarsFileContentAsync(implementationEntity, playerApiClient, ct);
}
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var workspaceId = await CasterApiExtensions.CreateCasterWorkspaceAsync(casterApiClient, implementationEntity, (Guid)definitionEntity.DirectoryId, varsFileContent, ct);
if (workspaceId != null)
{
implementationEntity.WorkspaceId = workspaceId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningLaunch;
implementationEntity.Status = ImplementationStatus.Planning;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Planning" state means that caster is planning a run
case ImplementationStatus.Planning:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.PlanningLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, false, ct);
if (runId != null)
{
implementationEntity.RunId = runId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlannedLaunch;
updateTheEntity = true;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.PlannedLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingLaunch;
implementationEntity.Status = ImplementationStatus.Applying;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Applying" state means caster is applying a run (deploying VM's, etc.)
case ImplementationStatus.Applying:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.ApplyingLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.AppliedLaunch;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.AppliedLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDeployMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.StartingSession:
{
// start the steamfitter session, if there is one
if (implementationEntity.SessionId != null)
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
updateTheEntity = await SteamfitterApiExtensions.StartSteamfitterSessionAsync(steamfitterApiClient, (Guid)implementationEntity.SessionId, ct);
}
else
{
updateTheEntity = true;
}
// moving on means that Launch is now complete
if (updateTheEntity)
{
var launchDate = DateTime.UtcNow;
implementationEntity.Name = definitionEntity.Name;
implementationEntity.Description = definitionEntity.Description;
implementationEntity.LaunchDate = launchDate;
implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
implementationEntity.Status = ImplementationStatus.Active;
implementationEntity.InternalStatus = InternalImplementationStatus.Launched;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Ending" state means all entities are being torn down
case ImplementationStatus.Ending:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.EndQueued:
case InternalImplementationStatus.DeletingExercise:
{
if (implementationEntity.ExerciseId != null)
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
updateTheEntity = await PlayerApiExtensions.DeletePlayerExerciseAsync(_clientOptions.CurrentValue.urls.playerApi, implementationEntity.ExerciseId, playerApiClient, ct);
}
else
{
updateTheEntity = true;
}
if (updateTheEntity)
{
implementationEntity.ExerciseId = null;
implementationEntity.InternalStatus = InternalImplementationStatus.DeletingSession;
}
break;
}
case InternalImplementationStatus.DeletingSession:
{
if (implementationEntity.SessionId != null)
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
updateTheEntity = await SteamfitterApiExtensions.EndSteamfitterSessionAsync(_clientOptions.CurrentValue.urls.steamfitterApi, implementationEntity.SessionId, steamfitterApiClient, ct);
}
else
{
updateTheEntity = true;
}
if (updateTheEntity)
{
implementationEntity.SessionId = null;
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.PlanningDestroy:
{
if (implementationEntity.WorkspaceId != null)
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, true, ct);
if (runId != null)
{
implementationEntity.RunId = runId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlannedDestroy;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
else
{
implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
implementationEntity.Status = ImplementationStatus.Ended;
updateTheEntity = true;
}
break;
}
case InternalImplementationStatus.PlannedDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.ApplyingDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.AppliedDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.AppliedDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDestroyMaxWaitMinutes, ct);
// all conditions in this case require an implementation entity update
updateTheEntity = true;
// make sure that the run successfully deleted the resources
var count = (await casterApiClient.GetResourcesByWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct)).Count();
implementationEntity.RunId = null;
if (count == 0)
{
// resources deleted, so continue to delete the workspace
implementationEntity.InternalStatus = InternalImplementationStatus.DeletingWorkspace;
}
else
{
if (count < resourceCount)
{
// still some resources, but making progress, try the whole process again
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
resourceRetryCount = 0;
}
else
{
// still some resources and not making progress. Check max retries.
if (resourceRetryCount < _clientOptions.CurrentValue.ApiClientFailureMaxRetries)
{
// try the whole process again after a wait
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
resourceRetryCount++;
Thread.Sleep(TimeSpan.FromMinutes(_clientOptions.CurrentValue.CasterDestroyRetryDelayMinutes));
}
else
{
// the caster workspace resources could not be destroyed
implementationEntity.InternalStatus = InternalImplementationStatus.FailedDestroy;
implementationEntity.Status = ImplementationStatus.Failed;
}
}
}
break;
}
case InternalImplementationStatus.DeletingWorkspace:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.DeleteCasterWorkspaceAsync(implementationEntity, casterApiClient, tokenResponse, ct);
if (updateTheEntity)
{
implementationEntity.WorkspaceId = null;
implementationEntity.Status = ImplementationStatus.Ended;
implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
}
// check for exceeding the max number of retries
if (!updateTheEntity)
{
if (retryCount >= _clientOptions.CurrentValue.ApiClientFailureMaxRetries && _clientOptions.CurrentValue.ApiClientFailureMaxRetries > 0)
{
_logger.LogError($"Retry count exceeded for Implementation {implementationEntity.Id}, with status of {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
}
else
{
Thread.Sleep(TimeSpan.FromSeconds(_clientOptions.CurrentValue.ApiClientRetryIntervalSeconds));
}
}
// update the entity in the context, if we are moving on
if (updateTheEntity)
{
retryCount = 0;
implementationEntity.StatusDate = DateTime.UtcNow;
await alloyContext.SaveChangesAsync(ct);
}
}
}
}
}
catch (Exception ex)
{
_logger.LogError($"Error processing implementation {implementationEntity.Id}", ex);
}
}
