public async Task <Event> EnlistAsync(string code, CancellationToken ct)
{
// user may not have access to the player api, so we get the resource owner token
var token = await ApiClientsExtensions.GetToken(_serviceProvider);
var playerApiClient = PlayerApiExtensions.GetPlayerApiClient(_httpClientFactory, _clientOptions.urls.playerApi, token);
var steamfitterApiClient = SteamfitterApiExtensions.GetSteamfitterApiClient(_httpClientFactory, _clientOptions.urls.steamfitterApi, token);
var alloyEvent = await GetEventByShareCodeAsync(code, ct);
if (alloyEvent.Status == EventStatus.Active || alloyEvent.Status == EventStatus.Paused)
{
if (alloyEvent != null)
{
if (alloyEvent.ViewId.HasValue)
{
await PlayerApiExtensions.AddUserToViewTeamAsync(playerApiClient, alloyEvent.ViewId.Value, _user.GetId(), ct);
}
if (alloyEvent.ScenarioId.HasValue)
{
await steamfitterApiClient.AddUsersToScenarioAsync(alloyEvent.ScenarioId.Value, new List <Guid> {
_user.GetId()
}, ct);
}
try
{
var entity = await _context.EventUsers.Where(e => e.UserId == _user.GetId() && e.EventId == alloyEvent.Id).FirstOrDefaultAsync();
if (entity == null)
{
var eventUser = new EventUserEntity
{
EventId = alloyEvent.Id,
UserId = _user.GetId(),
CreatedBy = _user.GetId()
};
_context.EventUsers.Add(eventUser);
await _context.SaveChangesAsync();
}
return(alloyEvent);
}
catch (Exception)
{
throw new InviteException("Invite Failed, Accepted Already");
}
}
}
throw new InviteException($"Invite Failed, Event Status: {Enum.GetName(typeof(EventStatus), alloyEvent.Status)}");
}
public async Task <Event> RedeployAsync(Guid eventId, CancellationToken ct)
{
try
{
var eventEntity = await GetTheEventAsync(eventId, true, false, ct);
if (eventEntity.Status != EventStatus.Active)
{
var msg = $"Only an Active Event can be redeployed";
_logger.LogError(msg);
throw new Exception(msg);
}
var tokenResponse = await ApiClientsExtensions.RequestTokenAsync(_resourceOwnerAuthorizationOptions, _httpClientFactory.CreateClient());
var casterApiClient = CasterApiExtensions.GetCasterApiClient(_httpClientFactory, _clientOptions.urls.casterApi, tokenResponse);
var result = await casterApiClient.TaintResourcesAsync(
eventEntity.WorkspaceId.Value,
new Caster.Api.Client.TaintResourcesCommand {
SelectAll = true
},
ct);
if (result.Resources.Any(r => r.Tainted == false))
{
var msg = $"Taint failed";
_logger.LogError(msg);
throw new Exception(msg);
}
eventEntity.Status = EventStatus.Planning;
eventEntity.InternalStatus = InternalEventStatus.PlanningRedeploy;
await _context.SaveChangesAsync(ct);
// add the event to the event queue for AlloyBackgrounsService to process the caster destroy.
_alloyEventQueue.Add(eventEntity);
}
catch (Exception ex)
{
_logger.LogError($"Error ending Event {eventId}.", ex);
throw;
}
return(await GetAsync(eventId, ct));
}
private async STT.Task <List <ResultEntity> > CreateResultsAsync(TaskEntity taskToExecute, ScenarioEntity scenarioEntity, Guid userId, CancellationToken ct)
{
var resultEntities = new List <ResultEntity>();
Guid?viewId = null;
if (taskToExecute.VmMask.Count() == 0)
{
// this task has no VM's associated. Create one result entity. Used to send a command to an API, etc.
var resultEntity = NewResultEntity(taskToExecute, userId);
resultEntities.Add(resultEntity);
}
else
{
// A scenario is assigned to a specific view
viewId = scenarioEntity.ViewId;
// at this point, the VmMask could contain an actual mask, or a comma separated list of VM ID's
var vmMaskList = taskToExecute.VmMask.Split(",").ToList();
var vmIdList = new List <Guid>();
var vmList = new List <Player.Vm.Api.Models.Vm>();
// create the ID list, if the mask is a list of Guid's. If not Guid's, vmIdList will end up empty.
foreach (var mask in vmMaskList)
{
Guid vmId;
if (Guid.TryParse(mask, out vmId))
{
vmIdList.Add(vmId);
}
}
// determine if VM's should match by Name or ID
var matchName = vmIdList.Count() == 0;
// create a VmList from the view ID and the VmMask matching criteria
try
{
using (var scope = _scopeFactory.CreateScope())
{
PlayerVmApiClient vmApiClient = null;
var tokenResponse = await ApiClientsExtensions.GetToken(scope);
vmApiClient = RefreshClient(vmApiClient, tokenResponse, ct);
var viewVms = await VmApiExtensions.GetViewVmsAsync(vmApiClient, (Guid)viewId, ct);
foreach (var vm in viewVms)
{
if ((!matchName && vmIdList.Contains((Guid)vm.Id)) || (matchName && vm.Name.ToLower().Contains(taskToExecute.VmMask.ToLower())))
{
vmList.Add(vm);
}
}
}
}
catch (System.Exception)
{
_logger.LogDebug($"CreateResultsAsync - No VM's found in view {viewId}");
}
// make sure we are only matching a SINGLE view
if (vmList.Count() > 0)
{
// create a result for each matched VM
foreach (var vm in vmList)
{
var resultEntity = NewResultEntity(taskToExecute, userId);
resultEntity.VmId = vm.Id;
resultEntity.VmName = vm.Name;
resultEntities.Add(resultEntity);
}
}
else
{
// Houston, we've had a problem. Create a single result to show the error
var resultEntity = NewResultEntity(taskToExecute, userId);
if (viewId != null)
{
// The problem is that NO VM's matched
resultEntity.ActualOutput = $"No matched VMs! VM Mask: {taskToExecute.VmMask}.";
}
else if (scenarioEntity != null)
{
// The problem is that this task did not have a scenario or a viewId
resultEntity.ActualOutput = $"There was no view associated with this task's scenario! {taskToExecute.Name} ({taskToExecute.Id})";
}
else
{
// The problem is that this task did not have a scenario or a viewId
resultEntity.ActualOutput = $"There was no scenario associated with this task! {taskToExecute.Name} ({taskToExecute.Id})";
}
_logger.LogError($"CreateResultsAsync - {resultEntity.ActualOutput}");
resultEntity.Status = Data.TaskStatus.error;
resultEntity.StatusDate = DateTime.UtcNow;
resultEntities.Add(resultEntity);
}
}
return(resultEntities);
}
private async void ProcessTheImplementation(Object implementationEntityAsObject)
{
var ct = new CancellationToken();
var implementationEntity = implementationEntityAsObject == null ? (ImplementationEntity)null : (ImplementationEntity)implementationEntityAsObject;
_logger.LogInformation($"Processing Implementation {implementationEntity.Id} for status '{implementationEntity.Status}'.");
try
{
using (var scope = _scopeFactory.CreateScope())
{
using (var alloyContext = scope.ServiceProvider.GetRequiredService <AlloyContext>())
{
var retryCount = 0;
var resourceCount = int.MaxValue;
var resourceRetryCount = 0;
// get the alloy context entities required
implementationEntity = alloyContext.Implementations.First(x => x.Id == implementationEntity.Id);
var definitionEntity = alloyContext.Definitions.First(x => x.Id == implementationEntity.DefinitionId);
// get the auth token
var tokenResponse = await ApiClientsExtensions.GetToken(scope);
CasterApiClient casterApiClient = null;
S3PlayerApiClient playerApiClient = null;
SteamfitterApiClient steamfitterApiClient = null;
// LOOP until this thread's process is complete
while (implementationEntity.Status == ImplementationStatus.Creating ||
implementationEntity.Status == ImplementationStatus.Planning ||
implementationEntity.Status == ImplementationStatus.Applying ||
implementationEntity.Status == ImplementationStatus.Ending)
{
// the updateTheEntity flag is used to indicate if the implementation entity state should be updated at the end of this loop
var updateTheEntity = false;
// each time through the loop, one state (case) is handled based on Status and InternalStatus. This allows for retries of a failed state.
switch (implementationEntity.Status)
{
// the "Creating" status means we are creating the initial player exercise, steamfitter session and caster workspace
case ImplementationStatus.Creating:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.LaunchQueued:
case InternalImplementationStatus.CreatingExercise:
{
if (definitionEntity.ExerciseId == null)
{
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
updateTheEntity = true;
}
else
{
try
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
var exerciseId = await PlayerApiExtensions.CreatePlayerExerciseAsync(playerApiClient, implementationEntity, (Guid)definitionEntity.ExerciseId, ct);
if (exerciseId != null)
{
implementationEntity.ExerciseId = exerciseId;
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
catch (Exception ex)
{
_logger.LogError($"Error creating the player exercise for Implementation {implementationEntity.Id}.", ex);
retryCount++;
}
}
break;
}
case InternalImplementationStatus.CreatingSession:
{
if (definitionEntity.ScenarioId == null)
{
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
updateTheEntity = true;
}
else
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
var session = await SteamfitterApiExtensions.CreateSteamfitterSessionAsync(steamfitterApiClient, implementationEntity, (Guid)definitionEntity.ScenarioId, ct);
if (session != null)
{
implementationEntity.SessionId = session.Id;
implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
break;
}
case InternalImplementationStatus.CreatingWorkspace:
{
if (definitionEntity.DirectoryId == null)
{
// There is no Caster directory, so start the session
var launchDate = DateTime.UtcNow;
implementationEntity.Name = definitionEntity.Name;
implementationEntity.Description = definitionEntity.Description;
implementationEntity.LaunchDate = launchDate;
implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
implementationEntity.Status = ImplementationStatus.Applying;
implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
updateTheEntity = true;
}
else
{
var varsFileContent = "";
if (implementationEntity.ExerciseId != null)
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
varsFileContent = await CasterApiExtensions.GetCasterVarsFileContentAsync(implementationEntity, playerApiClient, ct);
}
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var workspaceId = await CasterApiExtensions.CreateCasterWorkspaceAsync(casterApiClient, implementationEntity, (Guid)definitionEntity.DirectoryId, varsFileContent, ct);
if (workspaceId != null)
{
implementationEntity.WorkspaceId = workspaceId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningLaunch;
implementationEntity.Status = ImplementationStatus.Planning;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Planning" state means that caster is planning a run
case ImplementationStatus.Planning:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.PlanningLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, false, ct);
if (runId != null)
{
implementationEntity.RunId = runId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlannedLaunch;
updateTheEntity = true;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.PlannedLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingLaunch;
implementationEntity.Status = ImplementationStatus.Applying;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Applying" state means caster is applying a run (deploying VM's, etc.)
case ImplementationStatus.Applying:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.ApplyingLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.AppliedLaunch;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.AppliedLaunch:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDeployMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.StartingSession:
{
// start the steamfitter session, if there is one
if (implementationEntity.SessionId != null)
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
updateTheEntity = await SteamfitterApiExtensions.StartSteamfitterSessionAsync(steamfitterApiClient, (Guid)implementationEntity.SessionId, ct);
}
else
{
updateTheEntity = true;
}
// moving on means that Launch is now complete
if (updateTheEntity)
{
var launchDate = DateTime.UtcNow;
implementationEntity.Name = definitionEntity.Name;
implementationEntity.Description = definitionEntity.Description;
implementationEntity.LaunchDate = launchDate;
implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
implementationEntity.Status = ImplementationStatus.Active;
implementationEntity.InternalStatus = InternalImplementationStatus.Launched;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
// the "Ending" state means all entities are being torn down
case ImplementationStatus.Ending:
{
switch (implementationEntity.InternalStatus)
{
case InternalImplementationStatus.EndQueued:
case InternalImplementationStatus.DeletingExercise:
{
if (implementationEntity.ExerciseId != null)
{
playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
updateTheEntity = await PlayerApiExtensions.DeletePlayerExerciseAsync(_clientOptions.CurrentValue.urls.playerApi, implementationEntity.ExerciseId, playerApiClient, ct);
}
else
{
updateTheEntity = true;
}
if (updateTheEntity)
{
implementationEntity.ExerciseId = null;
implementationEntity.InternalStatus = InternalImplementationStatus.DeletingSession;
}
break;
}
case InternalImplementationStatus.DeletingSession:
{
if (implementationEntity.SessionId != null)
{
steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
updateTheEntity = await SteamfitterApiExtensions.EndSteamfitterSessionAsync(_clientOptions.CurrentValue.urls.steamfitterApi, implementationEntity.SessionId, steamfitterApiClient, ct);
}
else
{
updateTheEntity = true;
}
if (updateTheEntity)
{
implementationEntity.SessionId = null;
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.PlanningDestroy:
{
if (implementationEntity.WorkspaceId != null)
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, true, ct);
if (runId != null)
{
implementationEntity.RunId = runId;
implementationEntity.InternalStatus = InternalImplementationStatus.PlannedDestroy;
updateTheEntity = true;
}
else
{
retryCount++;
}
}
else
{
implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
implementationEntity.Status = ImplementationStatus.Ended;
updateTheEntity = true;
}
break;
}
case InternalImplementationStatus.PlannedDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.ApplyingDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);
if (updateTheEntity)
{
implementationEntity.InternalStatus = InternalImplementationStatus.AppliedDestroy;
}
else
{
retryCount++;
}
break;
}
case InternalImplementationStatus.AppliedDestroy:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDestroyMaxWaitMinutes, ct);
// all conditions in this case require an implementation entity update
updateTheEntity = true;
// make sure that the run successfully deleted the resources
var count = (await casterApiClient.GetResourcesByWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct)).Count();
implementationEntity.RunId = null;
if (count == 0)
{
// resources deleted, so continue to delete the workspace
implementationEntity.InternalStatus = InternalImplementationStatus.DeletingWorkspace;
}
else
{
if (count < resourceCount)
{
// still some resources, but making progress, try the whole process again
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
resourceRetryCount = 0;
}
else
{
// still some resources and not making progress. Check max retries.
if (resourceRetryCount < _clientOptions.CurrentValue.ApiClientFailureMaxRetries)
{
// try the whole process again after a wait
implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
resourceRetryCount++;
Thread.Sleep(TimeSpan.FromMinutes(_clientOptions.CurrentValue.CasterDestroyRetryDelayMinutes));
}
else
{
// the caster workspace resources could not be destroyed
implementationEntity.InternalStatus = InternalImplementationStatus.FailedDestroy;
implementationEntity.Status = ImplementationStatus.Failed;
}
}
}
break;
}
case InternalImplementationStatus.DeletingWorkspace:
{
casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
updateTheEntity = await CasterApiExtensions.DeleteCasterWorkspaceAsync(implementationEntity, casterApiClient, tokenResponse, ct);
if (updateTheEntity)
{
implementationEntity.WorkspaceId = null;
implementationEntity.Status = ImplementationStatus.Ended;
implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
}
else
{
retryCount++;
}
break;
}
default:
{
_logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
break;
}
}
break;
}
}
// check for exceeding the max number of retries
if (!updateTheEntity)
{
if (retryCount >= _clientOptions.CurrentValue.ApiClientFailureMaxRetries && _clientOptions.CurrentValue.ApiClientFailureMaxRetries > 0)
{
_logger.LogError($"Retry count exceeded for Implementation {implementationEntity.Id}, with status of {implementationEntity.Status} - {implementationEntity.InternalStatus}");
implementationEntity.Status = ImplementationStatus.Failed;
updateTheEntity = true;
}
else
{
Thread.Sleep(TimeSpan.FromSeconds(_clientOptions.CurrentValue.ApiClientRetryIntervalSeconds));
}
}
// update the entity in the context, if we are moving on
if (updateTheEntity)
{
retryCount = 0;
implementationEntity.StatusDate = DateTime.UtcNow;
await alloyContext.SaveChangesAsync(ct);
}
}
}
}
}
catch (Exception ex)
{
_logger.LogError($"Error processing implementation {implementationEntity.Id}", ex);
}
}
private async STT.Task <string> HttpTaskTask(TaskEntity taskToExecute)
{
HttpResponseMessage response;
using (var scope = _scopeFactory.CreateScope())
{
// TODO: re-use tokens
var tokenResponse = await ApiClientsExtensions.GetToken(scope);
var actionParameters = JsonSerializer.Deserialize <HttpInputString>(taskToExecute.InputString);
var url = actionParameters.Url;
var client = ApiClientsExtensions.GetHttpClient(_httpClientFactory, url, tokenResponse);
switch (taskToExecute.Action)
{
case TaskAction.http_get:
{
response = await client.GetAsync(url);
break;
}
case TaskAction.http_post:
{
StringContent content = new StringContent(actionParameters.Body, Encoding.UTF8, "application/json");
client.DefaultRequestHeaders.Add("Accept", "application/json");
response = await client.PostAsync(url, content);
break;
}
case TaskAction.http_put:
{
StringContent content = new StringContent(actionParameters.Body, Encoding.UTF8, "application/json");
client.DefaultRequestHeaders.Add("Accept", "application/json");
response = await client.PutAsync(url, content);
break;
}
case TaskAction.http_delete:
{
response = await client.DeleteAsync(url);
break;
}
default:
{
return($"Action '{taskToExecute.Action.ToString()}' is not a valid action for http tasks");
}
}
if (response.IsSuccessStatusCode)
{
// TODO: possibly return the response code, as well
var responseString = await response.Content.ReadAsStringAsync();
return(responseString);
}
else
{
return($"'{taskToExecute.Action.ToString()}' returned a status code of {response.StatusCode}.");
}
}
}
