Example #1
0
        public async Task <Event> EnlistAsync(string code, CancellationToken ct)
        {
            // user may not have access to the player api, so we get the resource owner token
            var token = await ApiClientsExtensions.GetToken(_serviceProvider);

            var playerApiClient      = PlayerApiExtensions.GetPlayerApiClient(_httpClientFactory, _clientOptions.urls.playerApi, token);
            var steamfitterApiClient = SteamfitterApiExtensions.GetSteamfitterApiClient(_httpClientFactory, _clientOptions.urls.steamfitterApi, token);

            var alloyEvent = await GetEventByShareCodeAsync(code, ct);

            if (alloyEvent.Status == EventStatus.Active || alloyEvent.Status == EventStatus.Paused)
            {
                if (alloyEvent != null)
                {
                    if (alloyEvent.ViewId.HasValue)
                    {
                        await PlayerApiExtensions.AddUserToViewTeamAsync(playerApiClient, alloyEvent.ViewId.Value, _user.GetId(), ct);
                    }

                    if (alloyEvent.ScenarioId.HasValue)
                    {
                        await steamfitterApiClient.AddUsersToScenarioAsync(alloyEvent.ScenarioId.Value, new List <Guid> {
                            _user.GetId()
                        }, ct);
                    }

                    try
                    {
                        var entity = await _context.EventUsers.Where(e => e.UserId == _user.GetId() && e.EventId == alloyEvent.Id).FirstOrDefaultAsync();

                        if (entity == null)
                        {
                            var eventUser = new EventUserEntity
                            {
                                EventId   = alloyEvent.Id,
                                UserId    = _user.GetId(),
                                CreatedBy = _user.GetId()
                            };

                            _context.EventUsers.Add(eventUser);
                            await _context.SaveChangesAsync();
                        }

                        return(alloyEvent);
                    }
                    catch (Exception)
                    {
                        throw new InviteException("Invite Failed, Accepted Already");
                    }
                }
            }
            throw new InviteException($"Invite Failed, Event Status: {Enum.GetName(typeof(EventStatus), alloyEvent.Status)}");
        }
Example #2
0
        public async Task <Event> RedeployAsync(Guid eventId, CancellationToken ct)
        {
            try
            {
                var eventEntity = await GetTheEventAsync(eventId, true, false, ct);

                if (eventEntity.Status != EventStatus.Active)
                {
                    var msg = $"Only an Active Event can be redeployed";
                    _logger.LogError(msg);
                    throw new Exception(msg);
                }

                var tokenResponse = await ApiClientsExtensions.RequestTokenAsync(_resourceOwnerAuthorizationOptions, _httpClientFactory.CreateClient());

                var casterApiClient = CasterApiExtensions.GetCasterApiClient(_httpClientFactory, _clientOptions.urls.casterApi, tokenResponse);

                var result = await casterApiClient.TaintResourcesAsync(
                    eventEntity.WorkspaceId.Value,
                    new Caster.Api.Client.TaintResourcesCommand {
                    SelectAll = true
                },
                    ct);

                if (result.Resources.Any(r => r.Tainted == false))
                {
                    var msg = $"Taint failed";
                    _logger.LogError(msg);
                    throw new Exception(msg);
                }

                eventEntity.Status         = EventStatus.Planning;
                eventEntity.InternalStatus = InternalEventStatus.PlanningRedeploy;
                await _context.SaveChangesAsync(ct);

                // add the event to the event queue for AlloyBackgrounsService to process the caster destroy.
                _alloyEventQueue.Add(eventEntity);
            }
            catch (Exception ex)
            {
                _logger.LogError($"Error ending Event {eventId}.", ex);
                throw;
            }

            return(await GetAsync(eventId, ct));
        }
Example #3
0
        private async STT.Task <List <ResultEntity> > CreateResultsAsync(TaskEntity taskToExecute, ScenarioEntity scenarioEntity, Guid userId, CancellationToken ct)
        {
            var  resultEntities = new List <ResultEntity>();
            Guid?viewId         = null;

            if (taskToExecute.VmMask.Count() == 0)
            {
                // this task has no VM's associated. Create one result entity.  Used to send a command to an API, etc.
                var resultEntity = NewResultEntity(taskToExecute, userId);
                resultEntities.Add(resultEntity);
            }
            else
            {
                // A scenario is assigned to a specific view
                viewId = scenarioEntity.ViewId;
                // at this point, the VmMask could contain an actual mask, or a comma separated list of VM ID's
                var vmMaskList = taskToExecute.VmMask.Split(",").ToList();
                var vmIdList   = new List <Guid>();
                var vmList     = new List <Player.Vm.Api.Models.Vm>();
                // create the ID list, if the mask is a list of Guid's. If not Guid's, vmIdList will end up empty.
                foreach (var mask in vmMaskList)
                {
                    Guid vmId;
                    if (Guid.TryParse(mask, out vmId))
                    {
                        vmIdList.Add(vmId);
                    }
                }
                // determine if VM's should match by Name or ID
                var matchName = vmIdList.Count() == 0;
                // create a VmList from the view ID and the VmMask matching criteria
                try
                {
                    using (var scope = _scopeFactory.CreateScope())
                    {
                        PlayerVmApiClient vmApiClient = null;
                        var tokenResponse             = await ApiClientsExtensions.GetToken(scope);

                        vmApiClient = RefreshClient(vmApiClient, tokenResponse, ct);
                        var viewVms = await VmApiExtensions.GetViewVmsAsync(vmApiClient, (Guid)viewId, ct);

                        foreach (var vm in viewVms)
                        {
                            if ((!matchName && vmIdList.Contains((Guid)vm.Id)) || (matchName && vm.Name.ToLower().Contains(taskToExecute.VmMask.ToLower())))
                            {
                                vmList.Add(vm);
                            }
                        }
                    }
                }
                catch (System.Exception)
                {
                    _logger.LogDebug($"CreateResultsAsync - No VM's found in view {viewId}");
                }
                // make sure we are only matching a SINGLE view
                if (vmList.Count() > 0)
                {
                    // create a result for each matched VM
                    foreach (var vm in vmList)
                    {
                        var resultEntity = NewResultEntity(taskToExecute, userId);
                        resultEntity.VmId   = vm.Id;
                        resultEntity.VmName = vm.Name;
                        resultEntities.Add(resultEntity);
                    }
                }
                else
                {
                    // Houston, we've had a problem.  Create a single result to show the error
                    var resultEntity = NewResultEntity(taskToExecute, userId);
                    if (viewId != null)
                    {
                        // The problem is that NO VM's matched
                        resultEntity.ActualOutput = $"No matched VMs!  VM Mask: {taskToExecute.VmMask}.";
                    }
                    else if (scenarioEntity != null)
                    {
                        // The problem is that this task did not have a scenario or a viewId
                        resultEntity.ActualOutput = $"There was no view associated with this task's scenario!  {taskToExecute.Name} ({taskToExecute.Id})";
                    }
                    else
                    {
                        // The problem is that this task did not have a scenario or a viewId
                        resultEntity.ActualOutput = $"There was no scenario associated with this task!  {taskToExecute.Name} ({taskToExecute.Id})";
                    }
                    _logger.LogError($"CreateResultsAsync - {resultEntity.ActualOutput}");
                    resultEntity.Status     = Data.TaskStatus.error;
                    resultEntity.StatusDate = DateTime.UtcNow;
                    resultEntities.Add(resultEntity);
                }
            }
            return(resultEntities);
        }
Example #4
0
        private async void ProcessTheImplementation(Object implementationEntityAsObject)
        {
            var ct = new CancellationToken();
            var implementationEntity = implementationEntityAsObject == null ? (ImplementationEntity)null : (ImplementationEntity)implementationEntityAsObject;

            _logger.LogInformation($"Processing Implementation {implementationEntity.Id} for status '{implementationEntity.Status}'.");

            try
            {
                using (var scope = _scopeFactory.CreateScope())
                {
                    using (var alloyContext = scope.ServiceProvider.GetRequiredService <AlloyContext>())
                    {
                        var retryCount         = 0;
                        var resourceCount      = int.MaxValue;
                        var resourceRetryCount = 0;
                        // get the alloy context entities required
                        implementationEntity = alloyContext.Implementations.First(x => x.Id == implementationEntity.Id);
                        var definitionEntity = alloyContext.Definitions.First(x => x.Id == implementationEntity.DefinitionId);
                        // get the auth token
                        var tokenResponse = await ApiClientsExtensions.GetToken(scope);

                        CasterApiClient      casterApiClient      = null;
                        S3PlayerApiClient    playerApiClient      = null;
                        SteamfitterApiClient steamfitterApiClient = null;
                        // LOOP until this thread's process is complete
                        while (implementationEntity.Status == ImplementationStatus.Creating ||
                               implementationEntity.Status == ImplementationStatus.Planning ||
                               implementationEntity.Status == ImplementationStatus.Applying ||
                               implementationEntity.Status == ImplementationStatus.Ending)
                        {
                            // the updateTheEntity flag is used to indicate if the implementation entity state should be updated at the end of this loop
                            var updateTheEntity = false;
                            // each time through the loop, one state (case) is handled based on Status and InternalStatus.  This allows for retries of a failed state.
                            switch (implementationEntity.Status)
                            {
                            // the "Creating" status means we are creating the initial player exercise, steamfitter session and caster workspace
                            case ImplementationStatus.Creating:
                            {
                                switch (implementationEntity.InternalStatus)
                                {
                                case InternalImplementationStatus.LaunchQueued:
                                case InternalImplementationStatus.CreatingExercise:
                                {
                                    if (definitionEntity.ExerciseId == null)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
                                        updateTheEntity = true;
                                    }
                                    else
                                    {
                                        try
                                        {
                                            playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
                                            var exerciseId = await PlayerApiExtensions.CreatePlayerExerciseAsync(playerApiClient, implementationEntity, (Guid)definitionEntity.ExerciseId, ct);

                                            if (exerciseId != null)
                                            {
                                                implementationEntity.ExerciseId     = exerciseId;
                                                implementationEntity.InternalStatus = InternalImplementationStatus.CreatingSession;
                                                updateTheEntity = true;
                                            }
                                            else
                                            {
                                                retryCount++;
                                            }
                                        }
                                        catch (Exception ex)
                                        {
                                            _logger.LogError($"Error creating the player exercise for Implementation {implementationEntity.Id}.", ex);
                                            retryCount++;
                                        }
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.CreatingSession:
                                {
                                    if (definitionEntity.ScenarioId == null)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
                                        updateTheEntity = true;
                                    }
                                    else
                                    {
                                        steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
                                        var session = await SteamfitterApiExtensions.CreateSteamfitterSessionAsync(steamfitterApiClient, implementationEntity, (Guid)definitionEntity.ScenarioId, ct);

                                        if (session != null)
                                        {
                                            implementationEntity.SessionId      = session.Id;
                                            implementationEntity.InternalStatus = InternalImplementationStatus.CreatingWorkspace;
                                            updateTheEntity = true;
                                        }
                                        else
                                        {
                                            retryCount++;
                                        }
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.CreatingWorkspace:
                                {
                                    if (definitionEntity.DirectoryId == null)
                                    {
                                        // There is no Caster directory, so start the session
                                        var launchDate = DateTime.UtcNow;
                                        implementationEntity.Name           = definitionEntity.Name;
                                        implementationEntity.Description    = definitionEntity.Description;
                                        implementationEntity.LaunchDate     = launchDate;
                                        implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
                                        implementationEntity.Status         = ImplementationStatus.Applying;
                                        implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
                                        updateTheEntity = true;
                                    }
                                    else
                                    {
                                        var varsFileContent = "";
                                        if (implementationEntity.ExerciseId != null)
                                        {
                                            playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
                                            varsFileContent = await CasterApiExtensions.GetCasterVarsFileContentAsync(implementationEntity, playerApiClient, ct);
                                        }
                                        casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                        var workspaceId = await CasterApiExtensions.CreateCasterWorkspaceAsync(casterApiClient, implementationEntity, (Guid)definitionEntity.DirectoryId, varsFileContent, ct);

                                        if (workspaceId != null)
                                        {
                                            implementationEntity.WorkspaceId    = workspaceId;
                                            implementationEntity.InternalStatus = InternalImplementationStatus.PlanningLaunch;
                                            implementationEntity.Status         = ImplementationStatus.Planning;
                                            updateTheEntity = true;
                                        }
                                        else
                                        {
                                            retryCount++;
                                        }
                                    }
                                    break;
                                }

                                default:
                                {
                                    _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
                                    implementationEntity.Status = ImplementationStatus.Failed;
                                    updateTheEntity             = true;
                                    break;
                                }
                                }
                                break;
                            }

                            // the "Planning" state means that caster is planning a run
                            case ImplementationStatus.Planning:
                            {
                                switch (implementationEntity.InternalStatus)
                                {
                                case InternalImplementationStatus.PlanningLaunch:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, false, ct);

                                    if (runId != null)
                                    {
                                        implementationEntity.RunId          = runId;
                                        implementationEntity.InternalStatus = InternalImplementationStatus.PlannedLaunch;
                                        updateTheEntity = true;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.PlannedLaunch:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);

                                    if (updateTheEntity)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingLaunch;
                                        implementationEntity.Status         = ImplementationStatus.Applying;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                default:
                                {
                                    _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
                                    implementationEntity.Status = ImplementationStatus.Failed;
                                    updateTheEntity             = true;
                                    break;
                                }
                                }
                                break;
                            }

                            // the "Applying" state means caster is applying a run (deploying VM's, etc.)
                            case ImplementationStatus.Applying:
                            {
                                switch (implementationEntity.InternalStatus)
                                {
                                case InternalImplementationStatus.ApplyingLaunch:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);

                                    if (updateTheEntity)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.AppliedLaunch;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.AppliedLaunch:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    updateTheEntity = await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDeployMaxWaitMinutes, ct);

                                    if (updateTheEntity)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.StartingSession;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.StartingSession:
                                {
                                    // start the steamfitter session, if there is one
                                    if (implementationEntity.SessionId != null)
                                    {
                                        steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
                                        updateTheEntity      = await SteamfitterApiExtensions.StartSteamfitterSessionAsync(steamfitterApiClient, (Guid)implementationEntity.SessionId, ct);
                                    }
                                    else
                                    {
                                        updateTheEntity = true;
                                    }
                                    // moving on means that Launch is now complete
                                    if (updateTheEntity)
                                    {
                                        var launchDate = DateTime.UtcNow;
                                        implementationEntity.Name           = definitionEntity.Name;
                                        implementationEntity.Description    = definitionEntity.Description;
                                        implementationEntity.LaunchDate     = launchDate;
                                        implementationEntity.ExpirationDate = launchDate.AddHours(definitionEntity.DurationHours);
                                        implementationEntity.Status         = ImplementationStatus.Active;
                                        implementationEntity.InternalStatus = InternalImplementationStatus.Launched;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                default:
                                {
                                    _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
                                    implementationEntity.Status = ImplementationStatus.Failed;
                                    updateTheEntity             = true;
                                    break;
                                }
                                }
                                break;
                            }

                            // the "Ending" state means all entities are being torn down
                            case ImplementationStatus.Ending:
                            {
                                switch (implementationEntity.InternalStatus)
                                {
                                case InternalImplementationStatus.EndQueued:
                                case InternalImplementationStatus.DeletingExercise:
                                {
                                    if (implementationEntity.ExerciseId != null)
                                    {
                                        playerApiClient = RefreshClient(playerApiClient, tokenResponse, ct);
                                        updateTheEntity = await PlayerApiExtensions.DeletePlayerExerciseAsync(_clientOptions.CurrentValue.urls.playerApi, implementationEntity.ExerciseId, playerApiClient, ct);
                                    }
                                    else
                                    {
                                        updateTheEntity = true;
                                    }
                                    if (updateTheEntity)
                                    {
                                        implementationEntity.ExerciseId     = null;
                                        implementationEntity.InternalStatus = InternalImplementationStatus.DeletingSession;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.DeletingSession:
                                {
                                    if (implementationEntity.SessionId != null)
                                    {
                                        steamfitterApiClient = RefreshClient(steamfitterApiClient, tokenResponse, ct);
                                        updateTheEntity      = await SteamfitterApiExtensions.EndSteamfitterSessionAsync(_clientOptions.CurrentValue.urls.steamfitterApi, implementationEntity.SessionId, steamfitterApiClient, ct);
                                    }
                                    else
                                    {
                                        updateTheEntity = true;
                                    }
                                    if (updateTheEntity)
                                    {
                                        implementationEntity.SessionId      = null;
                                        implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.PlanningDestroy:
                                {
                                    if (implementationEntity.WorkspaceId != null)
                                    {
                                        casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                        var runId = await CasterApiExtensions.CreateRunAsync(implementationEntity, casterApiClient, true, ct);

                                        if (runId != null)
                                        {
                                            implementationEntity.RunId          = runId;
                                            implementationEntity.InternalStatus = InternalImplementationStatus.PlannedDestroy;
                                            updateTheEntity = true;
                                        }
                                        else
                                        {
                                            retryCount++;
                                        }
                                    }
                                    else
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
                                        implementationEntity.Status         = ImplementationStatus.Ended;
                                        updateTheEntity = true;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.PlannedDestroy:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    updateTheEntity = await CasterApiExtensions.WaitForRunToBePlannedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterPlanningMaxWaitMinutes, ct);

                                    if (updateTheEntity)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.ApplyingDestroy;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.ApplyingDestroy:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    updateTheEntity = await CasterApiExtensions.ApplyRunAsync(implementationEntity, casterApiClient, ct);

                                    if (updateTheEntity)
                                    {
                                        implementationEntity.InternalStatus = InternalImplementationStatus.AppliedDestroy;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.AppliedDestroy:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    await CasterApiExtensions.WaitForRunToBeAppliedAsync(implementationEntity, casterApiClient, _clientOptions.CurrentValue.CasterCheckIntervalSeconds, _clientOptions.CurrentValue.CasterDestroyMaxWaitMinutes, ct);

                                    // all conditions in this case require an implementation entity update
                                    updateTheEntity = true;
                                    // make sure that the run successfully deleted the resources
                                    var count = (await casterApiClient.GetResourcesByWorkspaceAsync((Guid)implementationEntity.WorkspaceId, ct)).Count();
                                    implementationEntity.RunId = null;
                                    if (count == 0)
                                    {
                                        // resources deleted, so continue to delete the workspace
                                        implementationEntity.InternalStatus = InternalImplementationStatus.DeletingWorkspace;
                                    }
                                    else
                                    {
                                        if (count < resourceCount)
                                        {
                                            // still some resources, but making progress, try the whole process again
                                            implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
                                            resourceRetryCount = 0;
                                        }
                                        else
                                        {
                                            // still some resources and not making progress. Check max retries.
                                            if (resourceRetryCount < _clientOptions.CurrentValue.ApiClientFailureMaxRetries)
                                            {
                                                // try the whole process again after a wait
                                                implementationEntity.InternalStatus = InternalImplementationStatus.PlanningDestroy;
                                                resourceRetryCount++;
                                                Thread.Sleep(TimeSpan.FromMinutes(_clientOptions.CurrentValue.CasterDestroyRetryDelayMinutes));
                                            }
                                            else
                                            {
                                                // the caster workspace resources could not be destroyed
                                                implementationEntity.InternalStatus = InternalImplementationStatus.FailedDestroy;
                                                implementationEntity.Status         = ImplementationStatus.Failed;
                                            }
                                        }
                                    }
                                    break;
                                }

                                case InternalImplementationStatus.DeletingWorkspace:
                                {
                                    casterApiClient = RefreshClient(casterApiClient, tokenResponse, ct);
                                    updateTheEntity = await CasterApiExtensions.DeleteCasterWorkspaceAsync(implementationEntity, casterApiClient, tokenResponse, ct);

                                    if (updateTheEntity)
                                    {
                                        implementationEntity.WorkspaceId    = null;
                                        implementationEntity.Status         = ImplementationStatus.Ended;
                                        implementationEntity.InternalStatus = InternalImplementationStatus.Ended;
                                    }
                                    else
                                    {
                                        retryCount++;
                                    }
                                    break;
                                }

                                default:
                                {
                                    _logger.LogError($"Invalid status for Implementation {implementationEntity.Id}: {implementationEntity.Status} - {implementationEntity.InternalStatus}");
                                    implementationEntity.Status = ImplementationStatus.Failed;
                                    updateTheEntity             = true;
                                    break;
                                }
                                }
                                break;
                            }
                            }
                            // check for exceeding the max number of retries
                            if (!updateTheEntity)
                            {
                                if (retryCount >= _clientOptions.CurrentValue.ApiClientFailureMaxRetries && _clientOptions.CurrentValue.ApiClientFailureMaxRetries > 0)
                                {
                                    _logger.LogError($"Retry count exceeded for Implementation {implementationEntity.Id}, with status of {implementationEntity.Status} - {implementationEntity.InternalStatus}");
                                    implementationEntity.Status = ImplementationStatus.Failed;
                                    updateTheEntity             = true;
                                }
                                else
                                {
                                    Thread.Sleep(TimeSpan.FromSeconds(_clientOptions.CurrentValue.ApiClientRetryIntervalSeconds));
                                }
                            }
                            // update the entity in the context, if we are moving on
                            if (updateTheEntity)
                            {
                                retryCount = 0;
                                implementationEntity.StatusDate = DateTime.UtcNow;
                                await alloyContext.SaveChangesAsync(ct);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                _logger.LogError($"Error processing implementation {implementationEntity.Id}", ex);
            }
        }
        private async STT.Task <string> HttpTaskTask(TaskEntity taskToExecute)
        {
            HttpResponseMessage response;

            using (var scope = _scopeFactory.CreateScope())
            {
                // TODO: re-use tokens
                var tokenResponse = await ApiClientsExtensions.GetToken(scope);

                var actionParameters = JsonSerializer.Deserialize <HttpInputString>(taskToExecute.InputString);
                var url    = actionParameters.Url;
                var client = ApiClientsExtensions.GetHttpClient(_httpClientFactory, url, tokenResponse);
                switch (taskToExecute.Action)
                {
                case TaskAction.http_get:
                {
                    response = await client.GetAsync(url);

                    break;
                }

                case TaskAction.http_post:
                {
                    StringContent content = new StringContent(actionParameters.Body, Encoding.UTF8, "application/json");
                    client.DefaultRequestHeaders.Add("Accept", "application/json");
                    response = await client.PostAsync(url, content);

                    break;
                }

                case TaskAction.http_put:
                {
                    StringContent content = new StringContent(actionParameters.Body, Encoding.UTF8, "application/json");
                    client.DefaultRequestHeaders.Add("Accept", "application/json");
                    response = await client.PutAsync(url, content);

                    break;
                }

                case TaskAction.http_delete:
                {
                    response = await client.DeleteAsync(url);

                    break;
                }

                default:
                {
                    return($"Action '{taskToExecute.Action.ToString()}' is not a valid action for http tasks");
                }
                }
                if (response.IsSuccessStatusCode)
                {
                    // TODO:  possibly return the response code, as well
                    var responseString = await response.Content.ReadAsStringAsync();

                    return(responseString);
                }
                else
                {
                    return($"'{taskToExecute.Action.ToString()}' returned a status code of {response.StatusCode}.");
                }
            }
        }