public async Task EnsureTerminationsCompleteAsync() { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(new DescribeEnvironmentsRequest { ApplicationName = configurationProvider.LongApplicationName }); var isTerminating = describeEnvironmentsResponse.Environments.Any(env => env.Status == EnvironmentStatus.Terminating); if (!isTerminating) { return; } do { loggerProvider.GetLogger().Debug("Environments are termintating. Waiting..."); await Task.Delay(TIME_BETWEEN_STATUS_CHECKS_IN_MILLISECONDS); describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(new DescribeEnvironmentsRequest { ApplicationName = configurationProvider.LongApplicationName }); isTerminating = describeEnvironmentsResponse.Environments.Any(env => env.Status == EnvironmentStatus.Terminating); } while (isTerminating); } }
public static AmazonElasticBeanstalkClient GetElasticBeanstalkConnection() { if (elasticBeanstalk == null) { elasticBeanstalk = new AmazonElasticBeanstalkClient(AwsKeyProviders.Key, AwsKeyProviders.Secret, RegionEndpoint.EUWest1); } return elasticBeanstalk; }
public async Task<EnvironmentDescription> GetEnvironmentDescription(Environment environment) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { return await GetEnvironmentDescription(environment, ebClient); } }
private async Task UpdateEnvironmentInternalAsync(Environment environment, List<ConfigurationOptionSetting> baseConfigurationOptionSettings) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { loggerProvider.GetLogger().Debug("Existing environment - updating..."); var targetEnvironment = await GetEnvironmentDescription(environment, ebClient); if (targetEnvironment == null) { var msg = "Error state: On environment update, next environment is null"; loggerProvider.GetLogger().Error(msg); throw new ElasticBeanstalkDeployerException(msg); } loggerProvider.GetLogger().Debug("Updating environment."); var optionSettings = baseConfigurationOptionSettings; loggerProvider.GetLogger().Debug($"Updating environment: global option settings count - {optionSettings.Count}"); var optionSettingsApp = GetAppSpecificUpdateConfigurationOptionSettings(); optionSettings = InjectAppSpecificOptionSettings(optionSettings, optionSettingsApp); loggerProvider.GetLogger().Debug($"Updating environment: total option settings count - {optionSettings.Count}"); var updateEnvironmentRequest = new UpdateEnvironmentRequest { VersionLabel = configurationProvider.Version, EnvironmentId = targetEnvironment.EnvironmentId, OptionSettings = optionSettings }; loggerProvider.GetLogger() .Debug("Updating environment: {@updateEnvironmentRequest}", updateEnvironmentRequest); updateEnvironmentRequest.OptionSettings.ForEach(option => loggerProvider.GetLogger().Debug($"Updating environment option setting: namespace={option.Namespace} option name={option.OptionName}")); int tryCount = 0; bool isSuccessful = false; while (!isSuccessful) { try { var updateEnvironmentResponse = await ebClient.UpdateEnvironmentAsync(updateEnvironmentRequest); isSuccessful = true; loggerProvider.GetLogger() .Debug("Updated environment: {@updateEnvironmentResponse}", updateEnvironmentResponse); } catch (Exception ex) { loggerProvider.GetLogger().Debug($"Failed update with exception: {ex.Message}"); if (tryCount >= 5) { throw; } tryCount++; await Task.Delay(TIME_BETWEEN_STATUS_CHECKS_IN_MILLISECONDS); } } await WaitForEnvironmentCompletionAsync(targetEnvironment.EnvironmentId); } }
/// <summary> /// Calls DescribeEnvironmentsAsync with a retry count where if the response returns no environments, it'll retry <see cref="retryCount"/> times with a <see cref="interval"/> second sleep /// </summary> /// <param name="ebClient"></param> /// <param name="environmentId"></param> /// <param name="retryCount">Number of times to retry the function call if environments returned is 0</param> /// <param name="interval">Time in seconds between retry attempts</param> /// <returns></returns> private async Task<DescribeEnvironmentsResponse> DescribeEnvironmentsWithRetryAsync( AmazonElasticBeanstalkClient ebClient, string environmentId, int retryCount = 2, int interval = 5) { var currentAttempt = 0; bool shouldRetry; DescribeEnvironmentsResponse describeEnvironmentsResponse = null; do { currentAttempt++; try { var request = new DescribeEnvironmentsRequest { EnvironmentIds = new List<string> { environmentId } }; describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(request); shouldRetry = false; //check that there is an environment in the response, if not, we are going to assume that the environment is still spinning up and retry // the describeEnvironment request in interval seconds if (!describeEnvironmentsResponse.Environments.Any()) { loggerProvider.GetLogger().Warning($"No environments returned for environmentId. Waiting before retrying request. [EnvironmentId: {environmentId}]"); shouldRetry = true; } } catch (Exception ex) { loggerProvider.GetLogger().Error(ex, $"Error calling DescribeEnvironments. No Environments returned. [EnvironmentId: {environmentId}]"); shouldRetry = true; } //adding delay if we are going to retry if (shouldRetry && currentAttempt <= retryCount) { await Task.Delay(interval * 1000); } } while (shouldRetry && currentAttempt <= retryCount); return describeEnvironmentsResponse; }
private async Task<bool> IsEnvironmentReadyAsync(string environmentId) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(new DescribeEnvironmentsRequest { EnvironmentIds = new List<string> { environmentId } }); if (describeEnvironmentsResponse.Environments.Count == 0) { loggerProvider.GetLogger().Debug("Environment does not yet exist"); return false; } var matchingNonTerminatedEnvironment = describeEnvironmentsResponse.Environments.Single(env => env.Status != EnvironmentStatus.Terminated); loggerProvider.GetLogger().Debug($"Environment Status is {matchingNonTerminatedEnvironment.Status} Environment Health is {matchingNonTerminatedEnvironment.Health}"); //failsafe to keep the environment from spinning once it has gone red if (matchingNonTerminatedEnvironment.Status == EnvironmentStatus.Ready && matchingNonTerminatedEnvironment.Health == EnvironmentHealth.Red) { throw new Exception("Environment failed to start up as expected."); } return matchingNonTerminatedEnvironment.Status == EnvironmentStatus.Ready && matchingNonTerminatedEnvironment.Health == EnvironmentHealth.Green; } }
private async Task<bool> IsNewApplicationAsync(AmazonElasticBeanstalkClient client) { var describeApplicationsResponse = await client.DescribeApplicationsAsync(); return describeApplicationsResponse.Applications.All( description => description.ApplicationName != configurationProvider.LongApplicationName); }
private async Task<EnvironmentDescription> GetCurrentEnvironmentDescriptionAsync() { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(new DescribeEnvironmentsRequest { ApplicationName = configurationProvider.LongApplicationName }); var currentEnvironmentDescription = describeEnvironmentsResponse.Environments .Single(env => env.CNAME.StartsWith(currentCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); return currentEnvironmentDescription; } }
public async Task IisEnsureInitialEnvironmentSetupAsync(List<ConfigurationOptionSetting> optionSettings) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(new DescribeEnvironmentsRequest { IncludeDeleted = false }); loggerProvider.GetLogger().Debug("Environment Repponse: {@response}", describeEnvironmentsResponse); var current = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(currentCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); loggerProvider.GetLogger().Debug("Current: {@current}", current); var next = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(nextCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); loggerProvider.GetLogger().Debug("Next: {@next}", next); List<Task> terminatingTasks = new List<Task>(); if (current != null && current.Health != EnvironmentHealth.Green) { loggerProvider.GetLogger().Debug("Current Environment is not in green state. Deleting environment."); var terminateCurrentEnvTask = ebClient.TerminateEnvironmentAsync(new TerminateEnvironmentRequest { EnvironmentId = current.EnvironmentId, TerminateResources = true }); terminatingTasks.Add(terminateCurrentEnvTask); current = null; } if (next != null && next.Health != EnvironmentHealth.Green) { loggerProvider.GetLogger().Debug("Next Environment is not in green state. Deleting environment."); var terminateNextEnvTask = ebClient.TerminateEnvironmentAsync(new TerminateEnvironmentRequest { EnvironmentId = next.EnvironmentId, TerminateResources = true }); terminatingTasks.Add(terminateNextEnvTask); next = null; } if (terminatingTasks.Count > 0) { await Task.WhenAll(terminatingTasks); await EnsureTerminationsCompleteAsync(); } if (current != null && next != null) { loggerProvider.GetLogger().Debug("Environments are set up properly"); return; } if (current == null && next != null) { var msg = "Error state: no current running, but next is up"; loggerProvider.GetLogger().Error(msg); throw new ElasticBeanstalkDeployerException(msg); } if (current == null) { loggerProvider.GetLogger().Debug("Neither environment is up - create both"); Task.WaitAll(new List<Task> { CreateIisCurrentEnvironmentAsync("A", optionSettings), CreateIisNextEnvironmentAsync("B", configurationProvider.Version, optionSettings) }.ToArray()); } else { var useA = current.EnvironmentName.EndsWith("-B", StringComparison.CurrentCultureIgnoreCase); if (useA) { await CreateIisNextEnvironmentAsync("A", optionSettings); } else { var currentEnvironment = await GetCurrentEnvironmentDescriptionAsync(); await CreateIisNextEnvironmentAsync("B", currentEnvironment.VersionLabel, optionSettings); } } } }
private async Task CreateIisNextEnvironmentAsync(string environmentIdentifier, string version, List<ConfigurationOptionSetting> optionSettings) { loggerProvider.GetLogger() .Debug("New environment - creating \"Next\" {identifier} {version}...", environmentIdentifier, version); using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var optionSettingsApp = GetAppSpecificCreateConfigurationOptionSettings(); loggerProvider.GetLogger().Debug("Parsed configuration option settings: {@optionSettings}", optionSettingsApp); optionSettings = InjectAppSpecificOptionSettings(optionSettings, optionSettingsApp); loggerProvider.GetLogger() .Debug("Create next environment: total option settings count - {optionSettingsCount}", optionSettings.Count); var currentEnvironmentName = $"{configurationProvider.BeanstalkEnvironmentName}-{environmentIdentifier}"; var createEnvironmentRequest = new CreateEnvironmentRequest(configurationProvider.LongApplicationName, currentEnvironmentName) { Tier = new EnvironmentTier { Name = "WebServer", Type = "Standard" }, SolutionStackName = "64bit Windows Server Core 2012 R2 running IIS 8.5", CNAMEPrefix = nextCNamePrefix, VersionLabel = version, OptionSettings = optionSettings }; var createEnvironmentResponse = await ebClient.CreateEnvironmentAsync(createEnvironmentRequest); await EnsureCreateEnvironmentCompleteAsync(createEnvironmentResponse.EnvironmentId); loggerProvider.GetLogger() .Debug("Created \"Next\" environment: {@createEnvironmentResponse}", createEnvironmentResponse); // TODO: capture createEnvironmentResponse.Cname as TestUri - SRO } }
private async Task CreateDockerNextEnvironmentAsync(string environmentIdentifier, string version, List<ConfigurationOptionSetting> optionSettings) { loggerProvider.GetLogger() .Debug("New environment - creating \"Next\" {identifier} {version}...", environmentIdentifier, version); using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var optionSettingsApp = GetAppSpecificCreateConfigurationOptionSettings(); loggerProvider.GetLogger() .Debug("App-specific configuration options settings: {@optionSettingsApp}", optionSettingsApp); optionSettings = InjectAppSpecificOptionSettings(optionSettings, optionSettingsApp); loggerProvider.GetLogger() .Debug("Create next environment: total option settings count - {optionSettingsCount}", optionSettings.Count); var currentEnvironmentName = $"{configurationProvider.BeanstalkEnvironmentName}-{environmentIdentifier}"; var createEnvironmentRequest = GenerateCreateDockerEnvRequest(optionSettings, currentEnvironmentName, nextCNamePrefix); var createEnvironmentResponse = await ebClient.CreateEnvironmentAsync(createEnvironmentRequest); await EnsureCreateEnvironmentCompleteAsync(createEnvironmentResponse.EnvironmentId); loggerProvider.GetLogger() .Debug("Created \"Next\" environment: {@createEnvironmentResponse}", createEnvironmentResponse); } }
private async Task WaitForCnameSwapToCompleteAsync(string nextEnvironmentId, string nextUrlPrefix, string currentEnvironmentId, string currentUrlPrefix) { loggerProvider.GetLogger().Debug("Waiting for Cname swap to complete."); using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { bool nextEnvironmentComplete; bool currentEnvironmentComplete; do { await Task.Delay(TIME_BETWEEN_STATUS_CHECKS_IN_MILLISECONDS); var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(); currentEnvironmentComplete = describeEnvironmentsResponse.Environments .Any(env => env.EnvironmentId.IsEqualIgnoreCase(nextEnvironmentId) && env.Status == EnvironmentStatus.Ready && env.CNAME.IsEqualIgnoreCase(currentUrlPrefix)); nextEnvironmentComplete = describeEnvironmentsResponse.Environments .Any(env => env.EnvironmentId.IsEqualIgnoreCase(currentEnvironmentId) && env.Status == EnvironmentStatus.Ready && env.CNAME.IsEqualIgnoreCase(nextUrlPrefix)); loggerProvider.GetLogger().Debug($"Checking if CNAME Swap is complete. [IsCurrentEnv: {currentEnvironmentComplete}] [IsNextEnv: {nextEnvironmentComplete}]"); } while (!nextEnvironmentComplete && !currentEnvironmentComplete); loggerProvider.GetLogger().Debug("CName Swap Complete."); } }
public async Task PurgeOldApplicationVersionsAsync() { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeApplicationVersionsResponse = await ebClient.DescribeApplicationVersionsAsync(new DescribeApplicationVersionsRequest { ApplicationName = configurationProvider.LongApplicationName }); var oldVersions = describeApplicationVersionsResponse.ApplicationVersions .Where(av => av.DateCreated < DateTime.UtcNow.AddDays(-7)) .ToList(); loggerProvider.GetLogger().Debug($"Found {oldVersions.Count} old versions to delete"); foreach (var oldVersion in oldVersions) { var deleteApplicationVersionResponse = await ebClient.DeleteApplicationVersionAsync(new DeleteApplicationVersionRequest { ApplicationName = oldVersion.ApplicationName, VersionLabel = oldVersion.VersionLabel }); loggerProvider.GetLogger().Debug($"Deleting {oldVersion.VersionLabel} returned {deleteApplicationVersionResponse.HttpStatusCode}"); } } }
public async Task SwapEnvironmentCnamesAsync() { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { loggerProvider.GetLogger().Debug("Swapping Environment CNAMEs..."); var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(); var current = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(currentCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); var next = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(nextCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); if (current == null) { var environments = JsonConvert.SerializeObject(describeEnvironmentsResponse.Environments); var msg = $"Error state: On environment CNAME swap, could not find environment with CNAME: {currentCNamePrefix} in a READY state. Environments: {environments}"; loggerProvider.GetLogger().Error(msg); throw new ElasticBeanstalkDeployerException(msg); } if (next == null) { var environments = JsonConvert.SerializeObject(describeEnvironmentsResponse.Environments); var msg = $"Error state: On environment CNAME swap, could not find environment with CNAME: {nextCNamePrefix} in a READY state. Environments: {environments}"; loggerProvider.GetLogger().Error(msg); throw new ElasticBeanstalkDeployerException(msg); } var swapEnvironmentCnamesRequest = new SwapEnvironmentCNAMEsRequest { DestinationEnvironmentId = next.EnvironmentId, SourceEnvironmentId = current.EnvironmentId, }; await ebClient.SwapEnvironmentCNAMEsAsync(swapEnvironmentCnamesRequest); await WaitForCnameSwapToCompleteAsync(next.EnvironmentId, next.CNAME, current.EnvironmentId, current.CNAME); } }
/// <summary> /// /// </summary> /// <param name="createApplicationVersionRequest"></param> /// <returns></returns> public async Task CreateOrUpdateApplicationAsync(CreateApplicationVersionRequest createApplicationVersionRequest) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { if (await IsNewApplicationAsync(ebClient)) { loggerProvider.GetLogger().Debug("New application - creating..."); var createApplicationRequest = new CreateApplicationRequest { ApplicationName = configurationProvider.LongApplicationName }; var createApplicationResponse = await ebClient.CreateApplicationAsync(createApplicationRequest); loggerProvider.GetLogger().Debug($"Created application: {createApplicationResponse}"); } var describeApplicationsRequest = new DescribeApplicationVersionsRequest { ApplicationName = configurationProvider.LongApplicationName, VersionLabels = new List<string> { configurationProvider.Version } }; var apps = await ebClient.DescribeApplicationVersionsAsync(describeApplicationsRequest); if (apps.ApplicationVersions.Any()) { loggerProvider.GetLogger().Debug("Application Version already exists"); return; } loggerProvider.GetLogger().Debug("Creating application version..."); var createApplicationVersionResponse = await ebClient.CreateApplicationVersionAsync(createApplicationVersionRequest); loggerProvider.GetLogger().Debug($"Created application version: {createApplicationVersionResponse}"); } }
private async Task<EnvironmentDescription> GetEnvironmentDescription(Environment environment, AmazonElasticBeanstalkClient ebClient) { EnvironmentDescription targetEnvironment = null; var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(); if (environment == Environment.Current) { targetEnvironment = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(currentCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); } if (environment == Environment.Next) { targetEnvironment = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(nextCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); } return targetEnvironment; }
public async Task SingleSiteDockerApplicationEnsureSetupAsync(List<ConfigurationOptionSetting> optionSettings) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(); var current = describeEnvironmentsResponse.Environments .SingleOrDefault(env => env?.CNAME != null && env.CNAME.StartsWith(currentCNamePrefix, StringComparison.CurrentCultureIgnoreCase) && env.Status == EnvironmentStatus.Ready); loggerProvider.GetLogger().Debug("Current: {@current}", current); List<Task> terminatingTasks = new List<Task>(); if (current != null && current.Health != EnvironmentHealth.Green) { loggerProvider.GetLogger().Debug("Current Environment is not in green state. Deleting environment."); var terminateCurrentEnvTask = ebClient.TerminateEnvironmentAsync(new TerminateEnvironmentRequest { EnvironmentId = current.EnvironmentId, TerminateResources = true }); terminatingTasks.Add(terminateCurrentEnvTask); current = null; } if (terminatingTasks.Count > 0) { await Task.WhenAll(terminatingTasks); await EnsureTerminationsCompleteAsync(); } if (current != null) { loggerProvider.GetLogger().Debug("Environment is set up properly"); return; } loggerProvider.GetLogger().Debug("No environment is up - create"); Task.WaitAll(new List<Task> { CreateDockerCurrentEnvironmentAsync("A", optionSettings), }.ToArray()); } }
/// <summary> /// Checks that the environment is created before /// </summary> /// <param name="environmentId"></param> /// <returns></returns> private async Task EnsureCreateEnvironmentCompleteAsync(string environmentId) { using (var ebClient = new AmazonElasticBeanstalkClient(creds, configurationProvider.RegionEndpoint)) { var describeEnvironmentsResponse = await DescribeEnvironmentsWithRetryAsync(ebClient, environmentId); if (!describeEnvironmentsResponse.Environments.Any()) { loggerProvider.GetLogger().Warning("There are no environments!"); } var environmentDescription = describeEnvironmentsResponse.Environments.Single(); var retryStartUpState = 0; if (environmentDescription.Status == EnvironmentStatus.Ready) { loggerProvider.GetLogger().Debug("Environment {environmentId} is ready - no waiting!"); return; } do { //sleep await Task.Delay(TIME_BETWEEN_STATUS_CHECKS_IN_MILLISECONDS); describeEnvironmentsResponse = await ebClient.DescribeEnvironmentsAsync(new DescribeEnvironmentsRequest { EnvironmentIds = new List<string> { environmentId } }); environmentDescription = describeEnvironmentsResponse.Environments.Single(); loggerProvider.GetLogger().Debug($"Environment {environmentId} is Status: {environmentDescription.Status} and Health: {environmentDescription.Health}"); //failsafe to keep the environment from spinning once it has gone red if(environmentDescription.Status == EnvironmentStatus.Ready && environmentDescription.Health == EnvironmentHealth.Red) { retryStartUpState++; if (retryStartUpState >= 5) { throw new Exception("Environment failed to start up as expected."); } } } while (environmentDescription.Status != EnvironmentStatus.Ready || environmentDescription.Health != EnvironmentHealth.Green); } }