internal static async Task <ProcessResults> RunCommandAsync(
string filename,
string arguments,
string workingDirectory = null,
bool throwOnErrorExitCode = true,
string errorMessage = null,
CancellationToken?token = null)
{
// Output.Info($"Executing async command >{filename} {arguments}");
var results = await ProcessEx.RunAsync(filename, arguments, workingDirectory);
if (results.ExitCode != 0 && throwOnErrorExitCode)
{
var commandLog = $"Command failed with exit code {results.ExitCode}: > {filename} {arguments}";
if (token?.IsCancellationRequested != true)
{
if (!string.IsNullOrWhiteSpace(errorMessage))
{
Output.Error(errorMessage);
}
Output.Error(commandLog);
var indent = "==>";
Output.Error($"StandardError:\n{indent}{string.Join($"\n{indent}", results.StandardError)}");
Output.Verbose($"StandardOutput:\n{indent}{string.Join($"\n{indent}", results.StandardOutput)}");
}
throw new Exception(commandLog);
}
return(results);
}
private static async Task <bool> ValidateServiceConfigFile(IConfigurationFileValidator configValidator, string sourcePath, string completeConfigFile)
{
var result = await configValidator.ValidateConfigurationFile(sourcePath, completeConfigFile);
if (!result.IsValid)
{
Output.Error($"Invalid Service Configuration {completeConfigFile}: {result.Response}");
}
return(result.IsValid);
}
internal static string GetServiceName(string serviceConfigurationPath)
{
try {
var serviceYaml = new ServiceConfigurationProvider(serviceConfigurationPath).Load();
return(serviceYaml.Service.Name);
}
catch (Exception ex)
{
Output.Error($"Unable to read serviceName {ex.Message}");
return(null);
}
}
protected override async Task <int> Run(MarvelMicroserviceConfig config)
{
var authenticated = await Security.EnsureAuthenticatedWithEcr();
if (authenticated)
{
Output.Info("Authorization was successful.");
return(0);
}
else
{
Output.Error("Authorization failed.");
return(1);
}
}
internal static int ExecuteCommand(string executable, string args, bool writeOutput = true, string workingDirectory = "")
{
var command = $"{executable} {args}";
if (writeOutput)
{
Output.CommandExecution(command);
}
var consoleColour = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Gray;
Process p;
try
{
p = Process.Start(new ProcessStartInfo
{
FileName = executable,
Arguments = args,
RedirectStandardError = !writeOutput,
RedirectStandardOutput = !writeOutput,
WorkingDirectory = workingDirectory,
});
}
catch (Exception e)
{
throw new Exception($"Failed to start command `{command}`", e);
}
p.WaitForExit();
Console.ForegroundColor = consoleColour;
if (writeOutput)
{
Output.ExitCode(executable, p.ExitCode);
}
if (p.ExitCode != 0 && writeOutput)
{
Output.Error($"`{command}` exited with code {p.ExitCode}");
}
return(p.ExitCode);
}
public async Task <IValidatorResults> ValidateConfigurationFile(string baseDirectory, string fileToValidate)
{
var buildImageUri = await EcrResources.HsdImageUrl.EnsureImageIsPulled();
if (!buildImageUri.WasSuccessful)
{
throw new Exception($"Unable to get image {buildImageUri.Value} from ECR");
}
var validationOutputFile = $"{fileToValidate}.validation-output";
var dockerRunOptions = new List <string>
{
$"--rm",
$"-v {baseDirectory}:{ContainerPaths.MountedSourceDirectory}"
};
var hsdValidateCommand = $"/bin/bash hsd-validate.sh {ContainerPaths.MountedSourceDirectory}/{fileToValidate} {ContainerPaths.MountedSourceDirectory}/{validationOutputFile}";
var exitCode = CommandUtilities.ExecuteCommand("docker", $"run {string.Join(" ", dockerRunOptions)} {buildImageUri.Value} {hsdValidateCommand}");
if (exitCode != 0)
{
throw new Exception("HSD Validation failed.");
}
var outputFilePath = Path.Combine(baseDirectory, validationOutputFile);
var outputContents = File.ReadAllText(outputFilePath);
try
{
File.Delete(outputFilePath);
}
catch (Exception ex)
{
Output.Error($"Unable to delete validation output file: {ex.Message}");
}
var validationResult = new ValidationResult(outputContents);
return(validationResult);
}
public void Register(CommandLineApplication app, string currentDirectory, CommandOption verboseOption)
{
app.Command(CommandName, command =>
{
command.Description = Description;
AddHelpOption(command);
CreateOptions(command);
var commandVerboseOption = AddVerboseOption(command);
command.OnExecute(() =>
{
// Allow `-v` to be added before or after the subcommand, i.e., `dotnet marvel -v build` or `dotnet marvel build -v`
Output.UseVerbose = verboseOption.HasValue() || commandVerboseOption.HasValue();
var config = new MarvelMicroserviceConfig(currentDirectory);
Output.Info($"Running `dotnet {Program.CliCommandName} {CommandName}`...");
Output.Verbose("Using verbose logging");
Output.Info($"Using solution at {config.BaseDirectory}");
TrySetTestAuthentication();
var task = Task.Run(async() => await Run(config));
var code = task.GetAwaiter().GetResult();
if (code == 0)
{
Output.Success("Success.");
}
else
{
Output.Error("Failed.");
}
return(code);
});
});
}
private static async Task <GetAuthorizationTokenResponse> GetAuthorizationToken(string accessKeyId, string secretKey)
{
Output.Verbose($"Retrieving authorization token {nameof(accessKeyId)}={accessKeyId}, {nameof(secretKey)}={secretKey}.");
GetAuthorizationTokenResponse authorizationToken;
// We only want to get & save secrets from/to local configuration when we have not provided them
var needToStoreSecrets = false;
if (accessKeyId == null || secretKey == null)
{
Output.Verbose("Attempting to load AWS credentials from secrets store.");
var securityConfig = GetConfiguration();
if (accessKeyId == null)
{
accessKeyId = securityConfig[SecretStoreKeys.AccessKeyId];
}
if (secretKey == null)
{
secretKey = securityConfig[SecretStoreKeys.SecretKey];
}
}
var firstPrompt = true;
do
{
if (accessKeyId == null || secretKey == null)
{
Output.Verbose("Attempting to prompt for AWS credentials.");
needToStoreSecrets = true;
if (!ConsoleHelper.IsInteractive)
{
// CLI is running from within a script in a non-interactive terminal, e.g., from pressing F5 in VS Code. The
// auth command will get users to this same code path, but in an interactive terminal.
var cliCommand = Program.CliCommandName;
var authCommand = new AuthorizeCommand().CommandName;
Output.Error(Output.HorizontalLine());
Output.Error("Your machine is missing necessary AWS credentials. Authorize your machine by opening a terminal and " +
"running the following commands, then try your actions again. This authorization only needs to be performed once per user/machine:");
Output.Error($"> cd {Directory.GetCurrentDirectory()}");
Output.Error($"> dotnet {cliCommand} {authCommand}");
Output.Error($"If you have further issues, please contact #watch-ops.");
Output.Error(Output.HorizontalLine());
return(null);
}
// TODO add link to docs page for finding/creating AWS access keys and getting help from watch-ops to set up account
// or add permissions.
if (firstPrompt)
{
Output.Error("Your machine is missing necessary AWS credentials. The following authorization only needs to be performed " +
"once per user/machine. For assistance, please contact #watch-ops.");
firstPrompt = false;
}
Output.Error("Paste in your AWS ACCESS KEY ID:");
accessKeyId = ConsoleHelper.ReadPassword();
Output.Error("Now paste in your AWS SECRET ACCESS KEY:");
secretKey = ConsoleHelper.ReadPassword();
}
try
{
var ecrClient = new AmazonECRClient(accessKeyId, secretKey, _regionEndpoint);
// TODO Potentially slow. We might need to create a cached toked since it'll be valid for 12 hours
// Another thing is that there is a throttling for GetAuthorizationTokenAsync (1 call / second)
// There's not a good way to check the stored credentials without making
// an actual call to ECR (e.g. a docker pull). Doing that on every operation
// would be relatively slow. Could we write a hidden file on successful auth
// check and expire it after 8 hours?
Output.Verbose($"Retrieving ECR authentication token");
authorizationToken = await ecrClient.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest(),
new CancellationTokenSource(_awsTimeout).Token);
// We have successfully got authorization token let's save our access key and secret in user-secrets.
// We only save them when we've not provided access and secret via parameters.
if (needToStoreSecrets)
{
Output.Verbose($"Storing secret {SecretStoreKeys.AccessKeyId}");
await SaveSecret(SecretStoreKeys.AccessKeyId, accessKeyId);
Output.Verbose($"Storing secret {SecretStoreKeys.SecretKey}");
await SaveSecret(SecretStoreKeys.SecretKey, secretKey);
}
break;
}
catch (Exception e)
{
var amazonEcrException = e.InnerException as AmazonECRException;
if (amazonEcrException != null && amazonEcrException.ErrorCode.Equals("UnrecognizedClientException"))
{
Output.VerboseException(e);
Output.Error("Bad credentials. Please try again.");
accessKeyId = null;
secretKey = null;
continue;
}
// TODO handle case where permissions are inadequate (?)
Output.Error("Something went wrong while connecting to AWS. Please try again later.");
Output.Exception(e);
throw;
}
} while (true);
return(authorizationToken);
}
private static async Task <int> EnsureRegistratorContainerIsRunning(string eurekaServerUrl, string ipAddress)
{
var registratorImageUri = await EcrResources.RegistratorLatestImageUrl.EnsureImageIsPulled();
if (!registratorImageUri.WasSuccessful)
{
return(1);
}
var registratorContainerName = "registrator-marvel";
// Build arguments list for `docker create` call so we can hash them and ensure an existing container is compatible. Argument pairs are combined into one argument for readability.
var dockerCreateArgs = new List <string>
{
"--net=host",
"--volume=/var/run/docker.sock:/tmp/docker.sock"
};
var createContainerResult = await DockerCommands.GetOrCreateContainerWithName(
registratorContainerName,
registratorImageUri.Value,
dockerCreateArgs,
$"-ttl 30 -ttl-refresh 15 -ip {ipAddress} -require-label {FixEurekaServerUrlScheme(eurekaServerUrl ?? DefaultEurekaServerUrl)}");
var container = createContainerResult.Container;
var isNewContainer = createContainerResult.IsNewContainer;
if (!container.IsRunning)
{
Output.Info($"Starting container {registratorContainerName}");
var result = await CommandUtilities.RunCommandAsync("docker", $"start {container.ContainerId}", throwOnErrorExitCode : false);
if (result.ExitCode != 0)
{
var stdErr = string.Join("\n", result.StandardError);
// Message is `Bind for 0.0.0.0:5000 failed: port is already allocated`. trimming the port portion in case the port changes.
// if (stdErr.Contains("failed: port is already allocated"))
// {
// Output.Info("Webapp port is already in use. Attempting to stop other container using port.");
// // Find other containers using a partial match on suffix. This corresponds to the naming scheme defined in MarvelMicroserviceConfig.
// var containers = await DockerCommands.GetContainersByName("marvel-dev");
// var otherContainer = containers.FirstOrDefault(c => c.IsRunning);
// if (otherContainer == null)
// {
// Output.Error("Unable to find running container using same port.");
// Output.Error($"Failed to start container {containerName}. StandardError={stdErr}");
// return 1;
// }
// Output.Info($"Stopping container {otherContainer.ContainerId}");
// await DockerCommands.StopContainer(otherContainer);
// Output.Info($"Starting container {containerName} again");
// var restartResult = await ProcessEx.RunAsync("docker", $"start {container.ContainerId}");
// if (restartResult.ExitCode != 0)
// {
// Output.Error($"Failed to restart container {containerName}. StandardError={stdErr}");
// return result.ExitCode;
// }
// }
// else
// {
Output.Error($"Failed to start container {registratorContainerName}. StandardError={stdErr}");
return(result.ExitCode);
// }
}
// Ensure the container doesn't immediately exit.
// TODO Bumped this up for registrator specifically to ensure eureka host is valid. Might want to verify by scanning logs
// that this did in fact start up properly.
Thread.Sleep(500);
var runningContainer = await DockerCommands.GetContainerByName(registratorContainerName);
if (!runningContainer.IsRunning)
{
Output.Error($"Container {registratorContainerName} stopped unexpectedly. Check container logs by running {DockerCommands.GetLogsForContainerCommand(runningContainer)}.");
return(1);
}
}
// TODO - ensure registrator is ready? pause?
Output.Info($"Container {registratorContainerName} is running.");
return(0);
}
public static async Task <int> Launch(MarvelMicroserviceConfig config, LaunchOptions launchOptions)
{
var registratorRunResultTask = EnsureRegistratorContainerIsRunning(launchOptions.EurekaServer, launchOptions.LocalIpAddress);
var buildImageUriTask = EcrResources.DotnetMicroserviceBuildImageUrl.EnsureImageIsPulled();
var registratorRunResult = await registratorRunResultTask;
if (registratorRunResult != 0)
{
return(registratorRunResult);
}
var buildImageUri = await buildImageUriTask;
if (!buildImageUri.WasSuccessful)
{
return(1);
}
// # TODO include a hash of the tools version in the container name to ensure containers are recreated after tools update(?)
var containerName = config.DevContainerName;
var dockerTaskScriptPath = ContainerPaths.DockerTaskScriptPath;
// Build arguments list for `docker create` call so we can hash them and ensure an existing container is compatible. Argument pairs are combined into one argument for readability.
var hostname = launchOptions.Host ?? await NetworkUtility.GetFriendlyHostName();
var labelArgs = LabelUtilities.FormatLabelsAsArguments(await new LabelBuilder(config).GetLabelsForLocalDev(hostname));
var dockerCreateArgs = labelArgs.Concat(new List <string>
{
"-p 5000:5000",
"--dns-search=agilesports.local",
$"-e \"{ProjectNameEnvVariable}={config.ProjectName}\"",
$"-e \"{SolutionNameEnvVariable}={config.SolutionName}\"",
$"-e \"{WebappDirEnvVariable}={ContainerPaths.GetWebappDirectory(config)}\"",
$"-e \"{BrowserAppDirEnvVariable}={ContainerPaths.GetBrowserAppDirectory(config)}\"",
// This could probably live in the image
$"-e ASPNETCORE_ENVIRONMENT=Development",
$"-v {config.BaseDirectory}:{ContainerPaths.MountedSourceDirectory}",
});
if (launchOptions.UseSharedCache)
{
dockerCreateArgs = dockerCreateArgs.Concat(GetCacheVolumeArgs());
}
var createContainerResult = await DockerCommands.GetOrCreateContainerWithName(
containerName,
buildImageUri.Value,
dockerCreateArgs.ToList(),
$"/bin/bash {dockerTaskScriptPath} hang");
var container = createContainerResult.Container;
var isNewContainer = createContainerResult.IsNewContainer;
if (!container.IsRunning)
{
Output.Info($"Starting container {containerName}");
var result = await CommandUtilities.RunCommandAsync("docker", $"start {container.ContainerId}", throwOnErrorExitCode : false);
// Output.Info("StdErr=" + string.Join("\n", result.StandardError));
// Output.Info("StdOut=" + string.Join("\n", result.StandardOutput));
if (result.ExitCode != 0)
{
var stdErr = string.Join("\n", result.StandardError);
// Message is `Bind for 0.0.0.0:5000 failed: port is already allocated`. trimming the port portion in case the port changes.
if (stdErr.Contains("failed: port is already allocated"))
{
Output.Info("Webapp port is already in use. Attempting to stop other container using port.");
// Find other containers using a partial match on suffix. This corresponds to the naming scheme defined in MarvelMicroserviceConfig.
var containers = await DockerCommands.GetContainersByName("marvel-dev");
var otherContainer = containers.FirstOrDefault(c => c.IsRunning);
if (otherContainer == null)
{
Output.Error("Unable to find running container using same port.");
Output.Error($"Failed to start container {containerName}. StandardError={stdErr}");
return(1);
}
Output.Info($"Stopping container {otherContainer.ContainerId}");
await DockerCommands.StopContainer(otherContainer);
Output.Info($"Starting container {containerName} again");
var restartResult = await ProcessEx.RunAsync("docker", $"start {container.ContainerId}");
if (restartResult.ExitCode != 0)
{
Output.Error($"Failed to restart container {containerName}. StandardError={stdErr}");
return(result.ExitCode);
}
}
else
{
Output.Error($"Failed to start container {containerName}. StandardError={stdErr}");
return(result.ExitCode);
}
}
// TODO only perform this check after failed `docker exec` commands, for better reporting?
// Ensure the container doesn't immediately exit.
Thread.Sleep(10);
var runningContainer = await DockerCommands.GetContainerByName(containerName);
if (!runningContainer.IsRunning)
{
Output.Error($"Container {containerName} stopped unexpectedly. Check container logs.");
return(1);
}
}
if (isNewContainer)
{
Output.Info($"Attaching to container {containerName} to run first time launch command on new container");
var code = CommandUtilities.ExecuteCommand("docker", $"exec -i {container.ContainerId} /bin/bash {dockerTaskScriptPath} firstTimeLaunch");
if (code != 0)
{
Output.Info($"First time startup command failed. Removing container.");
await DockerCommands.RemoveContainer(container);
return(code);
}
}
else
{
Output.Info($"Attaching to container {containerName} to run relaunch command on existing container");
var code = CommandUtilities.ExecuteCommand("docker", $"exec -i {container.ContainerId} /bin/bash {dockerTaskScriptPath} relaunch");
if (code != 0)
{
return(code);
}
}
Output.Info($"Container {containerName} launched and ready to run");
Output.Info($"Using hostname: {hostname}");
Output.Info($"If debugging from VS Code, switch to the Debug Console (Cmd+Shift+Y / Ctrl+Shift+Y) for app and watch process logs");
return(0);
}
private static void LogValidationErrors(List <string> errors)
{
errors.ForEach(error => Output.Error($"Argument validation error: {error}"));
}
internal static async Task <int> Publish(MarvelMicroserviceConfig config, IBuildConfigurationBuilder configBuilder, IConfigurationFileMerger configMerger, IConfigurationFileUploader configUploader, IConfigurationFileValidator configValidator, IStaticAssetProcessor staticAssetProcessor, string nugetApiKey, string awsAccessKey, string awsAccessSecret, string branch, string gitSha, string buildNumber, bool mergeAndUploadServiceConfig, bool mergeServiceConfig)
{
Security.UseAwsCredentials(awsAccessKey, awsAccessSecret);
var publishImage = ImageNameBuilder.CreateImageNameAndTag(
config.ServiceName,
branch,
gitSha,
DateTime.UtcNow,
buildNumber);
string[] serviceConfigFiles = null;
if (mergeAndUploadServiceConfig || mergeServiceConfig)
{
GenerateBuildFile(configBuilder, config.BuildConfigFilePath, gitSha, branch, publishImage.FullPath, buildNumber);
serviceConfigFiles = await MergeAllServiceConfigFiles(configMerger, config.SourceDirectory, config.ServiceConfigFileName, config.BuildConfigFilePath);
var configIsValid = await ValidateAllServiceConfigFiles(configValidator, config.SourceDirectory, serviceConfigFiles);
if (!configIsValid)
{
Output.Error("Invalid service configuration.");
return(1);
}
}
var exitCode = await BuildCommand.Build(config, publishImage.FullPath, new BuildConfig
{
BranchName = branch,
BuildNumber = buildNumber,
}, staticAssetProcessor);
if (exitCode != 0)
{
return(exitCode);
}
try
{
exitCode = PublishClientPackage(config, nugetApiKey, awsAccessKey, awsAccessSecret, branch, gitSha, buildNumber);
if (exitCode != 0)
{
return(exitCode);
}
// Publish to ECR
Output.Info($"Publishing {publishImage.FullPath}");
await Security.EnsureAuthenticatedWithEcr();
exitCode = CommandUtilities.ExecuteCommand("docker", $"push {publishImage.FullPath}");
if (exitCode != 0)
{
return(exitCode);
}
}
finally
{
// TODO always remove image, even on publish failure
await CommandUtilities.RunCommandAsync("docker", $"rmi {publishImage.FullPath}", errorMessage : $"Failed to remove image {publishImage.FullPath}.");
Output.Info($"Removed local image {publishImage.FullPath}");
}
try
{
if (mergeAndUploadServiceConfig && serviceConfigFiles != null)
{
await UploadAllServiceConfigFiles(configUploader, config.SourceDirectory, serviceConfigFiles, publishImage.Tag);
}
}
catch (Exception ex)
{
Output.Error($"Unable to upload service configuration files. Error: {ex.Message}");
return(1);
}
File.WriteAllText(Path.Combine(config.WebappDirectory, "PublishedImageUrl.txt"), publishImage.FullPath);
Output.Info("Publish successful");
return(0);
}