private static int PublishClientPackage(MarvelMicroserviceConfig config, string nugetApiKey, string awsAccessKey, string awsAccessSecret, string branch, string gitSha, string buildNumber)
{
var clientDir = config.ClientPackageDirectory;
if (!Directory.Exists(clientDir))
{
Output.Info($"No client package directory found at {clientDir}.");
Output.Info($"Skipping client package publish.");
return(0);
}
Output.Info("Publishing Client NuGet package.");
var exitCode = CommandUtilities.ExecuteCommand("dotnet", $"restore", workingDirectory: clientDir);
if (exitCode != 0)
{
return(exitCode);
}
exitCode = CommandUtilities.ExecuteCommand("dotnet", $"library publish " +
$"-b {branch} " +
$"-k {nugetApiKey} " +
$"-a {awsAccessKey} " +
$"-s {awsAccessSecret} " +
$"-n {buildNumber} " +
$"--tests-optional", workingDirectory: clientDir);
if (exitCode != 0)
{
return(exitCode);
}
return(0);
}
protected override async Task <int> Run(MarvelMicroserviceConfig config)
{
var options = await _launchOptions.ProcessOptions();
if (!options.AreValid)
{
return(1);
}
return(await Launch(config, options.Value));
}
protected override Task <int> Run(MarvelMicroserviceConfig config)
{
// Used for image labeling purposes, so shouldn't really matter on local build command
var devBuildConfig = new BuildConfig
{
BranchName = "local-dev-build",
BuildNumber = "123456",
};
// Use short-term storage for `build` command because it is only used locally. `publish` is used on TeamCity.
_assetHostConfigurationAccessor.SetValue(S3AssetHostConfiguration.TestAssets);
return(Build(config, config.DevPublishedRuntimeImageName, devBuildConfig, _staticAssetProcessor));
}
protected override async Task <int> Run(MarvelMicroserviceConfig config)
{
var launchOptionsResult = await _launchOptions.ProcessOptions();
if (!launchOptionsResult.AreValid)
{
return(1);
}
var exitCode = await LaunchCommand.Launch(config, launchOptionsResult.Value);
if (exitCode != 0)
{
return(exitCode);
}
var containerName = config.DevContainerName;
var container = await DockerCommands.GetContainerByName(containerName);
if (container?.IsRunning != true)
{
Output.Info($"Could not find running container {containerName}");
return(1);
}
// Call basic command to see if we're executing in a TTY
var ttyTest = await CommandUtilities.RunCommandAsync("docker", $"exec -it {container.ContainerId} /bin/bash -c echo hi", throwOnErrorExitCode : false);
var isTty = true;
if (ttyTest.ExitCode != 0)
{
var stdErr = string.Join("\n", ttyTest.StandardError);
if (!stdErr.Contains("input device is not a TTY"))
{
throw new Exception($"Unexpected exception encounterd checking for TTY StandardError={stdErr}");
}
isTty = false;
}
Output.Info($"Attaching to container {containerName}");
// Use TTY option when available so that Ctrl+C in the terminal kills the process inside the container as well. Option cannot be used during integration test runs.
var ttyArg = isTty ? "t" : "";
exitCode = CommandUtilities.ExecuteCommand("docker", $"exec -i{ttyArg} {container.ContainerId} /bin/bash {ContainerPaths.DockerTaskScriptPath} watchAndRun");
return(exitCode);
}
protected override async Task <int> Run(MarvelMicroserviceConfig config)
{
var authenticated = await Security.EnsureAuthenticatedWithEcr();
if (authenticated)
{
Output.Info("Authorization was successful.");
return(0);
}
else
{
Output.Error("Authorization failed.");
return(1);
}
}
protected override async Task <int> Run(MarvelMicroserviceConfig config)
{
var exitCode = 0;
var containerName = config.DevContainerName;
var container = await DockerCommands.GetContainerByName(containerName);
if (container?.IsRunning != true)
{
Output.Info($"Could not find running container {containerName}");
return(1);
}
Output.Info($"Attaching to container {containerName}");
exitCode = CommandUtilities.ExecuteCommand("docker", $"exec -i {container.ContainerId} /bin/bash {ContainerPaths.DockerTaskScriptPath} watchAndDebug");
return(exitCode);
}
protected override async Task <int> Run(MarvelMicroserviceConfig config)
{
foreach (var kvp in new Dictionary <string, CommandOption> {
{ "branch", _branchOption },
{ "git SHA", _gitShaOption },
{ "build number", _buildNumberOption },
{ "NuGet API key", _nugetApiKeyOption },
{ "AWS access key id", _awsAccessKeyOption },
{ "AWS secret key", _awsAccessSecretOption },
{ "persist static assets", _persistStaticAssetsOption },
})
{
Output.Verbose($"Option {kvp.Key}: {kvp.Value.Value()}");
}
var paramValidationErrors = ValidateParams();
if (paramValidationErrors.Any())
{
paramValidationErrors.ForEach(Output.Error);
return(1);
}
var persistStaticAssets = _persistStaticAssetsOption.HasValue();
if (!persistStaticAssets)
{
Output.Info($"NOTE: Static assets will not be published to a persistent CDN. They will be temporarily available (minimum of 1 day) for testing with this build. " +
$"If this build is for a deployable app (e.g., a CI build for either prod or thor environments, or a locally-built emergency deploy), " +
$"you should specify the {_persistStaticAssetsOption.LongName} option. If this is a local build for manual testing or an automated testing run, you may ignore this.");
}
_assetHostConfigurationAccessor.SetValue(persistStaticAssets ? S3AssetHostConfiguration.ProductionAssets : S3AssetHostConfiguration.TestAssets);
return(await Publisher.Publish(config, _buildConfigurationBuilder, _configurationFileMerger, _configurationFileUploader, _configurationFileValidator, _staticAssetProcessor, _nugetApiKeyOption.Value(), _awsAccessKeyOption.Value(), _awsAccessSecretOption.Value(), _branchOption.Value(), _gitShaOption.Value(), _buildNumberOption.Value(), _mergeAndUploadServiceConfig.HasValue(), _mergeServiceConfig.HasValue()));
}
public void Register(CommandLineApplication app, string currentDirectory, CommandOption verboseOption)
{
app.Command(CommandName, command =>
{
command.Description = Description;
AddHelpOption(command);
CreateOptions(command);
var commandVerboseOption = AddVerboseOption(command);
command.OnExecute(() =>
{
// Allow `-v` to be added before or after the subcommand, i.e., `dotnet marvel -v build` or `dotnet marvel build -v`
Output.UseVerbose = verboseOption.HasValue() || commandVerboseOption.HasValue();
var config = new MarvelMicroserviceConfig(currentDirectory);
Output.Info($"Running `dotnet {Program.CliCommandName} {CommandName}`...");
Output.Verbose("Using verbose logging");
Output.Info($"Using solution at {config.BaseDirectory}");
TrySetTestAuthentication();
var task = Task.Run(async() => await Run(config));
var code = task.GetAwaiter().GetResult();
if (code == 0)
{
Output.Success("Success.");
}
else
{
Output.Error("Failed.");
}
return(code);
});
});
}
public static async Task <int> Launch(MarvelMicroserviceConfig config, LaunchOptions launchOptions)
{
var registratorRunResultTask = EnsureRegistratorContainerIsRunning(launchOptions.EurekaServer, launchOptions.LocalIpAddress);
var buildImageUriTask = EcrResources.DotnetMicroserviceBuildImageUrl.EnsureImageIsPulled();
var registratorRunResult = await registratorRunResultTask;
if (registratorRunResult != 0)
{
return(registratorRunResult);
}
var buildImageUri = await buildImageUriTask;
if (!buildImageUri.WasSuccessful)
{
return(1);
}
// # TODO include a hash of the tools version in the container name to ensure containers are recreated after tools update(?)
var containerName = config.DevContainerName;
var dockerTaskScriptPath = ContainerPaths.DockerTaskScriptPath;
// Build arguments list for `docker create` call so we can hash them and ensure an existing container is compatible. Argument pairs are combined into one argument for readability.
var hostname = launchOptions.Host ?? await NetworkUtility.GetFriendlyHostName();
var labelArgs = LabelUtilities.FormatLabelsAsArguments(await new LabelBuilder(config).GetLabelsForLocalDev(hostname));
var dockerCreateArgs = labelArgs.Concat(new List <string>
{
"-p 5000:5000",
"--dns-search=agilesports.local",
$"-e \"{ProjectNameEnvVariable}={config.ProjectName}\"",
$"-e \"{SolutionNameEnvVariable}={config.SolutionName}\"",
$"-e \"{WebappDirEnvVariable}={ContainerPaths.GetWebappDirectory(config)}\"",
$"-e \"{BrowserAppDirEnvVariable}={ContainerPaths.GetBrowserAppDirectory(config)}\"",
// This could probably live in the image
$"-e ASPNETCORE_ENVIRONMENT=Development",
$"-v {config.BaseDirectory}:{ContainerPaths.MountedSourceDirectory}",
});
if (launchOptions.UseSharedCache)
{
dockerCreateArgs = dockerCreateArgs.Concat(GetCacheVolumeArgs());
}
var createContainerResult = await DockerCommands.GetOrCreateContainerWithName(
containerName,
buildImageUri.Value,
dockerCreateArgs.ToList(),
$"/bin/bash {dockerTaskScriptPath} hang");
var container = createContainerResult.Container;
var isNewContainer = createContainerResult.IsNewContainer;
if (!container.IsRunning)
{
Output.Info($"Starting container {containerName}");
var result = await CommandUtilities.RunCommandAsync("docker", $"start {container.ContainerId}", throwOnErrorExitCode : false);
// Output.Info("StdErr=" + string.Join("\n", result.StandardError));
// Output.Info("StdOut=" + string.Join("\n", result.StandardOutput));
if (result.ExitCode != 0)
{
var stdErr = string.Join("\n", result.StandardError);
// Message is `Bind for 0.0.0.0:5000 failed: port is already allocated`. trimming the port portion in case the port changes.
if (stdErr.Contains("failed: port is already allocated"))
{
Output.Info("Webapp port is already in use. Attempting to stop other container using port.");
// Find other containers using a partial match on suffix. This corresponds to the naming scheme defined in MarvelMicroserviceConfig.
var containers = await DockerCommands.GetContainersByName("marvel-dev");
var otherContainer = containers.FirstOrDefault(c => c.IsRunning);
if (otherContainer == null)
{
Output.Error("Unable to find running container using same port.");
Output.Error($"Failed to start container {containerName}. StandardError={stdErr}");
return(1);
}
Output.Info($"Stopping container {otherContainer.ContainerId}");
await DockerCommands.StopContainer(otherContainer);
Output.Info($"Starting container {containerName} again");
var restartResult = await ProcessEx.RunAsync("docker", $"start {container.ContainerId}");
if (restartResult.ExitCode != 0)
{
Output.Error($"Failed to restart container {containerName}. StandardError={stdErr}");
return(result.ExitCode);
}
}
else
{
Output.Error($"Failed to start container {containerName}. StandardError={stdErr}");
return(result.ExitCode);
}
}
// TODO only perform this check after failed `docker exec` commands, for better reporting?
// Ensure the container doesn't immediately exit.
Thread.Sleep(10);
var runningContainer = await DockerCommands.GetContainerByName(containerName);
if (!runningContainer.IsRunning)
{
Output.Error($"Container {containerName} stopped unexpectedly. Check container logs.");
return(1);
}
}
if (isNewContainer)
{
Output.Info($"Attaching to container {containerName} to run first time launch command on new container");
var code = CommandUtilities.ExecuteCommand("docker", $"exec -i {container.ContainerId} /bin/bash {dockerTaskScriptPath} firstTimeLaunch");
if (code != 0)
{
Output.Info($"First time startup command failed. Removing container.");
await DockerCommands.RemoveContainer(container);
return(code);
}
}
else
{
Output.Info($"Attaching to container {containerName} to run relaunch command on existing container");
var code = CommandUtilities.ExecuteCommand("docker", $"exec -i {container.ContainerId} /bin/bash {dockerTaskScriptPath} relaunch");
if (code != 0)
{
return(code);
}
}
Output.Info($"Container {containerName} launched and ready to run");
Output.Info($"Using hostname: {hostname}");
Output.Info($"If debugging from VS Code, switch to the Debug Console (Cmd+Shift+Y / Ctrl+Shift+Y) for app and watch process logs");
return(0);
}
public static async Task <int> Build(MarvelMicroserviceConfig config, string publishedRuntimeImageName, BuildConfig buildConfig, IStaticAssetProcessor staticAssetProcessor)
{
var buildImageUri = await EcrResources.DotnetMicroserviceBuildImageUrl.EnsureImageIsPulled();
if (!buildImageUri.WasSuccessful)
{
return(1);
}
var taskTimer = Stopwatch.StartNew();
// Clear the old temp dir to ensure a fresh publish if running locally
var outputPath = config.DotnetPublishOutputPath;
if (Directory.Exists(outputPath))
{
Directory.Delete(outputPath, true);
}
Directory.CreateDirectory(outputPath);
var dockerRunOptions = new List <string>
{
$"--rm",
$"-e \"{LaunchCommand.ProjectNameEnvVariable}={config.ProjectName}\"",
$"-e \"{LaunchCommand.SolutionNameEnvVariable}={config.SolutionName}\"",
$"-e \"{WebappDirEnvVariable}={ContainerPaths.GetWebappDirectory(config)}\"",
$"-e \"{BrowserAppDirEnvVariable}={ContainerPaths.GetBrowserAppDirectory(config)}\"",
$"-v {config.BaseDirectory}:{ContainerPaths.MountedSourceDirectory}",
$"-v {outputPath}:{ContainerPaths.BuildOutputDirectory}",
};
var runtimeImageLabelsTask = new LabelBuilder(config).GetLabels(buildConfig);
var exitCode = 0;
var dotnetBuildTimer = Stopwatch.StartNew();
Output.Info($"Building dotnet webapp.");
// Run container to build app and copy published resources to mounted output directory
exitCode = CommandUtilities.ExecuteCommand("docker", $"run {string.Join(" ", dockerRunOptions)} {buildImageUri.Value} /bin/bash {ContainerPaths.DockerTaskScriptPath} buildWithoutCompose");
if (exitCode != 0)
{
return(exitCode);
}
Output.Info($"dotnet webapp build completed {dotnetBuildTimer.Elapsed}");
// Run static asset build from output directory
await staticAssetProcessor.ProcessStaticAssets(Path.Combine(config.DotnetPublishOutputPath, "wwwroot"));
// Build the image from the source output
var dockerBuildTimer = Stopwatch.StartNew();
Output.Info($"Building docker image {publishedRuntimeImageName}.");
var labelArgs = LabelUtilities.FormatLabelsAsArguments(await runtimeImageLabelsTask);
var buildArgs = labelArgs.Concat(new List <string> {
$"-t {publishedRuntimeImageName}",
$"--build-arg webappAssemblyPath={config.PublishedWebappAssemblyPath}",
config.DotnetPublishOutputPath,
});
exitCode = CommandUtilities.ExecuteCommand("docker", $"build {string.Join(" ", buildArgs)}");
if (exitCode != 0)
{
return(exitCode);
}
Output.Info($"Docker build completed {dockerBuildTimer.Elapsed}");
Output.Info($"Build time elapsed {taskTimer.Elapsed}");
return(exitCode);
}
public LabelBuilder(MarvelMicroserviceConfig serviceConfig)
{
_serviceConfig = serviceConfig;
}
internal static async Task <int> Publish(MarvelMicroserviceConfig config, IBuildConfigurationBuilder configBuilder, IConfigurationFileMerger configMerger, IConfigurationFileUploader configUploader, IConfigurationFileValidator configValidator, IStaticAssetProcessor staticAssetProcessor, string nugetApiKey, string awsAccessKey, string awsAccessSecret, string branch, string gitSha, string buildNumber, bool mergeAndUploadServiceConfig, bool mergeServiceConfig)
{
Security.UseAwsCredentials(awsAccessKey, awsAccessSecret);
var publishImage = ImageNameBuilder.CreateImageNameAndTag(
config.ServiceName,
branch,
gitSha,
DateTime.UtcNow,
buildNumber);
string[] serviceConfigFiles = null;
if (mergeAndUploadServiceConfig || mergeServiceConfig)
{
GenerateBuildFile(configBuilder, config.BuildConfigFilePath, gitSha, branch, publishImage.FullPath, buildNumber);
serviceConfigFiles = await MergeAllServiceConfigFiles(configMerger, config.SourceDirectory, config.ServiceConfigFileName, config.BuildConfigFilePath);
var configIsValid = await ValidateAllServiceConfigFiles(configValidator, config.SourceDirectory, serviceConfigFiles);
if (!configIsValid)
{
Output.Error("Invalid service configuration.");
return(1);
}
}
var exitCode = await BuildCommand.Build(config, publishImage.FullPath, new BuildConfig
{
BranchName = branch,
BuildNumber = buildNumber,
}, staticAssetProcessor);
if (exitCode != 0)
{
return(exitCode);
}
try
{
exitCode = PublishClientPackage(config, nugetApiKey, awsAccessKey, awsAccessSecret, branch, gitSha, buildNumber);
if (exitCode != 0)
{
return(exitCode);
}
// Publish to ECR
Output.Info($"Publishing {publishImage.FullPath}");
await Security.EnsureAuthenticatedWithEcr();
exitCode = CommandUtilities.ExecuteCommand("docker", $"push {publishImage.FullPath}");
if (exitCode != 0)
{
return(exitCode);
}
}
finally
{
// TODO always remove image, even on publish failure
await CommandUtilities.RunCommandAsync("docker", $"rmi {publishImage.FullPath}", errorMessage : $"Failed to remove image {publishImage.FullPath}.");
Output.Info($"Removed local image {publishImage.FullPath}");
}
try
{
if (mergeAndUploadServiceConfig && serviceConfigFiles != null)
{
await UploadAllServiceConfigFiles(configUploader, config.SourceDirectory, serviceConfigFiles, publishImage.Tag);
}
}
catch (Exception ex)
{
Output.Error($"Unable to upload service configuration files. Error: {ex.Message}");
return(1);
}
File.WriteAllText(Path.Combine(config.WebappDirectory, "PublishedImageUrl.txt"), publishImage.FullPath);
Output.Info("Publish successful");
return(0);
}
protected abstract Task <int> Run(MarvelMicroserviceConfig config);
