public static async Task <LayerPackageInfo> LoadLayerPackageInfos(IToolLogger logger, IAmazonLambda lambdaClient, IAmazonS3 s3Client, IEnumerable <string> layerVersionArns) { var info = new LayerPackageInfo(); if (layerVersionArns == null || !layerVersionArns.Any()) { return(info); } logger.WriteLine("Inspecting Lambda layers for runtime package store manifests"); foreach (var arn in layerVersionArns) { try { var p = ParseLayerVersionArn(arn); var getLayerResponse = await lambdaClient.GetLayerVersionAsync(new GetLayerVersionRequest { LayerName = p.Name, VersionNumber = p.VersionNumber }); LayerDescriptionManifest manifest; if (!LambdaUtilities.AttemptToParseLayerDescriptionManifest(getLayerResponse.Description, out manifest)) { logger.WriteLine($"... {arn}: Skipped, does not contain a layer description manifest"); continue; } if (manifest.Nlt != LayerDescriptionManifest.ManifestType.RuntimePackageStore) { logger.WriteLine($"... {arn}: Skipped, layer is of type {manifest.Nlt.ToString()}, not {LayerDescriptionManifest.ManifestType.RuntimePackageStore}"); continue; } string filePath = Path.GetTempFileName(); using (var getResponse = await s3Client.GetObjectAsync(manifest.Buc, manifest.Key)) using (var reader = new StreamReader(getResponse.ResponseStream)) { await getResponse.WriteResponseStreamToFileAsync(filePath, false, default(System.Threading.CancellationToken)); } logger.WriteLine($"... {arn}: Downloaded package manifest for runtime package store layer"); info.Items.Add(new LayerPackageInfo.LayerPackageInfoItem { Directory = manifest.Dir, ManifestPath = filePath }); } catch (Exception e) { logger.WriteLine($"... {arn}: Skipped, error inspecting layer. {e.Message}"); } } return(info); }
/// <summary> /// Search for the CloudFormation resources that references the app bundle sent to S3 and update them. /// </summary> /// <param name="templateBody"></param> /// <param name="s3Bucket"></param> /// <param name="s3Key"></param> /// <returns></returns> public static string UpdateCodeLocationInTemplate(string templateBody, string s3Bucket, string s3Key) { switch (LambdaUtilities.DetermineTemplateFormat(templateBody)) { case TemplateFormat.Json: return(UpdateCodeLocationInJsonTemplate(templateBody, s3Bucket, s3Key)); case TemplateFormat.Yaml: return(UpdateCodeLocationInYamlTemplate(templateBody, s3Bucket, s3Key)); default: throw new LambdaToolsException("Unable to determine template file format", LambdaToolsException.LambdaErrorCode.ServerlessTemplateParseError); } }
/// <summary> /// Execute the dotnet publish command and zip up the resulting publish folder. /// </summary> /// <param name="defaults"></param> /// <param name="logger"></param> /// <param name="workingDirectory"></param> /// <param name="projectLocation"></param> /// <param name="targetFramework"></param> /// <param name="configuration"></param> /// <param name="msbuildParameters"></param> /// <param name="disableVersionCheck"></param> /// <param name="publishLocation"></param> /// <param name="zipArchivePath"></param> public static bool CreateApplicationBundle(LambdaToolsDefaults defaults, IToolLogger logger, string workingDirectory, string projectLocation, string configuration, string targetFramework, string msbuildParameters, bool disableVersionCheck, LayerPackageInfo layerPackageInfo, out string publishLocation, ref string zipArchivePath) { LogDeprecationMessagesIfNecessary(logger, targetFramework); if (string.IsNullOrEmpty(configuration)) { configuration = LambdaConstants.DEFAULT_BUILD_CONFIGURATION; } var computedProjectLocation = Utilities.DetermineProjectLocation(workingDirectory, projectLocation); var lambdaRuntimePackageStoreManifestContent = LambdaUtilities.LoadPackageStoreManifest(logger, targetFramework); var publishManifestPath = new List <string>(); if (!string.IsNullOrEmpty(lambdaRuntimePackageStoreManifestContent)) { var tempFile = Path.GetTempFileName(); File.WriteAllText(tempFile, lambdaRuntimePackageStoreManifestContent); publishManifestPath.Add(tempFile); } if (layerPackageInfo != null) { foreach (var info in layerPackageInfo.Items) { publishManifestPath.Add(info.ManifestPath); } } var cli = new LambdaDotNetCLIWrapper(logger, workingDirectory); publishLocation = Utilities.DeterminePublishLocation(workingDirectory, projectLocation, configuration, targetFramework); logger?.WriteLine("Executing publish command"); if (cli.Publish(defaults, projectLocation, publishLocation, targetFramework, configuration, msbuildParameters, publishManifestPath) != 0) { return(false); } var buildLocation = Utilities.DetermineBuildLocation(workingDirectory, projectLocation, configuration, targetFramework); // This is here for legacy reasons. Some older versions of the dotnet CLI were not // copying the deps.json file into the publish folder. foreach (var file in Directory.GetFiles(buildLocation, "*.deps.json", SearchOption.TopDirectoryOnly)) { var destinationPath = Path.Combine(publishLocation, Path.GetFileName(file)); if (!File.Exists(destinationPath)) { File.Copy(file, destinationPath); } } bool flattenRuntime = false; var depsJsonTargetNode = GetDepsJsonTargetNode(logger, publishLocation); // If there is no target node then this means the tool is being used on a future version of .NET Core // then was available when the this tool was written. Go ahead and continue the deployment with warnings so the // user can see if the future version will work. if (depsJsonTargetNode != null && string.Equals(targetFramework, "netcoreapp1.0", StringComparison.OrdinalIgnoreCase)) { // Make sure the project is not pulling in dependencies requiring a later version of .NET Core then the declared target framework if (!ValidateDependencies(logger, targetFramework, depsJsonTargetNode, disableVersionCheck)) { return(false); } // Flatten the runtime folder which reduces the package size by not including native dependencies // for other platforms. flattenRuntime = FlattenRuntimeFolder(logger, publishLocation, depsJsonTargetNode); } if (zipArchivePath == null) { zipArchivePath = Path.Combine(Directory.GetParent(publishLocation).FullName, new DirectoryInfo(computedProjectLocation).Name + ".zip"); } zipArchivePath = Path.GetFullPath(zipArchivePath); logger?.WriteLine($"Zipping publish folder {publishLocation} to {zipArchivePath}"); if (File.Exists(zipArchivePath)) { File.Delete(zipArchivePath); } var zipArchiveParentDirectory = Path.GetDirectoryName(zipArchivePath); if (!Directory.Exists(zipArchiveParentDirectory)) { logger?.WriteLine($"Creating directory {zipArchiveParentDirectory}"); new DirectoryInfo(zipArchiveParentDirectory).Create(); } BundleDirectory(zipArchivePath, publishLocation, flattenRuntime, logger); return(true); }
/// <summary> /// Execute the dotnet store command on the provided package manifest /// </summary> /// <param name="defaults"></param> /// <param name="projectLocation"></param> /// <param name="outputLocation"></param> /// <param name="targetFramework"></param> /// <param name="packageManifest"></param> /// <param name="enableOptimization"></param> /// <returns></returns> public int Store(LambdaToolsDefaults defaults, string projectLocation, string outputLocation, string targetFramework, string packageManifest, bool enableOptimization) { if (outputLocation == null) { throw new ArgumentNullException(nameof(outputLocation)); } if (Directory.Exists(outputLocation)) { try { Directory.Delete(outputLocation, true); _logger?.WriteLine("Deleted previous publish folder"); } catch (Exception e) { _logger?.WriteLine($"Warning unable to delete previous publish folder: {e.Message}"); } } var dotnetCLI = FindExecutableInPath("dotnet.exe"); if (dotnetCLI == null) { dotnetCLI = FindExecutableInPath("dotnet"); } if (string.IsNullOrEmpty(dotnetCLI)) { throw new Exception("Failed to locate dotnet CLI executable. Make sure the dotnet CLI is installed in the environment PATH."); } var fullProjectLocation = this._workingDirectory; if (!string.IsNullOrEmpty(projectLocation)) { fullProjectLocation = Utilities.DetermineProjectLocation(this._workingDirectory, projectLocation); } var fullPackageManifest = Path.Combine(fullProjectLocation, packageManifest); _logger?.WriteLine($"... invoking 'dotnet store' for manifest {fullPackageManifest} into output directory {outputLocation}"); StringBuilder arguments = new StringBuilder("store"); if (!string.IsNullOrEmpty(outputLocation)) { arguments.Append($" --output \"{outputLocation}\""); } if (!string.IsNullOrEmpty(targetFramework)) { arguments.Append($" --framework \"{targetFramework}\""); } arguments.Append($" --manifest \"{fullPackageManifest}\""); arguments.Append($" --runtime {LambdaUtilities.DetermineRuntimeParameter(targetFramework)}"); if (!enableOptimization) { arguments.Append(" --skip-optimization"); } var psi = new ProcessStartInfo { FileName = dotnetCLI, Arguments = arguments.ToString(), WorkingDirectory = this._workingDirectory, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; var handler = (DataReceivedEventHandler)((o, e) => { if (string.IsNullOrEmpty(e.Data)) { return; } // Skip outputting this warning message as it adds a lot of noise to the output and is not actionable. // Full warning message being skipped: message NETSDK1062: // Unable to use package assets cache due to I/O error. This can occur when the same project is built // more than once in parallel. Performance may be degraded, but the build result will not be impacted. if (e.Data.Contains("message NETSDK1062")) { return; } _logger?.WriteLine("... store: " + e.Data); }); int exitCode; using (var proc = new Process()) { proc.StartInfo = psi; proc.Start(); proc.ErrorDataReceived += handler; proc.OutputDataReceived += handler; proc.BeginOutputReadLine(); proc.BeginErrorReadLine(); proc.EnableRaisingEvents = true; proc.WaitForExit(); exitCode = proc.ExitCode; } return(exitCode); }
/// <summary> /// Executes the dotnet publish command for the provided project /// </summary> /// <param name="defaults"></param> /// <param name="projectLocation"></param> /// <param name="outputLocation"></param> /// <param name="targetFramework"></param> /// <param name="configuration"></param> /// <param name="msbuildParameters"></param> /// <param name="deploymentTargetPackageStoreManifestContent"></param> public int Publish(LambdaToolsDefaults defaults, string projectLocation, string outputLocation, string targetFramework, string configuration, string msbuildParameters, IList <string> publishManifests) { if (outputLocation == null) { throw new ArgumentNullException(nameof(outputLocation)); } if (Directory.Exists(outputLocation)) { try { Directory.Delete(outputLocation, true); _logger?.WriteLine("Deleted previous publish folder"); } catch (Exception e) { _logger?.WriteLine($"Warning unable to delete previous publish folder: {e.Message}"); } } _logger?.WriteLine($"... invoking 'dotnet publish', working folder '{outputLocation}'"); var dotnetCLI = FindExecutableInPath("dotnet.exe"); if (dotnetCLI == null) { dotnetCLI = FindExecutableInPath("dotnet"); } if (string.IsNullOrEmpty(dotnetCLI)) { throw new Exception("Failed to locate dotnet CLI executable. Make sure the dotnet CLI is installed in the environment PATH."); } var fullProjectLocation = this._workingDirectory; if (!string.IsNullOrEmpty(projectLocation)) { fullProjectLocation = Utilities.DetermineProjectLocation(this._workingDirectory, projectLocation); } StringBuilder arguments = new StringBuilder("publish"); if (!string.IsNullOrEmpty(projectLocation)) { arguments.Append($" \"{fullProjectLocation}\""); } if (!string.IsNullOrEmpty(outputLocation)) { arguments.Append($" --output \"{outputLocation}\""); } if (!string.IsNullOrEmpty(configuration)) { arguments.Append($" --configuration \"{configuration}\""); } if (!string.IsNullOrEmpty(targetFramework)) { arguments.Append($" --framework \"{targetFramework}\""); } if (!string.IsNullOrEmpty(msbuildParameters)) { arguments.Append($" {msbuildParameters}"); } if (!string.Equals("netcoreapp1.0", targetFramework, StringComparison.OrdinalIgnoreCase)) { arguments.Append(" /p:GenerateRuntimeConfigurationFiles=true"); // Define an action to set the runtime and self-contained switches. var applyRuntimeSwitchAction = (Action)(() => { if (msbuildParameters == null || msbuildParameters.IndexOf("--runtime", StringComparison.InvariantCultureIgnoreCase) == -1) { arguments.Append($" --runtime {LambdaUtilities.DetermineRuntimeParameter(targetFramework)}"); } if (msbuildParameters == null || msbuildParameters.IndexOf("--self-contained", StringComparison.InvariantCultureIgnoreCase) == -1) { arguments.Append(" --self-contained false "); } }); // This is here to not change existing behavior for the 2.0 and 2.1 runtimes. For those runtimes if // cshtml files are being used we need to support that cshtml being compiled at runtime. In order to do that we // need to not turn PreserveCompilationContext which provides reference assemblies to the runtime // compilation and not set a runtime. // // If there are no cshtml then disable PreserveCompilationContext to reduce package size and continue // to use the same runtime identifier that we used when those runtimes were launched. if (new string[] { "netcoreapp2.0", "netcoreapp2.1" }.Contains(targetFramework)) { if (Directory.GetFiles(fullProjectLocation, "*.cshtml", SearchOption.AllDirectories).Length == 0) { applyRuntimeSwitchAction(); if (string.IsNullOrEmpty(msbuildParameters) || !msbuildParameters.Contains("PreserveCompilationContext")) { _logger?.WriteLine("... Disabling compilation context to reduce package size. If compilation context is needed pass in the \"/p:PreserveCompilationContext=false\" switch."); arguments.Append(" /p:PreserveCompilationContext=false"); } } } else { applyRuntimeSwitchAction(); } // If we have a manifest of packages already deploy in target deployment environment then write it to disk and add the // command line switch if (publishManifests != null && publishManifests.Count > 0) { foreach (var manifest in publishManifests) { arguments.Append($" --manifest \"{manifest}\""); } } } // echo the full dotnet command for debug _logger?.WriteLine($"... dotnet {arguments}"); var psi = new ProcessStartInfo { FileName = dotnetCLI, Arguments = arguments.ToString(), WorkingDirectory = this._workingDirectory, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; var handler = (DataReceivedEventHandler)((o, e) => { if (string.IsNullOrEmpty(e.Data)) { return; } _logger?.WriteLine("... publish: " + e.Data); }); int exitCode; using (var proc = new Process()) { proc.StartInfo = psi; proc.Start(); proc.ErrorDataReceived += handler; proc.OutputDataReceived += handler; proc.BeginOutputReadLine(); proc.BeginErrorReadLine(); proc.EnableRaisingEvents = true; proc.WaitForExit(); exitCode = proc.ExitCode; } if (exitCode == 0) { ProcessAdditionalFiles(defaults, outputLocation); var chmodPath = FindExecutableInPath("chmod"); var touchPath = FindExecutableInPath("touch"); if (!string.IsNullOrEmpty(chmodPath) && File.Exists(chmodPath)) { // as we are not invoking through a shell, which would handle // wildcard expansion for us, we need to invoke per-file var files = Directory.GetFiles(outputLocation, "*", SearchOption.TopDirectoryOnly); foreach (var file in files) { var filename = Path.GetFileName(file); ApplyCommand(outputLocation, chmodPath, $"+rx \"{filename}\"", handler); ApplyCommand(outputLocation, touchPath, $"-a -m -t 201512180130.09 \"{filename}\"", handler); } } } return(exitCode); }
/// <summary> /// Execute the dotnet publish command and zip up the resulting publish folder. /// </summary> /// <param name="defaults"></param> /// <param name="logger"></param> /// <param name="workingDirectory"></param> /// <param name="projectLocation"></param> /// <param name="targetFramework"></param> /// <param name="configuration"></param> /// <param name="msbuildParameters"></param> /// <param name="disableVersionCheck"></param> /// <param name="publishLocation"></param> /// <param name="zipArchivePath"></param> public static bool CreateApplicationBundle(LambdaToolsDefaults defaults, IToolLogger logger, string workingDirectory, string projectLocation, string configuration, string targetFramework, string msbuildParameters, bool disableVersionCheck, out string publishLocation, ref string zipArchivePath) { string lambdaRuntimePackageStoreManifestContent = null; var computedProjectLocation = Utilities.DetermineProjectLocation(workingDirectory, projectLocation); var packageStoreManifest = LambdaUtilities.LoadPackageStoreManifest(logger, targetFramework); if (!disableVersionCheck) { LambdaUtilities.ValidateMicrosoftAspNetCoreAllReferenceFromProjectPath(logger, targetFramework, packageStoreManifest, computedProjectLocation); } var cli = new LambdaDotNetCLIWrapper(logger, workingDirectory); publishLocation = Utilities.DeterminePublishLocation(workingDirectory, projectLocation, configuration, targetFramework); logger?.WriteLine("Executing publish command"); if (cli.Publish(defaults, projectLocation, publishLocation, targetFramework, configuration, msbuildParameters, lambdaRuntimePackageStoreManifestContent) != 0) { return(false); } var buildLocation = Utilities.DetermineBuildLocation(workingDirectory, projectLocation, configuration, targetFramework); // This is here for legacy reasons. Some older versions of the dotnet CLI were not // copying the deps.json file into the publish folder. foreach (var file in Directory.GetFiles(buildLocation, "*.deps.json", SearchOption.TopDirectoryOnly)) { var destinationPath = Path.Combine(publishLocation, Path.GetFileName(file)); if (!File.Exists(destinationPath)) { File.Copy(file, destinationPath); } } bool flattenRuntime = false; var depsJsonTargetNode = GetDepsJsonTargetNode(logger, publishLocation); // If there is no target node then this means the tool is being used on a future version of .NET Core // then was available when the this tool was written. Go ahead and continue the deployment with warnings so the // user can see if the future version will work. if (depsJsonTargetNode != null && string.Equals(targetFramework, "netcoreapp1.0", StringComparison.OrdinalIgnoreCase)) { // Make sure the project is not pulling in dependencies requiring a later version of .NET Core then the declared target framework if (!ValidateDependencies(logger, targetFramework, depsJsonTargetNode, disableVersionCheck)) { return(false); } // Flatten the runtime folder which reduces the package size by not including native dependencies // for other platforms. flattenRuntime = FlattenRuntimeFolder(logger, publishLocation, depsJsonTargetNode); } if (zipArchivePath == null) { zipArchivePath = Path.Combine(Directory.GetParent(publishLocation).FullName, new DirectoryInfo(computedProjectLocation).Name + ".zip"); } zipArchivePath = Path.GetFullPath(zipArchivePath); logger?.WriteLine($"Zipping publish folder {publishLocation} to {zipArchivePath}"); if (File.Exists(zipArchivePath)) { File.Delete(zipArchivePath); } var zipArchiveParentDirectory = Path.GetDirectoryName(zipArchivePath); if (!Directory.Exists(zipArchiveParentDirectory)) { logger?.WriteLine($"Creating directory {zipArchiveParentDirectory}"); new DirectoryInfo(zipArchiveParentDirectory).Create(); } #if NETCORE if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { BundleWithDotNetCompression(zipArchivePath, publishLocation, flattenRuntime, logger); } else { // Use the native zip utility if it exist which will maintain linux/osx file permissions var zipCLI = LambdaDotNetCLIWrapper.FindExecutableInPath("zip"); if (!string.IsNullOrEmpty(zipCLI)) { BundleWithZipCLI(zipCLI, zipArchivePath, publishLocation, flattenRuntime, logger); } else { throw new LambdaToolsException("Failed to find the \"zip\" utility program in path. This program is required to maintain Linux file permissions in the zip archive.", LambdaToolsException.LambdaErrorCode.FailedToFindZipProgram); } } #else BundleWithDotNetCompression(zipArchivePath, publishLocation, flattenRuntime, logger); #endif return(true); }