public static async Task <LayerPackageInfo> LoadLayerPackageInfos(IToolLogger logger, IAmazonLambda lambdaClient, IAmazonS3 s3Client, IEnumerable <string> layerVersionArns)
{
var info = new LayerPackageInfo();
if (layerVersionArns == null || !layerVersionArns.Any())
{
return(info);
}
logger.WriteLine("Inspecting Lambda layers for runtime package store manifests");
foreach (var arn in layerVersionArns)
{
try
{
var p = ParseLayerVersionArn(arn);
var getLayerResponse = await lambdaClient.GetLayerVersionAsync(new GetLayerVersionRequest { LayerName = p.Name, VersionNumber = p.VersionNumber });
LayerDescriptionManifest manifest;
if (!LambdaUtilities.AttemptToParseLayerDescriptionManifest(getLayerResponse.Description, out manifest))
{
logger.WriteLine($"... {arn}: Skipped, does not contain a layer description manifest");
continue;
}
if (manifest.Nlt != LayerDescriptionManifest.ManifestType.RuntimePackageStore)
{
logger.WriteLine($"... {arn}: Skipped, layer is of type {manifest.Nlt.ToString()}, not {LayerDescriptionManifest.ManifestType.RuntimePackageStore}");
continue;
}
string filePath = Path.GetTempFileName();
using (var getResponse = await s3Client.GetObjectAsync(manifest.Buc, manifest.Key))
using (var reader = new StreamReader(getResponse.ResponseStream))
{
await getResponse.WriteResponseStreamToFileAsync(filePath, false, default(System.Threading.CancellationToken));
}
logger.WriteLine($"... {arn}: Downloaded package manifest for runtime package store layer");
info.Items.Add(new LayerPackageInfo.LayerPackageInfoItem
{
Directory = manifest.Dir,
ManifestPath = filePath
});
}
catch (Exception e)
{
logger.WriteLine($"... {arn}: Skipped, error inspecting layer. {e.Message}");
}
}
return(info);
}
/// <summary>
/// Search for the CloudFormation resources that references the app bundle sent to S3 and update them.
/// </summary>
/// <param name="templateBody"></param>
/// <param name="s3Bucket"></param>
/// <param name="s3Key"></param>
/// <returns></returns>
public static string UpdateCodeLocationInTemplate(string templateBody, string s3Bucket, string s3Key)
{
switch (LambdaUtilities.DetermineTemplateFormat(templateBody))
{
case TemplateFormat.Json:
return(UpdateCodeLocationInJsonTemplate(templateBody, s3Bucket, s3Key));
case TemplateFormat.Yaml:
return(UpdateCodeLocationInYamlTemplate(templateBody, s3Bucket, s3Key));
default:
throw new LambdaToolsException("Unable to determine template file format", LambdaToolsException.LambdaErrorCode.ServerlessTemplateParseError);
}
}
/// <summary>
/// Execute the dotnet publish command and zip up the resulting publish folder.
/// </summary>
/// <param name="defaults"></param>
/// <param name="logger"></param>
/// <param name="workingDirectory"></param>
/// <param name="projectLocation"></param>
/// <param name="targetFramework"></param>
/// <param name="configuration"></param>
/// <param name="msbuildParameters"></param>
/// <param name="disableVersionCheck"></param>
/// <param name="publishLocation"></param>
/// <param name="zipArchivePath"></param>
public static bool CreateApplicationBundle(LambdaToolsDefaults defaults, IToolLogger logger, string workingDirectory,
string projectLocation, string configuration, string targetFramework, string msbuildParameters,
bool disableVersionCheck, LayerPackageInfo layerPackageInfo,
out string publishLocation, ref string zipArchivePath)
{
LogDeprecationMessagesIfNecessary(logger, targetFramework);
if (string.IsNullOrEmpty(configuration))
{
configuration = LambdaConstants.DEFAULT_BUILD_CONFIGURATION;
}
var computedProjectLocation = Utilities.DetermineProjectLocation(workingDirectory, projectLocation);
var lambdaRuntimePackageStoreManifestContent = LambdaUtilities.LoadPackageStoreManifest(logger, targetFramework);
var publishManifestPath = new List <string>();
if (!string.IsNullOrEmpty(lambdaRuntimePackageStoreManifestContent))
{
var tempFile = Path.GetTempFileName();
File.WriteAllText(tempFile, lambdaRuntimePackageStoreManifestContent);
publishManifestPath.Add(tempFile);
}
if (layerPackageInfo != null)
{
foreach (var info in layerPackageInfo.Items)
{
publishManifestPath.Add(info.ManifestPath);
}
}
var cli = new LambdaDotNetCLIWrapper(logger, workingDirectory);
publishLocation = Utilities.DeterminePublishLocation(workingDirectory, projectLocation, configuration, targetFramework);
logger?.WriteLine("Executing publish command");
if (cli.Publish(defaults, projectLocation, publishLocation, targetFramework, configuration, msbuildParameters, publishManifestPath) != 0)
{
return(false);
}
var buildLocation = Utilities.DetermineBuildLocation(workingDirectory, projectLocation, configuration, targetFramework);
// This is here for legacy reasons. Some older versions of the dotnet CLI were not
// copying the deps.json file into the publish folder.
foreach (var file in Directory.GetFiles(buildLocation, "*.deps.json", SearchOption.TopDirectoryOnly))
{
var destinationPath = Path.Combine(publishLocation, Path.GetFileName(file));
if (!File.Exists(destinationPath))
{
File.Copy(file, destinationPath);
}
}
bool flattenRuntime = false;
var depsJsonTargetNode = GetDepsJsonTargetNode(logger, publishLocation);
// If there is no target node then this means the tool is being used on a future version of .NET Core
// then was available when the this tool was written. Go ahead and continue the deployment with warnings so the
// user can see if the future version will work.
if (depsJsonTargetNode != null && string.Equals(targetFramework, "netcoreapp1.0", StringComparison.OrdinalIgnoreCase))
{
// Make sure the project is not pulling in dependencies requiring a later version of .NET Core then the declared target framework
if (!ValidateDependencies(logger, targetFramework, depsJsonTargetNode, disableVersionCheck))
{
return(false);
}
// Flatten the runtime folder which reduces the package size by not including native dependencies
// for other platforms.
flattenRuntime = FlattenRuntimeFolder(logger, publishLocation, depsJsonTargetNode);
}
if (zipArchivePath == null)
{
zipArchivePath = Path.Combine(Directory.GetParent(publishLocation).FullName, new DirectoryInfo(computedProjectLocation).Name + ".zip");
}
zipArchivePath = Path.GetFullPath(zipArchivePath);
logger?.WriteLine($"Zipping publish folder {publishLocation} to {zipArchivePath}");
if (File.Exists(zipArchivePath))
{
File.Delete(zipArchivePath);
}
var zipArchiveParentDirectory = Path.GetDirectoryName(zipArchivePath);
if (!Directory.Exists(zipArchiveParentDirectory))
{
logger?.WriteLine($"Creating directory {zipArchiveParentDirectory}");
new DirectoryInfo(zipArchiveParentDirectory).Create();
}
BundleDirectory(zipArchivePath, publishLocation, flattenRuntime, logger);
return(true);
}
/// <summary>
/// Execute the dotnet store command on the provided package manifest
/// </summary>
/// <param name="defaults"></param>
/// <param name="projectLocation"></param>
/// <param name="outputLocation"></param>
/// <param name="targetFramework"></param>
/// <param name="packageManifest"></param>
/// <param name="enableOptimization"></param>
/// <returns></returns>
public int Store(LambdaToolsDefaults defaults, string projectLocation, string outputLocation, string targetFramework, string packageManifest, bool enableOptimization)
{
if (outputLocation == null)
{
throw new ArgumentNullException(nameof(outputLocation));
}
if (Directory.Exists(outputLocation))
{
try
{
Directory.Delete(outputLocation, true);
_logger?.WriteLine("Deleted previous publish folder");
}
catch (Exception e)
{
_logger?.WriteLine($"Warning unable to delete previous publish folder: {e.Message}");
}
}
var dotnetCLI = FindExecutableInPath("dotnet.exe");
if (dotnetCLI == null)
{
dotnetCLI = FindExecutableInPath("dotnet");
}
if (string.IsNullOrEmpty(dotnetCLI))
{
throw new Exception("Failed to locate dotnet CLI executable. Make sure the dotnet CLI is installed in the environment PATH.");
}
var fullProjectLocation = this._workingDirectory;
if (!string.IsNullOrEmpty(projectLocation))
{
fullProjectLocation = Utilities.DetermineProjectLocation(this._workingDirectory, projectLocation);
}
var fullPackageManifest = Path.Combine(fullProjectLocation, packageManifest);
_logger?.WriteLine($"... invoking 'dotnet store' for manifest {fullPackageManifest} into output directory {outputLocation}");
StringBuilder arguments = new StringBuilder("store");
if (!string.IsNullOrEmpty(outputLocation))
{
arguments.Append($" --output \"{outputLocation}\"");
}
if (!string.IsNullOrEmpty(targetFramework))
{
arguments.Append($" --framework \"{targetFramework}\"");
}
arguments.Append($" --manifest \"{fullPackageManifest}\"");
arguments.Append($" --runtime {LambdaUtilities.DetermineRuntimeParameter(targetFramework)}");
if (!enableOptimization)
{
arguments.Append(" --skip-optimization");
}
var psi = new ProcessStartInfo
{
FileName = dotnetCLI,
Arguments = arguments.ToString(),
WorkingDirectory = this._workingDirectory,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
var handler = (DataReceivedEventHandler)((o, e) =>
{
if (string.IsNullOrEmpty(e.Data))
{
return;
}
// Skip outputting this warning message as it adds a lot of noise to the output and is not actionable.
// Full warning message being skipped: message NETSDK1062:
// Unable to use package assets cache due to I/O error. This can occur when the same project is built
// more than once in parallel. Performance may be degraded, but the build result will not be impacted.
if (e.Data.Contains("message NETSDK1062"))
{
return;
}
_logger?.WriteLine("... store: " + e.Data);
});
int exitCode;
using (var proc = new Process())
{
proc.StartInfo = psi;
proc.Start();
proc.ErrorDataReceived += handler;
proc.OutputDataReceived += handler;
proc.BeginOutputReadLine();
proc.BeginErrorReadLine();
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
return(exitCode);
}
/// <summary>
/// Executes the dotnet publish command for the provided project
/// </summary>
/// <param name="defaults"></param>
/// <param name="projectLocation"></param>
/// <param name="outputLocation"></param>
/// <param name="targetFramework"></param>
/// <param name="configuration"></param>
/// <param name="msbuildParameters"></param>
/// <param name="deploymentTargetPackageStoreManifestContent"></param>
public int Publish(LambdaToolsDefaults defaults, string projectLocation, string outputLocation, string targetFramework, string configuration, string msbuildParameters, IList <string> publishManifests)
{
if (outputLocation == null)
{
throw new ArgumentNullException(nameof(outputLocation));
}
if (Directory.Exists(outputLocation))
{
try
{
Directory.Delete(outputLocation, true);
_logger?.WriteLine("Deleted previous publish folder");
}
catch (Exception e)
{
_logger?.WriteLine($"Warning unable to delete previous publish folder: {e.Message}");
}
}
_logger?.WriteLine($"... invoking 'dotnet publish', working folder '{outputLocation}'");
var dotnetCLI = FindExecutableInPath("dotnet.exe");
if (dotnetCLI == null)
{
dotnetCLI = FindExecutableInPath("dotnet");
}
if (string.IsNullOrEmpty(dotnetCLI))
{
throw new Exception("Failed to locate dotnet CLI executable. Make sure the dotnet CLI is installed in the environment PATH.");
}
var fullProjectLocation = this._workingDirectory;
if (!string.IsNullOrEmpty(projectLocation))
{
fullProjectLocation = Utilities.DetermineProjectLocation(this._workingDirectory, projectLocation);
}
StringBuilder arguments = new StringBuilder("publish");
if (!string.IsNullOrEmpty(projectLocation))
{
arguments.Append($" \"{fullProjectLocation}\"");
}
if (!string.IsNullOrEmpty(outputLocation))
{
arguments.Append($" --output \"{outputLocation}\"");
}
if (!string.IsNullOrEmpty(configuration))
{
arguments.Append($" --configuration \"{configuration}\"");
}
if (!string.IsNullOrEmpty(targetFramework))
{
arguments.Append($" --framework \"{targetFramework}\"");
}
if (!string.IsNullOrEmpty(msbuildParameters))
{
arguments.Append($" {msbuildParameters}");
}
if (!string.Equals("netcoreapp1.0", targetFramework, StringComparison.OrdinalIgnoreCase))
{
arguments.Append(" /p:GenerateRuntimeConfigurationFiles=true");
// Define an action to set the runtime and self-contained switches.
var applyRuntimeSwitchAction = (Action)(() =>
{
if (msbuildParameters == null ||
msbuildParameters.IndexOf("--runtime", StringComparison.InvariantCultureIgnoreCase) == -1)
{
arguments.Append($" --runtime {LambdaUtilities.DetermineRuntimeParameter(targetFramework)}");
}
if (msbuildParameters == null ||
msbuildParameters.IndexOf("--self-contained", StringComparison.InvariantCultureIgnoreCase) == -1)
{
arguments.Append(" --self-contained false ");
}
});
// This is here to not change existing behavior for the 2.0 and 2.1 runtimes. For those runtimes if
// cshtml files are being used we need to support that cshtml being compiled at runtime. In order to do that we
// need to not turn PreserveCompilationContext which provides reference assemblies to the runtime
// compilation and not set a runtime.
//
// If there are no cshtml then disable PreserveCompilationContext to reduce package size and continue
// to use the same runtime identifier that we used when those runtimes were launched.
if (new string[] { "netcoreapp2.0", "netcoreapp2.1" }.Contains(targetFramework))
{
if (Directory.GetFiles(fullProjectLocation, "*.cshtml", SearchOption.AllDirectories).Length == 0)
{
applyRuntimeSwitchAction();
if (string.IsNullOrEmpty(msbuildParameters) ||
!msbuildParameters.Contains("PreserveCompilationContext"))
{
_logger?.WriteLine("... Disabling compilation context to reduce package size. If compilation context is needed pass in the \"/p:PreserveCompilationContext=false\" switch.");
arguments.Append(" /p:PreserveCompilationContext=false");
}
}
}
else
{
applyRuntimeSwitchAction();
}
// If we have a manifest of packages already deploy in target deployment environment then write it to disk and add the
// command line switch
if (publishManifests != null && publishManifests.Count > 0)
{
foreach (var manifest in publishManifests)
{
arguments.Append($" --manifest \"{manifest}\"");
}
}
}
// echo the full dotnet command for debug
_logger?.WriteLine($"... dotnet {arguments}");
var psi = new ProcessStartInfo
{
FileName = dotnetCLI,
Arguments = arguments.ToString(),
WorkingDirectory = this._workingDirectory,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
var handler = (DataReceivedEventHandler)((o, e) =>
{
if (string.IsNullOrEmpty(e.Data))
{
return;
}
_logger?.WriteLine("... publish: " + e.Data);
});
int exitCode;
using (var proc = new Process())
{
proc.StartInfo = psi;
proc.Start();
proc.ErrorDataReceived += handler;
proc.OutputDataReceived += handler;
proc.BeginOutputReadLine();
proc.BeginErrorReadLine();
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
if (exitCode == 0)
{
ProcessAdditionalFiles(defaults, outputLocation);
var chmodPath = FindExecutableInPath("chmod");
var touchPath = FindExecutableInPath("touch");
if (!string.IsNullOrEmpty(chmodPath) && File.Exists(chmodPath))
{
// as we are not invoking through a shell, which would handle
// wildcard expansion for us, we need to invoke per-file
var files = Directory.GetFiles(outputLocation, "*", SearchOption.TopDirectoryOnly);
foreach (var file in files)
{
var filename = Path.GetFileName(file);
ApplyCommand(outputLocation, chmodPath, $"+rx \"{filename}\"", handler);
ApplyCommand(outputLocation, touchPath, $"-a -m -t 201512180130.09 \"{filename}\"", handler);
}
}
}
return(exitCode);
}
/// <summary>
/// Execute the dotnet publish command and zip up the resulting publish folder.
/// </summary>
/// <param name="defaults"></param>
/// <param name="logger"></param>
/// <param name="workingDirectory"></param>
/// <param name="projectLocation"></param>
/// <param name="targetFramework"></param>
/// <param name="configuration"></param>
/// <param name="msbuildParameters"></param>
/// <param name="disableVersionCheck"></param>
/// <param name="publishLocation"></param>
/// <param name="zipArchivePath"></param>
public static bool CreateApplicationBundle(LambdaToolsDefaults defaults, IToolLogger logger, string workingDirectory,
string projectLocation, string configuration, string targetFramework, string msbuildParameters, bool disableVersionCheck,
out string publishLocation, ref string zipArchivePath)
{
string lambdaRuntimePackageStoreManifestContent = null;
var computedProjectLocation = Utilities.DetermineProjectLocation(workingDirectory, projectLocation);
var packageStoreManifest = LambdaUtilities.LoadPackageStoreManifest(logger, targetFramework);
if (!disableVersionCheck)
{
LambdaUtilities.ValidateMicrosoftAspNetCoreAllReferenceFromProjectPath(logger, targetFramework, packageStoreManifest, computedProjectLocation);
}
var cli = new LambdaDotNetCLIWrapper(logger, workingDirectory);
publishLocation = Utilities.DeterminePublishLocation(workingDirectory, projectLocation, configuration, targetFramework);
logger?.WriteLine("Executing publish command");
if (cli.Publish(defaults, projectLocation, publishLocation, targetFramework, configuration, msbuildParameters, lambdaRuntimePackageStoreManifestContent) != 0)
{
return(false);
}
var buildLocation = Utilities.DetermineBuildLocation(workingDirectory, projectLocation, configuration, targetFramework);
// This is here for legacy reasons. Some older versions of the dotnet CLI were not
// copying the deps.json file into the publish folder.
foreach (var file in Directory.GetFiles(buildLocation, "*.deps.json", SearchOption.TopDirectoryOnly))
{
var destinationPath = Path.Combine(publishLocation, Path.GetFileName(file));
if (!File.Exists(destinationPath))
{
File.Copy(file, destinationPath);
}
}
bool flattenRuntime = false;
var depsJsonTargetNode = GetDepsJsonTargetNode(logger, publishLocation);
// If there is no target node then this means the tool is being used on a future version of .NET Core
// then was available when the this tool was written. Go ahead and continue the deployment with warnings so the
// user can see if the future version will work.
if (depsJsonTargetNode != null && string.Equals(targetFramework, "netcoreapp1.0", StringComparison.OrdinalIgnoreCase))
{
// Make sure the project is not pulling in dependencies requiring a later version of .NET Core then the declared target framework
if (!ValidateDependencies(logger, targetFramework, depsJsonTargetNode, disableVersionCheck))
{
return(false);
}
// Flatten the runtime folder which reduces the package size by not including native dependencies
// for other platforms.
flattenRuntime = FlattenRuntimeFolder(logger, publishLocation, depsJsonTargetNode);
}
if (zipArchivePath == null)
{
zipArchivePath = Path.Combine(Directory.GetParent(publishLocation).FullName, new DirectoryInfo(computedProjectLocation).Name + ".zip");
}
zipArchivePath = Path.GetFullPath(zipArchivePath);
logger?.WriteLine($"Zipping publish folder {publishLocation} to {zipArchivePath}");
if (File.Exists(zipArchivePath))
{
File.Delete(zipArchivePath);
}
var zipArchiveParentDirectory = Path.GetDirectoryName(zipArchivePath);
if (!Directory.Exists(zipArchiveParentDirectory))
{
logger?.WriteLine($"Creating directory {zipArchiveParentDirectory}");
new DirectoryInfo(zipArchiveParentDirectory).Create();
}
#if NETCORE
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
BundleWithDotNetCompression(zipArchivePath, publishLocation, flattenRuntime, logger);
}
else
{
// Use the native zip utility if it exist which will maintain linux/osx file permissions
var zipCLI = LambdaDotNetCLIWrapper.FindExecutableInPath("zip");
if (!string.IsNullOrEmpty(zipCLI))
{
BundleWithZipCLI(zipCLI, zipArchivePath, publishLocation, flattenRuntime, logger);
}
else
{
throw new LambdaToolsException("Failed to find the \"zip\" utility program in path. This program is required to maintain Linux file permissions in the zip archive.", LambdaToolsException.LambdaErrorCode.FailedToFindZipProgram);
}
}
#else
BundleWithDotNetCompression(zipArchivePath, publishLocation, flattenRuntime, logger);
#endif
return(true);
}
