public async Task ShouldCorrectlyPackageNodeDirectoryLambdaWithDependency()
{
var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "NodeLambda1");
var template = Path.Combine(templateDir, "template-complex.yaml");
var modulesDirectory = Path.Combine(templateDir, "Lambda", "node_modules");
this.SetupMocks(template);
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
this.Logger,
new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
new OSInfo());
await packager.ProcessTemplate(template, workingDirectory);
// Verify by checking messages output by the zip library
this.Logger.VerboseMessages.Should().Contain(
new[] { "Adding my_lambda.js", "Adding other.js", "Adding node_modules/mylibrary/libfile.js" },
"the function itself and its dependency should be in right places in zip");
// Check vendor directory does not exist (was temporarily created to make the package)
Directory.Exists(modulesDirectory).Should().BeFalse("node_modules directory transient to create package");
}
public async Task ShouldCorrectlyPackageRubySingleFileLambdaWithLocalAndExternalDependencies()
{
var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "RubyLambda2");
var template = Path.Combine(templateDir, "template.yaml");
var vendorDirectory = Path.Combine(templateDir, "Lambda", "vendor");
this.SetupMocks(template);
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
this.Logger,
new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
new OSInfo());
await packager.ProcessTemplate(template, workingDirectory);
// Verify by checking messages output by the zip library
this.Logger.VerboseMessages.Should().Contain(
new[]
{
"Adding my_lambda.rb", "Adding vendor/bundle/ruby/2.7.0/cache/mylibrary/libfile.rb",
"Adding vendor/bundle/ruby/2.7.0/cache/local_lib/local_lib.rb"
},
"the function itself and its dependency should be in right places in zip");
// Check vendor directory does not exist (was temporarily created to make the package)
Directory.Exists(vendorDirectory).Should().BeTrue("vendor directory existed prior to packaging");
}
public async Task ShouldNotUploadNewVersionOfDirectoryArtifactWhenHashesMatch()
{
using var templateDir = this.deepNestedStack;
// Hash of lambda directory content before zipping
// Zips are not idempotent - fields e.g. timestamps in central directory change with successive zips of the same content.
var directoryHash = new DirectoryInfo(Path.Combine(templateDir, "lambdacomplex")).MD5();
var template = Path.Combine(templateDir, "base-stack.json");
var projectId = S3Util.GenerateProjectId(template);
var logger = new TestLogger(this.output);
var mockSts = TestHelpers.GetSTSMock();
var mockS3 = TestHelpers.GetS3ClientWithBucketMock();
var mockContext = new Mock <IPSCloudFormationContext>();
mockContext.Setup(c => c.Logger).Returns(logger);
mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1);
mockS3.SetupSequence(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default)).ReturnsAsync(
new ListObjectsV2Response
{
S3Objects = new List <S3Object>
{
new S3Object
{
BucketName = "test-bucket",
Key = $"lambdacomplex-{projectId}-0000.zip"
}
}
}).ReturnsAsync(this.fileNotFound).ReturnsAsync(this.fileNotFound);
mockS3.Setup(s3 => s3.GetObjectMetadataAsync(It.IsAny <GetObjectMetadataRequest>(), default)).ReturnsAsync(
() =>
{
var resp = new GetObjectMetadataResponse();
resp.Metadata.Add(S3Util.PackagerHashKey, directoryHash);
return(resp);
});
var mockClientFactory = new Mock <IPSAwsClientFactory>();
mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object);
mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object);
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
logger,
new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null),
new OSInfo());
var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory);
this.output.WriteLine(string.Empty);
this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath));
// Three objects should have been uploaded to S3
mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(2));
}
public async Task ShouldCorrectlyPackagePythonSingleFileLambdaWithRequirementsTxt(string platform)
{
var templateDir = Path.Combine(
this.LambdaDependencies.FullPath,
platform == "Windows" ? "PythonLambda" : "PythonLambdaLinux");
var template = Path.Combine(templateDir, "template.yaml");
this.SetupMocks(template);
var mockOSInfo = new Mock <IOSInfo>();
mockOSInfo.Setup(i => i.OSPlatform).Returns(platform == "Windows" ? OSPlatform.Windows : OSPlatform.Linux);
// Mock the virtualenv
Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv"));
var moduleMetadata = new string[] { "METADATA.txt", "RECORD.txt" };
// Fix the fact we have to put a ".txt" extension on these files to get round namespace rules in the embedded resources.
foreach (var file in Directory.EnumerateFiles(templateDir, "*.txt", SearchOption.AllDirectories).Where(f => moduleMetadata.Contains(Path.GetFileName(f))))
{
File.Move(file, Path.Combine(Path.GetDirectoryName(file), Path.GetFileNameWithoutExtension(file)));
}
// Place a requriements.txt
File.WriteAllText(Path.Combine(templateDir, "Lambda", "requirements.txt"), "mylibrary\nboto3");
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
this.Logger,
new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
mockOSInfo.Object);
await packager.ProcessTemplate(template, workingDirectory);
this.Logger.VerboseMessages.Should().Contain(
new[]
{
"Adding mylibrary/", "Adding my_lambda.py", "Adding standalone_module.py",
"Adding mylibrary/__init__.py",
"Package 'boto3' will not be included because it exists by default in the AWS execution environment."
},
"the function itself and its dependency should be in right places in zip");
}
public async Task ShouldCorrectlyPackagePythonSingleFileLambdaWithoutDependency()
{
var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "PythonLambda");
var template = Path.Combine(templateDir, "template.yaml");
this.SetupMocks(template);
// Mock the virtualenv
Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv"));
// Remove the materialized lambda_dependencies so this is seen as a single file lambda
foreach (var dep in Directory.EnumerateFiles(
templateDir,
"lambda-dependencies.*",
SearchOption.AllDirectories))
{
File.Delete(dep);
}
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
this.Logger,
new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
new OSInfo());
await packager.ProcessTemplate(template, workingDirectory);
// Verify by checking messages output by the zip library
this.Logger.VerboseMessages.Should().Contain(
"Adding my_lambda.py",
"the function itself and its dependency should be in right places in zip");
this.Logger.VerboseMessages.Should().NotContain(
new[] { "Adding other.py", "Adding mylibrary/__init__.py" },
"lambda is a single script");
this.Logger.VerboseMessages.Should().NotContain(
"*__pycache__*",
"__pycache__ should not be included in lambda packages");
}
public async Task ShouldCorrectlyPackagePythonDirectoryLambdaWithDependency(string platform)
{
var templateDir = Path.Combine(
this.LambdaDependencies.FullPath,
platform == "Windows" ? "PythonLambda" : "PythonLambdaLinux");
var template = Path.Combine(templateDir, "template-complex.yaml");
this.SetupMocks(template);
var mockOSInfo = new Mock <IOSInfo>();
mockOSInfo.Setup(i => i.OSPlatform).Returns(platform == "Windows" ? OSPlatform.Windows : OSPlatform.OSX);
// Mock the virtualenv
Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv"));
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
this.Logger,
new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
mockOSInfo.Object);
await packager.ProcessTemplate(template, workingDirectory);
// Verify by checking messages output by the zip library
this.Logger.VerboseMessages.Should().Contain(
new[]
{
"Adding my_lambda.py", "Adding other.py", "Adding mylibrary/__init__.py",
"Adding standalone_module.py"
},
"the function itself and its dependency should be in right places in zip");
this.Logger.VerboseMessages.Should().NotContain(
"*__pycache__*",
"__pycache__ should not be included in lambda packages");
}
public async Task ShouldUploadAllArtifactsWhenNoneExistInS3()
{
var logger = new TestLogger(this.output);
var mockSts = TestHelpers.GetSTSMock();
var mockS3 = TestHelpers.GetS3ClientWithBucketMock();
mockS3.Setup(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default))
.ReturnsAsync(this.fileNotFound);
var mockClientFactory = new Mock <IPSAwsClientFactory>();
mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object);
mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object);
var mockContext = new Mock <IPSCloudFormationContext>();
mockContext.Setup(c => c.Logger).Returns(logger);
mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1);
using var workingDirectory = new TempDirectory();
var template = Path.Combine(this.deepNestedStack, "base-stack.json");
var packager = new PackagerUtils(
new TestPathResolver(),
logger,
new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null),
new OSInfo());
var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory);
this.output.WriteLine(string.Empty);
this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath));
// Three objects should have been uploaded to S3
mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(3));
}
public async Task ShouldNotUploadNewVersionOfTemplateArtifactWhenHashesMatch()
{
var templateDir = this.deepNestedStack;
var template = Path.Combine(templateDir, "base-stack.json");
var projectId = S3Util.GenerateProjectId(template);
var logger = new TestLogger(this.output);
var mockSts = TestHelpers.GetSTSMock();
var mockS3 = TestHelpers.GetS3ClientWithBucketMock();
var mockContext = new Mock <IPSCloudFormationContext>();
mockContext.Setup(c => c.Logger).Returns(logger);
mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1);
mockS3.SetupSequence(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default))
.ReturnsAsync(this.fileNotFound)
.ReturnsAsync(
new ListObjectsV2Response
{
S3Objects = new List <S3Object>
{
new S3Object
{
BucketName = "test-bucket",
Key = $"sub-nested-2-{projectId}-0000.json"
}
}
}).ReturnsAsync(this.fileNotFound);
mockS3.Setup(s3 => s3.GetObjectMetadataAsync(It.IsAny <GetObjectMetadataRequest>(), default)).ReturnsAsync(
() =>
{
var resp = new GetObjectMetadataResponse();
resp.Metadata.Add(S3Util.PackagerHashKey, GetModifiedTemplateHash());
return(resp);
});
var mockClientFactory = new Mock <IPSAwsClientFactory>();
mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object);
mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object);
using var workingDirectory = new TempDirectory();
var packager = new PackagerUtils(
new TestPathResolver(),
logger,
new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null),
new OSInfo());
var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory);
this.output.WriteLine(string.Empty);
this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath));
// Three objects should have been uploaded to S3
mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(2));
// Bit hacky, but we need to know the hash of the template after modification.
// Different on Windows and Linux due to line endings.
string GetModifiedTemplateHash()
{
var re = new Regex(@"sub-nested-2\.json.*Hash: (?<hash>[0-9a-f]+)");
var logLine = logger.DebugMessages.FirstOrDefault(line => re.IsMatch(line));
if (logLine == null)
{
return("0");
}
var mc = re.Match(logLine);
return(mc.Groups["hash"].Value);
}
}
