public async Task ShouldCorrectlyPackageNodeDirectoryLambdaWithDependency() { var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "NodeLambda1"); var template = Path.Combine(templateDir, "template-complex.yaml"); var modulesDirectory = Path.Combine(templateDir, "Lambda", "node_modules"); this.SetupMocks(template); using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), this.Logger, new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null), new OSInfo()); await packager.ProcessTemplate(template, workingDirectory); // Verify by checking messages output by the zip library this.Logger.VerboseMessages.Should().Contain( new[] { "Adding my_lambda.js", "Adding other.js", "Adding node_modules/mylibrary/libfile.js" }, "the function itself and its dependency should be in right places in zip"); // Check vendor directory does not exist (was temporarily created to make the package) Directory.Exists(modulesDirectory).Should().BeFalse("node_modules directory transient to create package"); }
public async Task ShouldCorrectlyPackageRubySingleFileLambdaWithLocalAndExternalDependencies() { var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "RubyLambda2"); var template = Path.Combine(templateDir, "template.yaml"); var vendorDirectory = Path.Combine(templateDir, "Lambda", "vendor"); this.SetupMocks(template); using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), this.Logger, new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null), new OSInfo()); await packager.ProcessTemplate(template, workingDirectory); // Verify by checking messages output by the zip library this.Logger.VerboseMessages.Should().Contain( new[] { "Adding my_lambda.rb", "Adding vendor/bundle/ruby/2.7.0/cache/mylibrary/libfile.rb", "Adding vendor/bundle/ruby/2.7.0/cache/local_lib/local_lib.rb" }, "the function itself and its dependency should be in right places in zip"); // Check vendor directory does not exist (was temporarily created to make the package) Directory.Exists(vendorDirectory).Should().BeTrue("vendor directory existed prior to packaging"); }
public async Task ShouldNotUploadNewVersionOfDirectoryArtifactWhenHashesMatch() { using var templateDir = this.deepNestedStack; // Hash of lambda directory content before zipping // Zips are not idempotent - fields e.g. timestamps in central directory change with successive zips of the same content. var directoryHash = new DirectoryInfo(Path.Combine(templateDir, "lambdacomplex")).MD5(); var template = Path.Combine(templateDir, "base-stack.json"); var projectId = S3Util.GenerateProjectId(template); var logger = new TestLogger(this.output); var mockSts = TestHelpers.GetSTSMock(); var mockS3 = TestHelpers.GetS3ClientWithBucketMock(); var mockContext = new Mock <IPSCloudFormationContext>(); mockContext.Setup(c => c.Logger).Returns(logger); mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1); mockS3.SetupSequence(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default)).ReturnsAsync( new ListObjectsV2Response { S3Objects = new List <S3Object> { new S3Object { BucketName = "test-bucket", Key = $"lambdacomplex-{projectId}-0000.zip" } } }).ReturnsAsync(this.fileNotFound).ReturnsAsync(this.fileNotFound); mockS3.Setup(s3 => s3.GetObjectMetadataAsync(It.IsAny <GetObjectMetadataRequest>(), default)).ReturnsAsync( () => { var resp = new GetObjectMetadataResponse(); resp.Metadata.Add(S3Util.PackagerHashKey, directoryHash); return(resp); }); var mockClientFactory = new Mock <IPSAwsClientFactory>(); mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object); mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object); using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), logger, new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null), new OSInfo()); var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory); this.output.WriteLine(string.Empty); this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath)); // Three objects should have been uploaded to S3 mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(2)); }
public async Task ShouldCorrectlyPackagePythonSingleFileLambdaWithRequirementsTxt(string platform) { var templateDir = Path.Combine( this.LambdaDependencies.FullPath, platform == "Windows" ? "PythonLambda" : "PythonLambdaLinux"); var template = Path.Combine(templateDir, "template.yaml"); this.SetupMocks(template); var mockOSInfo = new Mock <IOSInfo>(); mockOSInfo.Setup(i => i.OSPlatform).Returns(platform == "Windows" ? OSPlatform.Windows : OSPlatform.Linux); // Mock the virtualenv Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv")); var moduleMetadata = new string[] { "METADATA.txt", "RECORD.txt" }; // Fix the fact we have to put a ".txt" extension on these files to get round namespace rules in the embedded resources. foreach (var file in Directory.EnumerateFiles(templateDir, "*.txt", SearchOption.AllDirectories).Where(f => moduleMetadata.Contains(Path.GetFileName(f)))) { File.Move(file, Path.Combine(Path.GetDirectoryName(file), Path.GetFileNameWithoutExtension(file))); } // Place a requriements.txt File.WriteAllText(Path.Combine(templateDir, "Lambda", "requirements.txt"), "mylibrary\nboto3"); using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), this.Logger, new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null), mockOSInfo.Object); await packager.ProcessTemplate(template, workingDirectory); this.Logger.VerboseMessages.Should().Contain( new[] { "Adding mylibrary/", "Adding my_lambda.py", "Adding standalone_module.py", "Adding mylibrary/__init__.py", "Package 'boto3' will not be included because it exists by default in the AWS execution environment." }, "the function itself and its dependency should be in right places in zip"); }
public async Task ShouldCorrectlyPackagePythonSingleFileLambdaWithoutDependency() { var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "PythonLambda"); var template = Path.Combine(templateDir, "template.yaml"); this.SetupMocks(template); // Mock the virtualenv Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv")); // Remove the materialized lambda_dependencies so this is seen as a single file lambda foreach (var dep in Directory.EnumerateFiles( templateDir, "lambda-dependencies.*", SearchOption.AllDirectories)) { File.Delete(dep); } using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), this.Logger, new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null), new OSInfo()); await packager.ProcessTemplate(template, workingDirectory); // Verify by checking messages output by the zip library this.Logger.VerboseMessages.Should().Contain( "Adding my_lambda.py", "the function itself and its dependency should be in right places in zip"); this.Logger.VerboseMessages.Should().NotContain( new[] { "Adding other.py", "Adding mylibrary/__init__.py" }, "lambda is a single script"); this.Logger.VerboseMessages.Should().NotContain( "*__pycache__*", "__pycache__ should not be included in lambda packages"); }
public async Task ShouldCorrectlyPackagePythonDirectoryLambdaWithDependency(string platform) { var templateDir = Path.Combine( this.LambdaDependencies.FullPath, platform == "Windows" ? "PythonLambda" : "PythonLambdaLinux"); var template = Path.Combine(templateDir, "template-complex.yaml"); this.SetupMocks(template); var mockOSInfo = new Mock <IOSInfo>(); mockOSInfo.Setup(i => i.OSPlatform).Returns(platform == "Windows" ? OSPlatform.Windows : OSPlatform.OSX); // Mock the virtualenv Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv")); using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), this.Logger, new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null), mockOSInfo.Object); await packager.ProcessTemplate(template, workingDirectory); // Verify by checking messages output by the zip library this.Logger.VerboseMessages.Should().Contain( new[] { "Adding my_lambda.py", "Adding other.py", "Adding mylibrary/__init__.py", "Adding standalone_module.py" }, "the function itself and its dependency should be in right places in zip"); this.Logger.VerboseMessages.Should().NotContain( "*__pycache__*", "__pycache__ should not be included in lambda packages"); }
public async Task ShouldUploadAllArtifactsWhenNoneExistInS3() { var logger = new TestLogger(this.output); var mockSts = TestHelpers.GetSTSMock(); var mockS3 = TestHelpers.GetS3ClientWithBucketMock(); mockS3.Setup(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default)) .ReturnsAsync(this.fileNotFound); var mockClientFactory = new Mock <IPSAwsClientFactory>(); mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object); mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object); var mockContext = new Mock <IPSCloudFormationContext>(); mockContext.Setup(c => c.Logger).Returns(logger); mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1); using var workingDirectory = new TempDirectory(); var template = Path.Combine(this.deepNestedStack, "base-stack.json"); var packager = new PackagerUtils( new TestPathResolver(), logger, new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null), new OSInfo()); var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory); this.output.WriteLine(string.Empty); this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath)); // Three objects should have been uploaded to S3 mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(3)); }
public async Task ShouldNotUploadNewVersionOfTemplateArtifactWhenHashesMatch() { var templateDir = this.deepNestedStack; var template = Path.Combine(templateDir, "base-stack.json"); var projectId = S3Util.GenerateProjectId(template); var logger = new TestLogger(this.output); var mockSts = TestHelpers.GetSTSMock(); var mockS3 = TestHelpers.GetS3ClientWithBucketMock(); var mockContext = new Mock <IPSCloudFormationContext>(); mockContext.Setup(c => c.Logger).Returns(logger); mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1); mockS3.SetupSequence(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default)) .ReturnsAsync(this.fileNotFound) .ReturnsAsync( new ListObjectsV2Response { S3Objects = new List <S3Object> { new S3Object { BucketName = "test-bucket", Key = $"sub-nested-2-{projectId}-0000.json" } } }).ReturnsAsync(this.fileNotFound); mockS3.Setup(s3 => s3.GetObjectMetadataAsync(It.IsAny <GetObjectMetadataRequest>(), default)).ReturnsAsync( () => { var resp = new GetObjectMetadataResponse(); resp.Metadata.Add(S3Util.PackagerHashKey, GetModifiedTemplateHash()); return(resp); }); var mockClientFactory = new Mock <IPSAwsClientFactory>(); mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object); mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object); using var workingDirectory = new TempDirectory(); var packager = new PackagerUtils( new TestPathResolver(), logger, new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null), new OSInfo()); var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory); this.output.WriteLine(string.Empty); this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath)); // Three objects should have been uploaded to S3 mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(2)); // Bit hacky, but we need to know the hash of the template after modification. // Different on Windows and Linux due to line endings. string GetModifiedTemplateHash() { var re = new Regex(@"sub-nested-2\.json.*Hash: (?<hash>[0-9a-f]+)"); var logLine = logger.DebugMessages.FirstOrDefault(line => re.IsMatch(line)); if (logLine == null) { return("0"); } var mc = re.Match(logLine); return(mc.Groups["hash"].Value); } }