Beispiel #1
0
        public async Task ShouldCorrectlyPackageNodeDirectoryLambdaWithDependency()
        {
            var templateDir      = Path.Combine(this.LambdaDependencies.FullPath, "NodeLambda1");
            var template         = Path.Combine(templateDir, "template-complex.yaml");
            var modulesDirectory = Path.Combine(templateDir, "Lambda", "node_modules");

            this.SetupMocks(template);

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                this.Logger,
                new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
                new OSInfo());

            await packager.ProcessTemplate(template, workingDirectory);

            // Verify by checking messages output by the zip library
            this.Logger.VerboseMessages.Should().Contain(
                new[] { "Adding my_lambda.js", "Adding other.js", "Adding node_modules/mylibrary/libfile.js" },
                "the function itself and its dependency should be in right places in zip");

            // Check vendor directory does not exist (was temporarily created to make the package)
            Directory.Exists(modulesDirectory).Should().BeFalse("node_modules directory transient to create package");
        }
Beispiel #2
0
        public async Task ShouldCorrectlyPackageRubySingleFileLambdaWithLocalAndExternalDependencies()
        {
            var templateDir     = Path.Combine(this.LambdaDependencies.FullPath, "RubyLambda2");
            var template        = Path.Combine(templateDir, "template.yaml");
            var vendorDirectory = Path.Combine(templateDir, "Lambda", "vendor");

            this.SetupMocks(template);

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                this.Logger,
                new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
                new OSInfo());

            await packager.ProcessTemplate(template, workingDirectory);

            // Verify by checking messages output by the zip library
            this.Logger.VerboseMessages.Should().Contain(
                new[]
            {
                "Adding my_lambda.rb", "Adding vendor/bundle/ruby/2.7.0/cache/mylibrary/libfile.rb",
                "Adding vendor/bundle/ruby/2.7.0/cache/local_lib/local_lib.rb"
            },
                "the function itself and its dependency should be in right places in zip");

            // Check vendor directory does not exist (was temporarily created to make the package)
            Directory.Exists(vendorDirectory).Should().BeTrue("vendor directory existed prior to packaging");
        }
        public async Task ShouldNotUploadNewVersionOfDirectoryArtifactWhenHashesMatch()
        {
            using var templateDir = this.deepNestedStack;
            // Hash of lambda directory content before zipping
            // Zips are not idempotent - fields e.g. timestamps in central directory change with successive zips of the same content.
            var directoryHash = new DirectoryInfo(Path.Combine(templateDir, "lambdacomplex")).MD5();
            var template      = Path.Combine(templateDir, "base-stack.json");
            var projectId     = S3Util.GenerateProjectId(template);
            var logger        = new TestLogger(this.output);
            var mockSts       = TestHelpers.GetSTSMock();
            var mockS3        = TestHelpers.GetS3ClientWithBucketMock();
            var mockContext   = new Mock <IPSCloudFormationContext>();

            mockContext.Setup(c => c.Logger).Returns(logger);
            mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1);
            mockS3.SetupSequence(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default)).ReturnsAsync(
                new ListObjectsV2Response
            {
                S3Objects = new List <S3Object>
                {
                    new S3Object
                    {
                        BucketName = "test-bucket",
                        Key        = $"lambdacomplex-{projectId}-0000.zip"
                    }
                }
            }).ReturnsAsync(this.fileNotFound).ReturnsAsync(this.fileNotFound);

            mockS3.Setup(s3 => s3.GetObjectMetadataAsync(It.IsAny <GetObjectMetadataRequest>(), default)).ReturnsAsync(
                () =>
            {
                var resp = new GetObjectMetadataResponse();

                resp.Metadata.Add(S3Util.PackagerHashKey, directoryHash);
                return(resp);
            });

            var mockClientFactory = new Mock <IPSAwsClientFactory>();

            mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object);
            mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object);

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                logger,
                new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null),
                new OSInfo());

            var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory);

            this.output.WriteLine(string.Empty);
            this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath));

            // Three objects should have been uploaded to S3
            mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(2));
        }
Beispiel #4
0
        public async Task ShouldCorrectlyPackagePythonSingleFileLambdaWithRequirementsTxt(string platform)
        {
            var templateDir = Path.Combine(
                this.LambdaDependencies.FullPath,
                platform == "Windows" ? "PythonLambda" : "PythonLambdaLinux");
            var template = Path.Combine(templateDir, "template.yaml");

            this.SetupMocks(template);

            var mockOSInfo = new Mock <IOSInfo>();

            mockOSInfo.Setup(i => i.OSPlatform).Returns(platform == "Windows" ? OSPlatform.Windows : OSPlatform.Linux);

            // Mock the virtualenv
            Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv"));

            var moduleMetadata = new string[] { "METADATA.txt", "RECORD.txt" };

            // Fix the fact we have to put a ".txt" extension on these files to get round namespace rules in the embedded resources.
            foreach (var file in Directory.EnumerateFiles(templateDir, "*.txt", SearchOption.AllDirectories).Where(f => moduleMetadata.Contains(Path.GetFileName(f))))
            {
                File.Move(file, Path.Combine(Path.GetDirectoryName(file), Path.GetFileNameWithoutExtension(file)));
            }

            // Place a requriements.txt
            File.WriteAllText(Path.Combine(templateDir, "Lambda", "requirements.txt"), "mylibrary\nboto3");

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                this.Logger,
                new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
                mockOSInfo.Object);

            await packager.ProcessTemplate(template, workingDirectory);

            this.Logger.VerboseMessages.Should().Contain(
                new[]
            {
                "Adding mylibrary/", "Adding my_lambda.py", "Adding standalone_module.py",
                "Adding mylibrary/__init__.py",
                "Package 'boto3' will not be included because it exists by default in the AWS execution environment."
            },
                "the function itself and its dependency should be in right places in zip");
        }
Beispiel #5
0
        public async Task ShouldCorrectlyPackagePythonSingleFileLambdaWithoutDependency()
        {
            var templateDir = Path.Combine(this.LambdaDependencies.FullPath, "PythonLambda");
            var template    = Path.Combine(templateDir, "template.yaml");

            this.SetupMocks(template);

            // Mock the virtualenv
            Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv"));

            // Remove the materialized lambda_dependencies so this is seen as a single file lambda
            foreach (var dep in Directory.EnumerateFiles(
                         templateDir,
                         "lambda-dependencies.*",
                         SearchOption.AllDirectories))
            {
                File.Delete(dep);
            }

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                this.Logger,
                new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
                new OSInfo());

            await packager.ProcessTemplate(template, workingDirectory);

            // Verify by checking messages output by the zip library
            this.Logger.VerboseMessages.Should().Contain(
                "Adding my_lambda.py",
                "the function itself and its dependency should be in right places in zip");

            this.Logger.VerboseMessages.Should().NotContain(
                new[] { "Adding other.py", "Adding mylibrary/__init__.py" },
                "lambda is a single script");

            this.Logger.VerboseMessages.Should().NotContain(
                "*__pycache__*",
                "__pycache__ should not be included in lambda packages");
        }
Beispiel #6
0
        public async Task ShouldCorrectlyPackagePythonDirectoryLambdaWithDependency(string platform)
        {
            var templateDir = Path.Combine(
                this.LambdaDependencies.FullPath,
                platform == "Windows" ? "PythonLambda" : "PythonLambdaLinux");
            var template = Path.Combine(templateDir, "template-complex.yaml");

            this.SetupMocks(template);

            var mockOSInfo = new Mock <IOSInfo>();

            mockOSInfo.Setup(i => i.OSPlatform).Returns(platform == "Windows" ? OSPlatform.Windows : OSPlatform.OSX);

            // Mock the virtualenv
            Environment.SetEnvironmentVariable("VIRTUAL_ENV", Path.Combine(templateDir, "venv"));

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                this.Logger,
                new S3Util(this.ClientFactory, this.Context, template, "test-bucket", null, null),
                mockOSInfo.Object);

            await packager.ProcessTemplate(template, workingDirectory);

            // Verify by checking messages output by the zip library
            this.Logger.VerboseMessages.Should().Contain(
                new[]
            {
                "Adding my_lambda.py", "Adding other.py", "Adding mylibrary/__init__.py",
                "Adding standalone_module.py"
            },
                "the function itself and its dependency should be in right places in zip");
            this.Logger.VerboseMessages.Should().NotContain(
                "*__pycache__*",
                "__pycache__ should not be included in lambda packages");
        }
        public async Task ShouldUploadAllArtifactsWhenNoneExistInS3()
        {
            var logger  = new TestLogger(this.output);
            var mockSts = TestHelpers.GetSTSMock();
            var mockS3  = TestHelpers.GetS3ClientWithBucketMock();

            mockS3.Setup(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default))
            .ReturnsAsync(this.fileNotFound);

            var mockClientFactory = new Mock <IPSAwsClientFactory>();

            mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object);
            mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object);

            var mockContext = new Mock <IPSCloudFormationContext>();

            mockContext.Setup(c => c.Logger).Returns(logger);
            mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1);

            using var workingDirectory = new TempDirectory();

            var template = Path.Combine(this.deepNestedStack, "base-stack.json");

            var packager = new PackagerUtils(
                new TestPathResolver(),
                logger,
                new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null),
                new OSInfo());

            var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory);

            this.output.WriteLine(string.Empty);
            this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath));

            // Three objects should have been uploaded to S3
            mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(3));
        }
        public async Task ShouldNotUploadNewVersionOfTemplateArtifactWhenHashesMatch()
        {
            var templateDir = this.deepNestedStack;
            var template    = Path.Combine(templateDir, "base-stack.json");
            var projectId   = S3Util.GenerateProjectId(template);
            var logger      = new TestLogger(this.output);
            var mockSts     = TestHelpers.GetSTSMock();
            var mockS3      = TestHelpers.GetS3ClientWithBucketMock();
            var mockContext = new Mock <IPSCloudFormationContext>();

            mockContext.Setup(c => c.Logger).Returns(logger);
            mockContext.Setup(c => c.Region).Returns(RegionEndpoint.EUWest1);

            mockS3.SetupSequence(s3 => s3.ListObjectsV2Async(It.IsAny <ListObjectsV2Request>(), default))
            .ReturnsAsync(this.fileNotFound)
            .ReturnsAsync(
                new ListObjectsV2Response
            {
                S3Objects = new List <S3Object>
                {
                    new S3Object
                    {
                        BucketName = "test-bucket",
                        Key        = $"sub-nested-2-{projectId}-0000.json"
                    }
                }
            }).ReturnsAsync(this.fileNotFound);

            mockS3.Setup(s3 => s3.GetObjectMetadataAsync(It.IsAny <GetObjectMetadataRequest>(), default)).ReturnsAsync(
                () =>
            {
                var resp = new GetObjectMetadataResponse();

                resp.Metadata.Add(S3Util.PackagerHashKey, GetModifiedTemplateHash());
                return(resp);
            });

            var mockClientFactory = new Mock <IPSAwsClientFactory>();

            mockClientFactory.Setup(f => f.CreateS3Client()).Returns(mockS3.Object);
            mockClientFactory.Setup(f => f.CreateSTSClient()).Returns(mockSts.Object);

            using var workingDirectory = new TempDirectory();

            var packager = new PackagerUtils(
                new TestPathResolver(),
                logger,
                new S3Util(mockClientFactory.Object, mockContext.Object, template, "test-bucket", null, null),
                new OSInfo());

            var outputTemplatePath = await packager.ProcessTemplate(template, workingDirectory);

            this.output.WriteLine(string.Empty);
            this.output.WriteLine(await File.ReadAllTextAsync(outputTemplatePath));

            // Three objects should have been uploaded to S3
            mockS3.Verify(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>(), default), Times.Exactly(2));

            // Bit hacky, but we need to know the hash of the template after modification.
            // Different on Windows and Linux due to line endings.
            string GetModifiedTemplateHash()
            {
                var re = new Regex(@"sub-nested-2\.json.*Hash: (?<hash>[0-9a-f]+)");

                var logLine = logger.DebugMessages.FirstOrDefault(line => re.IsMatch(line));

                if (logLine == null)
                {
                    return("0");
                }

                var mc = re.Match(logLine);

                return(mc.Groups["hash"].Value);
            }
        }