internal GrpcBenchmarkStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
        {
            var vpc = new Vpc(this, "vpc", new VpcProps
            {
                MaxAzs              = 1,
                NatGateways         = 0,
                SubnetConfiguration = new[] { new SubnetConfiguration {
                                                  Name = "public", SubnetType = SubnetType.PUBLIC
                                              } },
            });
            var subnets = new SubnetSelection {
                Subnets = vpc.PublicSubnets
            };
            var sg = new SecurityGroup(this, "MasterSg", new SecurityGroupProps
            {
                AllowAllOutbound = true,
                Vpc = vpc,
            });
            var role = new Role(this, "MasterRole", new RoleProps
            {
                AssumedBy = new ServicePrincipal("ec2.amazonaws.com"),
            });

            role.AddManagedPolicy(ManagedPolicy.FromAwsManagedPolicyName("AmazonSSMManagedInstanceCore"));

            var spot = new AutoScalingGroup(this, "instances", new AutoScalingGroupProps
            {
                // Monitoring is default DETAILED.
                SpotPrice                = "1.0", // 0.0096 for spot price average for m3.medium
                Vpc                      = vpc,
                SecurityGroup            = sg,
                VpcSubnets               = subnets,
                InstanceType             = InstanceType.Of(InstanceClass.COMPUTE5_AMD, InstanceSize.XLARGE4),
                DesiredCapacity          = 1,
                MaxCapacity              = 1,
                MinCapacity              = 0,
                AssociatePublicIpAddress = true,
                MachineImage             = new AmazonLinuxImage(new AmazonLinuxImageProps
                {
                    CpuType        = AmazonLinuxCpuType.X86_64,
                    Generation     = AmazonLinuxGeneration.AMAZON_LINUX_2,
                    Storage        = AmazonLinuxStorage.GENERAL_PURPOSE,
                    Virtualization = AmazonLinuxVirt.HVM,
                }),
                AllowAllOutbound = true,
                GroupMetrics     = new[] { GroupMetrics.All() },
                Role             = role,
                UpdatePolicy     = UpdatePolicy.ReplacingUpdate(),
            });

            // https://gist.github.com/npearce/6f3c7826c7499587f00957fee62f8ee9
            spot.AddUserData(new[]
            {
                "amazon-linux-extras install docker -y",
                "service docker start",
                "chkconfig docker on",
                "usermod -a -G docker ec2-user",
                "curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose",
                "chmod +x /usr/local/bin/docker-compose",
                "yum install -y git",
                "reboot",
            });
        }
Esempio n. 2
0
        internal CdkStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
        {
            var stackProps                = ReportStackProps.ParseOrDefault(props);
            var dframeWorkerLogGroup      = "MagicOnionBenchWorkerLogGroup";
            var dframeMasterLogGroup      = "MagicOnionBenchMasterLogGroup";
            var benchNetwork              = stackProps.GetBenchNetwork();
            var recreateMagicOnionTrigger = stackProps.GetBenchmarkServerBinariesHash();

            // s3
            var s3 = new Bucket(this, "Bucket", new BucketProps
            {
                AutoDeleteObjects = true,
                RemovalPolicy     = RemovalPolicy.DESTROY,
                AccessControl     = BucketAccessControl.PRIVATE,
            });
            var lifecycleRule = new LifecycleRule
            {
                Enabled    = true,
                Prefix     = "reports/",
                Expiration = Duration.Days(stackProps.DaysKeepReports),
                AbortIncompleteMultipartUploadAfter = Duration.Days(1),
            };

            s3.AddLifecycleRule(lifecycleRule);
            s3.AddToResourcePolicy(new PolicyStatement(new PolicyStatementProps
            {
                Sid        = "AllowPublicRead",
                Effect     = Effect.ALLOW,
                Principals = new[] { new AnyPrincipal() },
                Actions    = new[] { "s3:GetObject*" },
                Resources  = new[] { $"{s3.BucketArn}/html/*" },
            }));
            s3.AddToResourcePolicy(new PolicyStatement(new PolicyStatementProps
            {
                Sid        = "AllowAwsAccountAccess",
                Effect     = Effect.ALLOW,
                Principals = new[] { new AccountRootPrincipal() },
                Actions    = new[] { "s3:*" },
                Resources  = new[] { $"{s3.BucketArn}/*" },
            }));

            // s3 deploy
            var masterDllDeployment = new BucketDeployment(this, "DeployMasterDll", new BucketDeploymentProps
            {
                DestinationBucket    = s3,
                Sources              = new[] { Source.Asset(Path.Combine(Directory.GetCurrentDirectory(), $"out/linux/server")) },
                DestinationKeyPrefix = $"assembly/linux/server"
            });
            var userdataDeployment = new BucketDeployment(this, "UserData", new BucketDeploymentProps
            {
                DestinationBucket    = s3,
                Sources              = new[] { Source.Asset(Path.Combine(Directory.GetCurrentDirectory(), "userdata/")) },
                DestinationKeyPrefix = "userdata/"
            });

            // docker deploy
            var dockerImage = new DockerImageAsset(this, "dframeWorkerImage", new DockerImageAssetProps
            {
                Directory = Path.Combine(Directory.GetCurrentDirectory(), "app"),
                File      = "ConsoleAppEcs/Dockerfile.Ecs",
            });
            var dframeImage = ContainerImage.FromDockerImageAsset(dockerImage);

            // network
            var vpc = new Vpc(this, "Vpc", new VpcProps
            {
                MaxAzs              = 2,
                NatGateways         = 0,
                SubnetConfiguration = new[] { new SubnetConfiguration {
                                                  Name = "public", SubnetType = SubnetType.PUBLIC
                                              } },
            });
            var allsubnets = new SubnetSelection {
                Subnets = vpc.PublicSubnets
            };
            var singleSubnets = new SubnetSelection {
                Subnets = new[] { vpc.PublicSubnets.First() }
            };
            var sg = new SecurityGroup(this, "MasterSg", new SecurityGroupProps
            {
                AllowAllOutbound = true,
                Vpc = vpc,
            });

            foreach (var subnet in vpc.PublicSubnets)
            {
                sg.AddIngressRule(Peer.Ipv4(vpc.VpcCidrBlock), Port.AllTcp(), "VPC", true);
            }

            // service discovery
            var serviceDiscoveryDomain = "local";
            var serverMapName          = "server";
            var dframeMapName          = "dframe-master";
            var ns = new PrivateDnsNamespace(this, "Namespace", new PrivateDnsNamespaceProps
            {
                Vpc  = vpc,
                Name = serviceDiscoveryDomain,
            });
            var serviceDiscoveryServer = ns.CreateService("server", new DnsServiceProps
            {
                Name          = serverMapName,
                DnsRecordType = DnsRecordType.A,
                RoutingPolicy = RoutingPolicy.MULTIVALUE,
            });

            // alb
            var albDnsName = "benchmark-alb";
            var benchToMagicOnionDnsName = benchNetwork.RequireAlb
                ? $"{benchNetwork.EndpointScheme}://{albDnsName}.{stackProps.AlbDomain.domain}"
                : $"{benchNetwork.EndpointScheme}://{serverMapName}.{serviceDiscoveryDomain}";
            IApplicationTargetGroup grpcTargetGroup  = null;
            IApplicationTargetGroup httpsTargetGroup = null;

            if (benchNetwork.RequireAlb)
            {
                // route53
                var hostedZone = HostedZone.FromHostedZoneAttributes(this, "HostedZone", new HostedZoneAttributes
                {
                    HostedZoneId = stackProps.AlbDomain.zoneId,
                    ZoneName     = stackProps.AlbDomain.domain,
                });

                // acm
                var certificate = new DnsValidatedCertificate(this, "certificate", new DnsValidatedCertificateProps
                {
                    DomainName = $"{albDnsName}.{hostedZone.ZoneName}",
                    HostedZone = hostedZone,
                });
                // alb
                var lb = new ApplicationLoadBalancer(this, "LB", new ApplicationLoadBalancerProps
                {
                    Vpc           = vpc,
                    VpcSubnets    = allsubnets,
                    SecurityGroup = new SecurityGroup(this, "AlbSg", new SecurityGroupProps
                    {
                        AllowAllOutbound = true,
                        Vpc = vpc,
                    }),
                    InternetFacing = false,
                    Http2Enabled   = true,
                });
                grpcTargetGroup  = AddGrpcTargetGroup(benchNetwork, vpc, certificate, lb);
                httpsTargetGroup = AddHttpsTargetGroup(benchNetwork, vpc, certificate, lb);

                // Dns Record
                _ = new CnameRecord(this, "alb-alias-record", new CnameRecordProps
                {
                    RecordName = $"{albDnsName}.{stackProps.AlbDomain.domain}",
                    Ttl        = Duration.Seconds(60),
                    Zone       = hostedZone,
                    DomainName = lb.LoadBalancerDnsName,
                });
            }

            // iam
            var iamEc2MagicOnionRole  = GetIamEc2MagicOnionRole(s3, serviceDiscoveryServer);
            var iamEcsTaskExecuteRole = GetIamEcsTaskExecuteRole(new[] { dframeWorkerLogGroup, dframeMasterLogGroup });
            var iamDFrameTaskDefRole  = GetIamEcsDframeTaskDefRole(s3);
            var iamWorkerTaskDefRole  = GetIamEcsWorkerTaskDefRole(s3);

            // secrets
            var ddToken = stackProps.UseEc2DatadogAgentProfiler || stackProps.UseFargateDatadogAgentProfiler
                ? Amazon.CDK.AWS.SecretsManager.Secret.FromSecretNameV2(this, "dd-token", "magiconion-benchmark-datadog-token")
                : null;

            // MagicOnion
            var asg = new AutoScalingGroup(this, "MagicOnionAsg", new AutoScalingGroupProps
            {
                // Monitoring is default DETAILED.
                SpotPrice                = "1.0", // 0.0096 for spot price average for m3.medium
                Vpc                      = vpc,
                SecurityGroup            = sg,
                VpcSubnets               = singleSubnets,
                InstanceType             = stackProps.MagicOnionInstanceType,
                DesiredCapacity          = 1,
                MaxCapacity              = 1,
                MinCapacity              = 0,
                AssociatePublicIpAddress = true,
                MachineImage             = new AmazonLinuxImage(new AmazonLinuxImageProps
                {
                    CpuType        = AmazonLinuxCpuType.X86_64,
                    Generation     = AmazonLinuxGeneration.AMAZON_LINUX_2,
                    Storage        = AmazonLinuxStorage.GENERAL_PURPOSE,
                    Virtualization = AmazonLinuxVirt.HVM,
                }),
                AllowAllOutbound = true,
                GroupMetrics     = new[] { GroupMetrics.All() },
                Role             = iamEc2MagicOnionRole,
                UpdatePolicy     = UpdatePolicy.ReplacingUpdate(),
                Signals          = Signals.WaitForCount(1, new SignalsOptions
                {
                    Timeout = Duration.Minutes(10),
                }),
            });

            asg.AddSecretsReadGrant(ddToken, () => stackProps.UseEc2DatadogAgentProfiler);
            var userdata = GetUserData(recreateMagicOnionTrigger, s3.BucketName, stackProps.BenchmarkBinaryNames, serviceDiscoveryServer.ServiceId, stackProps.UseEc2CloudWatchAgentProfiler, stackProps.UseEc2DatadogAgentProfiler);

            asg.AddUserData(userdata);
            asg.UserData.AddSignalOnExitCommand(asg);
            asg.Node.AddDependency(masterDllDeployment);
            asg.Node.AddDependency(userdataDeployment);
            if (stackProps.EnableMagicOnionScaleInCron)
            {
                asg.ScaleOnSchedule("ScheduleOut", new BasicScheduledActionProps
                {
                    DesiredCapacity = 1,
                    MaxCapacity     = 1,
                    // AM9:00 (JST+9) on Monday to Wednesday
                    Schedule = Schedule.Expression("0 0 * 1-3 *"),
                });
                asg.ScaleOnSchedule("ScheduleIn", new BasicScheduledActionProps
                {
                    DesiredCapacity = 0,
                    MaxCapacity     = 0,
                    // PM9:00 (JST+9) on Everyday
                    Schedule = Schedule.Expression("0 12 * 1-7 *"),
                });
            }
            if (benchNetwork.RequireAlb)
            {
                asg.AttachToApplicationTargetGroup(grpcTargetGroup);
                asg.AttachToApplicationTargetGroup(httpsTargetGroup);
            }

            // ECS
            var cluster = new Cluster(this, "WorkerCluster", new ClusterProps
            {
                Vpc = vpc,
            });

            cluster.Node.AddDependency(asg); // wait until asg is up

            // dframe-worker
            var dframeWorkerContainerName = "worker";
            var dframeWorkerTaskDef       = new FargateTaskDefinition(this, "DFrameWorkerTaskDef", new FargateTaskDefinitionProps
            {
                ExecutionRole  = iamEcsTaskExecuteRole,
                TaskRole       = iamWorkerTaskDefRole,
                Cpu            = stackProps.WorkerFargate.CpuSize,
                MemoryLimitMiB = stackProps.WorkerFargate.MemorySize,
            });

            dframeWorkerTaskDef.AddContainer(dframeWorkerContainerName, new ContainerDefinitionOptions
            {
                Image       = dframeImage,
                Command     = new[] { "--worker-flag" },
                Environment = new Dictionary <string, string>
                {
                    { "DFRAME_MASTER_CONNECT_TO_HOST", $"{dframeMapName}.{serviceDiscoveryDomain}" },
                    { "DFRAME_MASTER_CONNECT_TO_PORT", "12345" },
                    { "BENCH_SERVER_HOST", benchToMagicOnionDnsName },
                    { "BENCH_REPORTID", stackProps.ReportId },
                    { "BENCH_S3BUCKET", s3.BucketName },
                },
                Logging = LogDriver.AwsLogs(new AwsLogDriverProps
                {
                    LogGroup = new LogGroup(this, "WorkerLogGroup", new LogGroupProps
                    {
                        LogGroupName  = dframeWorkerLogGroup,
                        RemovalPolicy = RemovalPolicy.DESTROY,
                        Retention     = RetentionDays.TWO_WEEKS,
                    }),
                    StreamPrefix = dframeWorkerLogGroup,
                }),
            });
            dframeWorkerTaskDef.AddDatadogContainer($"{dframeWorkerContainerName}-datadog", ddToken, () => stackProps.UseFargateDatadogAgentProfiler);
            var dframeWorkerService = new FargateService(this, "DFrameWorkerService", new FargateServiceProps
            {
                ServiceName       = "DFrameWorkerService",
                DesiredCount      = 0,
                Cluster           = cluster,
                TaskDefinition    = dframeWorkerTaskDef,
                VpcSubnets        = singleSubnets,
                SecurityGroups    = new[] { sg },
                PlatformVersion   = FargatePlatformVersion.VERSION1_4,
                MinHealthyPercent = 0,
                AssignPublicIp    = true,
            });

            // dframe-master
            var dframeMasterTaskDef = new FargateTaskDefinition(this, "DFrameMasterTaskDef", new FargateTaskDefinitionProps
            {
                ExecutionRole  = iamEcsTaskExecuteRole,
                TaskRole       = iamDFrameTaskDefRole,
                Cpu            = stackProps.MasterFargate.CpuSize,
                MemoryLimitMiB = stackProps.MasterFargate.MemorySize,
            });

            dframeMasterTaskDef.AddContainer("dframe", new ContainerDefinitionOptions
            {
                Image       = dframeImage,
                Environment = new Dictionary <string, string>
                {
                    { "DFRAME_CLUSTER_NAME", cluster.ClusterName },
                    { "DFRAME_MASTER_SERVICE_NAME", "DFrameMasterService" },
                    { "DFRAME_WORKER_CONTAINER_NAME", dframeWorkerContainerName },
                    { "DFRAME_WORKER_SERVICE_NAME", dframeWorkerService.ServiceName },
                    { "DFRAME_WORKER_TASK_NAME", Fn.Select(1, Fn.Split("/", dframeWorkerTaskDef.TaskDefinitionArn)) },
                    { "DFRAME_WORKER_IMAGE", dockerImage.ImageUri },
                    { "BENCH_REPORTID", stackProps.ReportId },
                    { "BENCH_S3BUCKET", s3.BucketName },
                },
                Logging = LogDriver.AwsLogs(new AwsLogDriverProps
                {
                    LogGroup = new LogGroup(this, "MasterLogGroup", new LogGroupProps
                    {
                        LogGroupName  = dframeMasterLogGroup,
                        RemovalPolicy = RemovalPolicy.DESTROY,
                        Retention     = RetentionDays.TWO_WEEKS,
                    }),
                    StreamPrefix = dframeMasterLogGroup,
                }),
            });
            dframeMasterTaskDef.AddDatadogContainer($"dframe-datadog", ddToken, () => stackProps.UseFargateDatadogAgentProfiler);
            var dframeMasterService = new FargateService(this, "DFrameMasterService", new FargateServiceProps
            {
                ServiceName       = "DFrameMasterService",
                DesiredCount      = 1,
                Cluster           = cluster,
                TaskDefinition    = dframeMasterTaskDef,
                VpcSubnets        = singleSubnets,
                SecurityGroups    = new[] { sg },
                PlatformVersion   = FargatePlatformVersion.VERSION1_4,
                MinHealthyPercent = 0,
                AssignPublicIp    = true,
            });

            dframeMasterService.EnableCloudMap(new CloudMapOptions
            {
                CloudMapNamespace = ns,
                Name          = dframeMapName,
                DnsRecordType = DnsRecordType.A,
                DnsTtl        = Duration.Seconds(300),
            });

            // output
            new CfnOutput(this, "ReportUrl", new CfnOutputProps {
                Value = $"https://{s3.BucketRegionalDomainName}/html/{stackProps.ReportId}/index.html"
            });
            new CfnOutput(this, "EndPointStyle", new CfnOutputProps {
                Value = stackProps.BenchmarkEndpoint.ToString()
            });
            new CfnOutput(this, "AsgName", new CfnOutputProps {
                Value = asg.AutoScalingGroupName
            });
            new CfnOutput(this, "EcsClusterName", new CfnOutputProps {
                Value = cluster.ClusterName
            });
            new CfnOutput(this, "DFrameWorkerEcsTaskdefImage", new CfnOutputProps {
                Value = dockerImage.ImageUri
            });
        }