internal Amazon.CDK.AWS.ServiceDiscovery.PrivateDnsNamespace CreateCloudMapNamespace(string id, IVpc vpc) { var privateNamespace = new PrivateDnsNamespace(this, $"{id}-cloudmap-private-namespace", new PrivateDnsNamespaceProps { Description = $"Cloudmap Namespace for {id}", Vpc = vpc, Name = $"{id}" }); return(privateNamespace); }
private (FargateService, Amazon.CDK.AWS.ServiceDiscovery.IService) CreateService(string id, string serviceName, string branch, IVpc vpc, Cluster cluster, PrivateDnsNamespace cloudMapNamespace, string appMeshVirtualNodeName, Mesh mesh, IRole taskExecutionRole, RepositoryImage envoyImage, ContainerImage containerImage) { var taskDefinition = new TaskDefinition(this, $"{id}_{serviceName}-{branch}-task-definiton", new TaskDefinitionProps { Compatibility = Compatibility.FARGATE, MemoryMiB = "512", Cpu = "256", ProxyConfiguration = new AppMeshProxyConfiguration(new AppMeshProxyConfigurationConfigProps() { ContainerName = "envoy", Properties = new AppMeshProxyConfigurationProps() { AppPorts = new double[1] { 5000 }, ProxyIngressPort = 15000, ProxyEgressPort = 15001, IgnoredUID = 1337, EgressIgnoredIPs = new string[2] { "169.254.170.2", "169.254.169.254" } } }), Family = $"{id}_{serviceName}-task-definition", ExecutionRole = taskExecutionRole }); var envoyContainer = taskDefinition.AddContainer("envoy", new ContainerDefinitionOptions { User = "******", Image = envoyImage, Essential = true, Environment = new Dictionary <string, string> { { "APPMESH_VIRTUAL_NODE_NAME", $"mesh/{mesh.MeshName}/virtualNode/{appMeshVirtualNodeName}" } }, HealthCheck = new Amazon.CDK.AWS.ECS.HealthCheck() { Command = new string[] { "CMD-SHELL", "curl -s http://localhost:9901/server_info | grep state | grep -q LIVE" }, Interval = Duration.Seconds(5), Timeout = Duration.Seconds(2), StartPeriod = Duration.Seconds(10), Retries = 3 }, MemoryLimitMiB = 500, Logging = new AwsLogDriver(new AwsLogDriverProps() { StreamPrefix = $"{id}_{serviceName}-{branch}-envoy", LogRetention = RetentionDays.ONE_DAY }), }); var container = taskDefinition.AddContainer($"{serviceName}-container", new ContainerDefinitionOptions() { Image = containerImage, Logging = new AwsLogDriver(new AwsLogDriverProps() { StreamPrefix = $"{id}_{serviceName}-{branch}-service", LogRetention = RetentionDays.ONE_DAY }), Essential = true, Environment = new Dictionary <string, string> { { "BRANCH", branch }, { "APPMESH_NAMESPACE", cloudMapNamespace.PrivateDnsNamespaceName } } }); container.AddPortMappings(new Amazon.CDK.AWS.ECS.PortMapping() { ContainerPort = 5000 }); container.AddContainerDependencies(new ContainerDependency() { Condition = ContainerDependencyCondition.HEALTHY, Container = envoyContainer }); // Cloudmap will append the namespace to the dns entry in R53. // We're explicitly checking for master here because for service to service lookups to go via the envoy proxy, the DNS name must resolve. // see https://github.com/aws/aws-app-mesh-roadmap/issues/65 // i.e I want the ping service to call http://pong-service.{namespace}:5000/ and for this to be routed correctly by the proxy. // If you create the fargate task with cloudmap service integration with a more specific (branched) DNS name then pong-service.{namespace} r53 entry will never be created // and routing doesn't work through envoy. var dnsName = $"{serviceName}-service{(branch == "master" ? "" : "-" + branch)}"; var sg = new SecurityGroup(this, $"{id}_{serviceName}-{branch}-sg", new SecurityGroupProps() { AllowAllOutbound = true, SecurityGroupName = $"{id}_{serviceName}-{branch}-sg", Vpc = vpc, }); sg.AddIngressRule(Peer.AnyIpv4(), new Port(new PortProps() { Protocol = Amazon.CDK.AWS.EC2.Protocol.TCP, FromPort = 5000, ToPort = 5000, StringRepresentation = "tcp:5000:5000" }), "allow access from outside."); var fargateService = new Amazon.CDK.AWS.ECS.FargateService(this, $"{serviceName}-{branch}-service", new FargateServiceProps { ServiceName = $"{serviceName}-{branch}-service", AssignPublicIp = true, Cluster = cluster, TaskDefinition = taskDefinition, VpcSubnets = new SubnetSelection() { Subnets = vpc.PublicSubnets }, CloudMapOptions = new CloudMapOptions() { Name = dnsName, DnsRecordType = DnsRecordType.A, CloudMapNamespace = cloudMapNamespace }, SecurityGroup = sg }); return(fargateService, fargateService.CloudMapService); }
private VirtualService AddVirtualService(string id, string virtualServiceName, Mesh mesh, VirtualRouter router, PrivateDnsNamespace cloudmapNamespace) { return(mesh.AddVirtualService($"{id}-{virtualServiceName}-service", new VirtualServiceProps() { Mesh = mesh, VirtualRouter = router, VirtualServiceName = $"{virtualServiceName}-service.{cloudmapNamespace.NamespaceName}" })); }
internal CdkStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props) { var stackProps = ReportStackProps.ParseOrDefault(props); var dframeWorkerLogGroup = "MagicOnionBenchWorkerLogGroup"; var dframeMasterLogGroup = "MagicOnionBenchMasterLogGroup"; var benchNetwork = stackProps.GetBenchNetwork(); var recreateMagicOnionTrigger = stackProps.GetBenchmarkServerBinariesHash(); // s3 var s3 = new Bucket(this, "Bucket", new BucketProps { AutoDeleteObjects = true, RemovalPolicy = RemovalPolicy.DESTROY, AccessControl = BucketAccessControl.PRIVATE, }); var lifecycleRule = new LifecycleRule { Enabled = true, Prefix = "reports/", Expiration = Duration.Days(stackProps.DaysKeepReports), AbortIncompleteMultipartUploadAfter = Duration.Days(1), }; s3.AddLifecycleRule(lifecycleRule); s3.AddToResourcePolicy(new PolicyStatement(new PolicyStatementProps { Sid = "AllowPublicRead", Effect = Effect.ALLOW, Principals = new[] { new AnyPrincipal() }, Actions = new[] { "s3:GetObject*" }, Resources = new[] { $"{s3.BucketArn}/html/*" }, })); s3.AddToResourcePolicy(new PolicyStatement(new PolicyStatementProps { Sid = "AllowAwsAccountAccess", Effect = Effect.ALLOW, Principals = new[] { new AccountRootPrincipal() }, Actions = new[] { "s3:*" }, Resources = new[] { $"{s3.BucketArn}/*" }, })); // s3 deploy var masterDllDeployment = new BucketDeployment(this, "DeployMasterDll", new BucketDeploymentProps { DestinationBucket = s3, Sources = new[] { Source.Asset(Path.Combine(Directory.GetCurrentDirectory(), $"out/linux/server")) }, DestinationKeyPrefix = $"assembly/linux/server" }); var userdataDeployment = new BucketDeployment(this, "UserData", new BucketDeploymentProps { DestinationBucket = s3, Sources = new[] { Source.Asset(Path.Combine(Directory.GetCurrentDirectory(), "userdata/")) }, DestinationKeyPrefix = "userdata/" }); // docker deploy var dockerImage = new DockerImageAsset(this, "dframeWorkerImage", new DockerImageAssetProps { Directory = Path.Combine(Directory.GetCurrentDirectory(), "app"), File = "ConsoleAppEcs/Dockerfile.Ecs", }); var dframeImage = ContainerImage.FromDockerImageAsset(dockerImage); // network var vpc = new Vpc(this, "Vpc", new VpcProps { MaxAzs = 2, NatGateways = 0, SubnetConfiguration = new[] { new SubnetConfiguration { Name = "public", SubnetType = SubnetType.PUBLIC } }, }); var allsubnets = new SubnetSelection { Subnets = vpc.PublicSubnets }; var singleSubnets = new SubnetSelection { Subnets = new[] { vpc.PublicSubnets.First() } }; var sg = new SecurityGroup(this, "MasterSg", new SecurityGroupProps { AllowAllOutbound = true, Vpc = vpc, }); foreach (var subnet in vpc.PublicSubnets) { sg.AddIngressRule(Peer.Ipv4(vpc.VpcCidrBlock), Port.AllTcp(), "VPC", true); } // service discovery var serviceDiscoveryDomain = "local"; var serverMapName = "server"; var dframeMapName = "dframe-master"; var ns = new PrivateDnsNamespace(this, "Namespace", new PrivateDnsNamespaceProps { Vpc = vpc, Name = serviceDiscoveryDomain, }); var serviceDiscoveryServer = ns.CreateService("server", new DnsServiceProps { Name = serverMapName, DnsRecordType = DnsRecordType.A, RoutingPolicy = RoutingPolicy.MULTIVALUE, }); // alb var albDnsName = "benchmark-alb"; var benchToMagicOnionDnsName = benchNetwork.RequireAlb ? $"{benchNetwork.EndpointScheme}://{albDnsName}.{stackProps.AlbDomain.domain}" : $"{benchNetwork.EndpointScheme}://{serverMapName}.{serviceDiscoveryDomain}"; IApplicationTargetGroup grpcTargetGroup = null; IApplicationTargetGroup httpsTargetGroup = null; if (benchNetwork.RequireAlb) { // route53 var hostedZone = HostedZone.FromHostedZoneAttributes(this, "HostedZone", new HostedZoneAttributes { HostedZoneId = stackProps.AlbDomain.zoneId, ZoneName = stackProps.AlbDomain.domain, }); // acm var certificate = new DnsValidatedCertificate(this, "certificate", new DnsValidatedCertificateProps { DomainName = $"{albDnsName}.{hostedZone.ZoneName}", HostedZone = hostedZone, }); // alb var lb = new ApplicationLoadBalancer(this, "LB", new ApplicationLoadBalancerProps { Vpc = vpc, VpcSubnets = allsubnets, SecurityGroup = new SecurityGroup(this, "AlbSg", new SecurityGroupProps { AllowAllOutbound = true, Vpc = vpc, }), InternetFacing = false, Http2Enabled = true, }); grpcTargetGroup = AddGrpcTargetGroup(benchNetwork, vpc, certificate, lb); httpsTargetGroup = AddHttpsTargetGroup(benchNetwork, vpc, certificate, lb); // Dns Record _ = new CnameRecord(this, "alb-alias-record", new CnameRecordProps { RecordName = $"{albDnsName}.{stackProps.AlbDomain.domain}", Ttl = Duration.Seconds(60), Zone = hostedZone, DomainName = lb.LoadBalancerDnsName, }); } // iam var iamEc2MagicOnionRole = GetIamEc2MagicOnionRole(s3, serviceDiscoveryServer); var iamEcsTaskExecuteRole = GetIamEcsTaskExecuteRole(new[] { dframeWorkerLogGroup, dframeMasterLogGroup }); var iamDFrameTaskDefRole = GetIamEcsDframeTaskDefRole(s3); var iamWorkerTaskDefRole = GetIamEcsWorkerTaskDefRole(s3); // secrets var ddToken = stackProps.UseEc2DatadogAgentProfiler || stackProps.UseFargateDatadogAgentProfiler ? Amazon.CDK.AWS.SecretsManager.Secret.FromSecretNameV2(this, "dd-token", "magiconion-benchmark-datadog-token") : null; // MagicOnion var asg = new AutoScalingGroup(this, "MagicOnionAsg", new AutoScalingGroupProps { // Monitoring is default DETAILED. SpotPrice = "1.0", // 0.0096 for spot price average for m3.medium Vpc = vpc, SecurityGroup = sg, VpcSubnets = singleSubnets, InstanceType = stackProps.MagicOnionInstanceType, DesiredCapacity = 1, MaxCapacity = 1, MinCapacity = 0, AssociatePublicIpAddress = true, MachineImage = new AmazonLinuxImage(new AmazonLinuxImageProps { CpuType = AmazonLinuxCpuType.X86_64, Generation = AmazonLinuxGeneration.AMAZON_LINUX_2, Storage = AmazonLinuxStorage.GENERAL_PURPOSE, Virtualization = AmazonLinuxVirt.HVM, }), AllowAllOutbound = true, GroupMetrics = new[] { GroupMetrics.All() }, Role = iamEc2MagicOnionRole, UpdatePolicy = UpdatePolicy.ReplacingUpdate(), Signals = Signals.WaitForCount(1, new SignalsOptions { Timeout = Duration.Minutes(10), }), }); asg.AddSecretsReadGrant(ddToken, () => stackProps.UseEc2DatadogAgentProfiler); var userdata = GetUserData(recreateMagicOnionTrigger, s3.BucketName, stackProps.BenchmarkBinaryNames, serviceDiscoveryServer.ServiceId, stackProps.UseEc2CloudWatchAgentProfiler, stackProps.UseEc2DatadogAgentProfiler); asg.AddUserData(userdata); asg.UserData.AddSignalOnExitCommand(asg); asg.Node.AddDependency(masterDllDeployment); asg.Node.AddDependency(userdataDeployment); if (stackProps.EnableMagicOnionScaleInCron) { asg.ScaleOnSchedule("ScheduleOut", new BasicScheduledActionProps { DesiredCapacity = 1, MaxCapacity = 1, // AM9:00 (JST+9) on Monday to Wednesday Schedule = Schedule.Expression("0 0 * 1-3 *"), }); asg.ScaleOnSchedule("ScheduleIn", new BasicScheduledActionProps { DesiredCapacity = 0, MaxCapacity = 0, // PM9:00 (JST+9) on Everyday Schedule = Schedule.Expression("0 12 * 1-7 *"), }); } if (benchNetwork.RequireAlb) { asg.AttachToApplicationTargetGroup(grpcTargetGroup); asg.AttachToApplicationTargetGroup(httpsTargetGroup); } // ECS var cluster = new Cluster(this, "WorkerCluster", new ClusterProps { Vpc = vpc, }); cluster.Node.AddDependency(asg); // wait until asg is up // dframe-worker var dframeWorkerContainerName = "worker"; var dframeWorkerTaskDef = new FargateTaskDefinition(this, "DFrameWorkerTaskDef", new FargateTaskDefinitionProps { ExecutionRole = iamEcsTaskExecuteRole, TaskRole = iamWorkerTaskDefRole, Cpu = stackProps.WorkerFargate.CpuSize, MemoryLimitMiB = stackProps.WorkerFargate.MemorySize, }); dframeWorkerTaskDef.AddContainer(dframeWorkerContainerName, new ContainerDefinitionOptions { Image = dframeImage, Command = new[] { "--worker-flag" }, Environment = new Dictionary <string, string> { { "DFRAME_MASTER_CONNECT_TO_HOST", $"{dframeMapName}.{serviceDiscoveryDomain}" }, { "DFRAME_MASTER_CONNECT_TO_PORT", "12345" }, { "BENCH_SERVER_HOST", benchToMagicOnionDnsName }, { "BENCH_REPORTID", stackProps.ReportId }, { "BENCH_S3BUCKET", s3.BucketName }, }, Logging = LogDriver.AwsLogs(new AwsLogDriverProps { LogGroup = new LogGroup(this, "WorkerLogGroup", new LogGroupProps { LogGroupName = dframeWorkerLogGroup, RemovalPolicy = RemovalPolicy.DESTROY, Retention = RetentionDays.TWO_WEEKS, }), StreamPrefix = dframeWorkerLogGroup, }), }); dframeWorkerTaskDef.AddDatadogContainer($"{dframeWorkerContainerName}-datadog", ddToken, () => stackProps.UseFargateDatadogAgentProfiler); var dframeWorkerService = new FargateService(this, "DFrameWorkerService", new FargateServiceProps { ServiceName = "DFrameWorkerService", DesiredCount = 0, Cluster = cluster, TaskDefinition = dframeWorkerTaskDef, VpcSubnets = singleSubnets, SecurityGroups = new[] { sg }, PlatformVersion = FargatePlatformVersion.VERSION1_4, MinHealthyPercent = 0, AssignPublicIp = true, }); // dframe-master var dframeMasterTaskDef = new FargateTaskDefinition(this, "DFrameMasterTaskDef", new FargateTaskDefinitionProps { ExecutionRole = iamEcsTaskExecuteRole, TaskRole = iamDFrameTaskDefRole, Cpu = stackProps.MasterFargate.CpuSize, MemoryLimitMiB = stackProps.MasterFargate.MemorySize, }); dframeMasterTaskDef.AddContainer("dframe", new ContainerDefinitionOptions { Image = dframeImage, Environment = new Dictionary <string, string> { { "DFRAME_CLUSTER_NAME", cluster.ClusterName }, { "DFRAME_MASTER_SERVICE_NAME", "DFrameMasterService" }, { "DFRAME_WORKER_CONTAINER_NAME", dframeWorkerContainerName }, { "DFRAME_WORKER_SERVICE_NAME", dframeWorkerService.ServiceName }, { "DFRAME_WORKER_TASK_NAME", Fn.Select(1, Fn.Split("/", dframeWorkerTaskDef.TaskDefinitionArn)) }, { "DFRAME_WORKER_IMAGE", dockerImage.ImageUri }, { "BENCH_REPORTID", stackProps.ReportId }, { "BENCH_S3BUCKET", s3.BucketName }, }, Logging = LogDriver.AwsLogs(new AwsLogDriverProps { LogGroup = new LogGroup(this, "MasterLogGroup", new LogGroupProps { LogGroupName = dframeMasterLogGroup, RemovalPolicy = RemovalPolicy.DESTROY, Retention = RetentionDays.TWO_WEEKS, }), StreamPrefix = dframeMasterLogGroup, }), }); dframeMasterTaskDef.AddDatadogContainer($"dframe-datadog", ddToken, () => stackProps.UseFargateDatadogAgentProfiler); var dframeMasterService = new FargateService(this, "DFrameMasterService", new FargateServiceProps { ServiceName = "DFrameMasterService", DesiredCount = 1, Cluster = cluster, TaskDefinition = dframeMasterTaskDef, VpcSubnets = singleSubnets, SecurityGroups = new[] { sg }, PlatformVersion = FargatePlatformVersion.VERSION1_4, MinHealthyPercent = 0, AssignPublicIp = true, }); dframeMasterService.EnableCloudMap(new CloudMapOptions { CloudMapNamespace = ns, Name = dframeMapName, DnsRecordType = DnsRecordType.A, DnsTtl = Duration.Seconds(300), }); // output new CfnOutput(this, "ReportUrl", new CfnOutputProps { Value = $"https://{s3.BucketRegionalDomainName}/html/{stackProps.ReportId}/index.html" }); new CfnOutput(this, "EndPointStyle", new CfnOutputProps { Value = stackProps.BenchmarkEndpoint.ToString() }); new CfnOutput(this, "AsgName", new CfnOutputProps { Value = asg.AutoScalingGroupName }); new CfnOutput(this, "EcsClusterName", new CfnOutputProps { Value = cluster.ClusterName }); new CfnOutput(this, "DFrameWorkerEcsTaskdefImage", new CfnOutputProps { Value = dockerImage.ImageUri }); }