internal Stack(Construct scope, string id, IStackProps props = null) : base(scope, id, props) { //setup the image var asset = new DockerImageAsset(this, $"{Config.AppName}Image", new DockerImageAssetProps { Directory = Path.Combine(System.Environment.CurrentDirectory, "api"), }); //Create the Fargate service var vpc = Vpc.FromLookup( this, "sandbox", new VpcLookupOptions { VpcName = "sandbox_vpc" } ); var cluster = new Cluster(this, $"{Config.AppName}Cluster", new ClusterProps { Vpc = vpc }); var applicationDomain = $"{Config.ApplicationSubdomain}.{Config.DomainName}"; var hostedZone = HostedZone.FromLookup( this, "HostedZone", new HostedZoneProviderProps { DomainName = $"{Config.DomainName}.", PrivateZone = false } ); // Create a load-balanced Fargate service and make it public var fargateService = new ApplicationLoadBalancedFargateService(this, $"{Config.AppName}Service", new ApplicationLoadBalancedFargateServiceProps { Cluster = cluster, // Required DesiredCount = 1, // Default is 1 TaskImageOptions = new ApplicationLoadBalancedTaskImageOptions { Image = ContainerImage.FromDockerImageAsset(asset) }, MemoryLimitMiB = 1024, // Default is 256 PublicLoadBalancer = true, // Default is false DomainName = applicationDomain, DomainZone = hostedZone, } ); new CfnOutput( this, "Route53Url", new CfnOutputProps { Value = applicationDomain, Description = "Nice Route53 Url" } ); }
internal CdkStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props) { var stackProps = ReportStackProps.ParseOrDefault(props); var dframeWorkerLogGroup = "MagicOnionBenchWorkerLogGroup"; var dframeMasterLogGroup = "MagicOnionBenchMasterLogGroup"; var benchNetwork = stackProps.GetBenchNetwork(); var recreateMagicOnionTrigger = stackProps.GetBenchmarkServerBinariesHash(); // s3 var s3 = new Bucket(this, "Bucket", new BucketProps { AutoDeleteObjects = true, RemovalPolicy = RemovalPolicy.DESTROY, AccessControl = BucketAccessControl.PRIVATE, }); var lifecycleRule = new LifecycleRule { Enabled = true, Prefix = "reports/", Expiration = Duration.Days(stackProps.DaysKeepReports), AbortIncompleteMultipartUploadAfter = Duration.Days(1), }; s3.AddLifecycleRule(lifecycleRule); s3.AddToResourcePolicy(new PolicyStatement(new PolicyStatementProps { Sid = "AllowPublicRead", Effect = Effect.ALLOW, Principals = new[] { new AnyPrincipal() }, Actions = new[] { "s3:GetObject*" }, Resources = new[] { $"{s3.BucketArn}/html/*" }, })); s3.AddToResourcePolicy(new PolicyStatement(new PolicyStatementProps { Sid = "AllowAwsAccountAccess", Effect = Effect.ALLOW, Principals = new[] { new AccountRootPrincipal() }, Actions = new[] { "s3:*" }, Resources = new[] { $"{s3.BucketArn}/*" }, })); // s3 deploy var masterDllDeployment = new BucketDeployment(this, "DeployMasterDll", new BucketDeploymentProps { DestinationBucket = s3, Sources = new[] { Source.Asset(Path.Combine(Directory.GetCurrentDirectory(), $"out/linux/server")) }, DestinationKeyPrefix = $"assembly/linux/server" }); var userdataDeployment = new BucketDeployment(this, "UserData", new BucketDeploymentProps { DestinationBucket = s3, Sources = new[] { Source.Asset(Path.Combine(Directory.GetCurrentDirectory(), "userdata/")) }, DestinationKeyPrefix = "userdata/" }); // docker deploy var dockerImage = new DockerImageAsset(this, "dframeWorkerImage", new DockerImageAssetProps { Directory = Path.Combine(Directory.GetCurrentDirectory(), "app"), File = "ConsoleAppEcs/Dockerfile.Ecs", }); var dframeImage = ContainerImage.FromDockerImageAsset(dockerImage); // network var vpc = new Vpc(this, "Vpc", new VpcProps { MaxAzs = 2, NatGateways = 0, SubnetConfiguration = new[] { new SubnetConfiguration { Name = "public", SubnetType = SubnetType.PUBLIC } }, }); var allsubnets = new SubnetSelection { Subnets = vpc.PublicSubnets }; var singleSubnets = new SubnetSelection { Subnets = new[] { vpc.PublicSubnets.First() } }; var sg = new SecurityGroup(this, "MasterSg", new SecurityGroupProps { AllowAllOutbound = true, Vpc = vpc, }); foreach (var subnet in vpc.PublicSubnets) { sg.AddIngressRule(Peer.Ipv4(vpc.VpcCidrBlock), Port.AllTcp(), "VPC", true); } // service discovery var serviceDiscoveryDomain = "local"; var serverMapName = "server"; var dframeMapName = "dframe-master"; var ns = new PrivateDnsNamespace(this, "Namespace", new PrivateDnsNamespaceProps { Vpc = vpc, Name = serviceDiscoveryDomain, }); var serviceDiscoveryServer = ns.CreateService("server", new DnsServiceProps { Name = serverMapName, DnsRecordType = DnsRecordType.A, RoutingPolicy = RoutingPolicy.MULTIVALUE, }); // alb var albDnsName = "benchmark-alb"; var benchToMagicOnionDnsName = benchNetwork.RequireAlb ? $"{benchNetwork.EndpointScheme}://{albDnsName}.{stackProps.AlbDomain.domain}" : $"{benchNetwork.EndpointScheme}://{serverMapName}.{serviceDiscoveryDomain}"; IApplicationTargetGroup grpcTargetGroup = null; IApplicationTargetGroup httpsTargetGroup = null; if (benchNetwork.RequireAlb) { // route53 var hostedZone = HostedZone.FromHostedZoneAttributes(this, "HostedZone", new HostedZoneAttributes { HostedZoneId = stackProps.AlbDomain.zoneId, ZoneName = stackProps.AlbDomain.domain, }); // acm var certificate = new DnsValidatedCertificate(this, "certificate", new DnsValidatedCertificateProps { DomainName = $"{albDnsName}.{hostedZone.ZoneName}", HostedZone = hostedZone, }); // alb var lb = new ApplicationLoadBalancer(this, "LB", new ApplicationLoadBalancerProps { Vpc = vpc, VpcSubnets = allsubnets, SecurityGroup = new SecurityGroup(this, "AlbSg", new SecurityGroupProps { AllowAllOutbound = true, Vpc = vpc, }), InternetFacing = false, Http2Enabled = true, }); grpcTargetGroup = AddGrpcTargetGroup(benchNetwork, vpc, certificate, lb); httpsTargetGroup = AddHttpsTargetGroup(benchNetwork, vpc, certificate, lb); // Dns Record _ = new CnameRecord(this, "alb-alias-record", new CnameRecordProps { RecordName = $"{albDnsName}.{stackProps.AlbDomain.domain}", Ttl = Duration.Seconds(60), Zone = hostedZone, DomainName = lb.LoadBalancerDnsName, }); } // iam var iamEc2MagicOnionRole = GetIamEc2MagicOnionRole(s3, serviceDiscoveryServer); var iamEcsTaskExecuteRole = GetIamEcsTaskExecuteRole(new[] { dframeWorkerLogGroup, dframeMasterLogGroup }); var iamDFrameTaskDefRole = GetIamEcsDframeTaskDefRole(s3); var iamWorkerTaskDefRole = GetIamEcsWorkerTaskDefRole(s3); // secrets var ddToken = stackProps.UseEc2DatadogAgentProfiler || stackProps.UseFargateDatadogAgentProfiler ? Amazon.CDK.AWS.SecretsManager.Secret.FromSecretNameV2(this, "dd-token", "magiconion-benchmark-datadog-token") : null; // MagicOnion var asg = new AutoScalingGroup(this, "MagicOnionAsg", new AutoScalingGroupProps { // Monitoring is default DETAILED. SpotPrice = "1.0", // 0.0096 for spot price average for m3.medium Vpc = vpc, SecurityGroup = sg, VpcSubnets = singleSubnets, InstanceType = stackProps.MagicOnionInstanceType, DesiredCapacity = 1, MaxCapacity = 1, MinCapacity = 0, AssociatePublicIpAddress = true, MachineImage = new AmazonLinuxImage(new AmazonLinuxImageProps { CpuType = AmazonLinuxCpuType.X86_64, Generation = AmazonLinuxGeneration.AMAZON_LINUX_2, Storage = AmazonLinuxStorage.GENERAL_PURPOSE, Virtualization = AmazonLinuxVirt.HVM, }), AllowAllOutbound = true, GroupMetrics = new[] { GroupMetrics.All() }, Role = iamEc2MagicOnionRole, UpdatePolicy = UpdatePolicy.ReplacingUpdate(), Signals = Signals.WaitForCount(1, new SignalsOptions { Timeout = Duration.Minutes(10), }), }); asg.AddSecretsReadGrant(ddToken, () => stackProps.UseEc2DatadogAgentProfiler); var userdata = GetUserData(recreateMagicOnionTrigger, s3.BucketName, stackProps.BenchmarkBinaryNames, serviceDiscoveryServer.ServiceId, stackProps.UseEc2CloudWatchAgentProfiler, stackProps.UseEc2DatadogAgentProfiler); asg.AddUserData(userdata); asg.UserData.AddSignalOnExitCommand(asg); asg.Node.AddDependency(masterDllDeployment); asg.Node.AddDependency(userdataDeployment); if (stackProps.EnableMagicOnionScaleInCron) { asg.ScaleOnSchedule("ScheduleOut", new BasicScheduledActionProps { DesiredCapacity = 1, MaxCapacity = 1, // AM9:00 (JST+9) on Monday to Wednesday Schedule = Schedule.Expression("0 0 * 1-3 *"), }); asg.ScaleOnSchedule("ScheduleIn", new BasicScheduledActionProps { DesiredCapacity = 0, MaxCapacity = 0, // PM9:00 (JST+9) on Everyday Schedule = Schedule.Expression("0 12 * 1-7 *"), }); } if (benchNetwork.RequireAlb) { asg.AttachToApplicationTargetGroup(grpcTargetGroup); asg.AttachToApplicationTargetGroup(httpsTargetGroup); } // ECS var cluster = new Cluster(this, "WorkerCluster", new ClusterProps { Vpc = vpc, }); cluster.Node.AddDependency(asg); // wait until asg is up // dframe-worker var dframeWorkerContainerName = "worker"; var dframeWorkerTaskDef = new FargateTaskDefinition(this, "DFrameWorkerTaskDef", new FargateTaskDefinitionProps { ExecutionRole = iamEcsTaskExecuteRole, TaskRole = iamWorkerTaskDefRole, Cpu = stackProps.WorkerFargate.CpuSize, MemoryLimitMiB = stackProps.WorkerFargate.MemorySize, }); dframeWorkerTaskDef.AddContainer(dframeWorkerContainerName, new ContainerDefinitionOptions { Image = dframeImage, Command = new[] { "--worker-flag" }, Environment = new Dictionary <string, string> { { "DFRAME_MASTER_CONNECT_TO_HOST", $"{dframeMapName}.{serviceDiscoveryDomain}" }, { "DFRAME_MASTER_CONNECT_TO_PORT", "12345" }, { "BENCH_SERVER_HOST", benchToMagicOnionDnsName }, { "BENCH_REPORTID", stackProps.ReportId }, { "BENCH_S3BUCKET", s3.BucketName }, }, Logging = LogDriver.AwsLogs(new AwsLogDriverProps { LogGroup = new LogGroup(this, "WorkerLogGroup", new LogGroupProps { LogGroupName = dframeWorkerLogGroup, RemovalPolicy = RemovalPolicy.DESTROY, Retention = RetentionDays.TWO_WEEKS, }), StreamPrefix = dframeWorkerLogGroup, }), }); dframeWorkerTaskDef.AddDatadogContainer($"{dframeWorkerContainerName}-datadog", ddToken, () => stackProps.UseFargateDatadogAgentProfiler); var dframeWorkerService = new FargateService(this, "DFrameWorkerService", new FargateServiceProps { ServiceName = "DFrameWorkerService", DesiredCount = 0, Cluster = cluster, TaskDefinition = dframeWorkerTaskDef, VpcSubnets = singleSubnets, SecurityGroups = new[] { sg }, PlatformVersion = FargatePlatformVersion.VERSION1_4, MinHealthyPercent = 0, AssignPublicIp = true, }); // dframe-master var dframeMasterTaskDef = new FargateTaskDefinition(this, "DFrameMasterTaskDef", new FargateTaskDefinitionProps { ExecutionRole = iamEcsTaskExecuteRole, TaskRole = iamDFrameTaskDefRole, Cpu = stackProps.MasterFargate.CpuSize, MemoryLimitMiB = stackProps.MasterFargate.MemorySize, }); dframeMasterTaskDef.AddContainer("dframe", new ContainerDefinitionOptions { Image = dframeImage, Environment = new Dictionary <string, string> { { "DFRAME_CLUSTER_NAME", cluster.ClusterName }, { "DFRAME_MASTER_SERVICE_NAME", "DFrameMasterService" }, { "DFRAME_WORKER_CONTAINER_NAME", dframeWorkerContainerName }, { "DFRAME_WORKER_SERVICE_NAME", dframeWorkerService.ServiceName }, { "DFRAME_WORKER_TASK_NAME", Fn.Select(1, Fn.Split("/", dframeWorkerTaskDef.TaskDefinitionArn)) }, { "DFRAME_WORKER_IMAGE", dockerImage.ImageUri }, { "BENCH_REPORTID", stackProps.ReportId }, { "BENCH_S3BUCKET", s3.BucketName }, }, Logging = LogDriver.AwsLogs(new AwsLogDriverProps { LogGroup = new LogGroup(this, "MasterLogGroup", new LogGroupProps { LogGroupName = dframeMasterLogGroup, RemovalPolicy = RemovalPolicy.DESTROY, Retention = RetentionDays.TWO_WEEKS, }), StreamPrefix = dframeMasterLogGroup, }), }); dframeMasterTaskDef.AddDatadogContainer($"dframe-datadog", ddToken, () => stackProps.UseFargateDatadogAgentProfiler); var dframeMasterService = new FargateService(this, "DFrameMasterService", new FargateServiceProps { ServiceName = "DFrameMasterService", DesiredCount = 1, Cluster = cluster, TaskDefinition = dframeMasterTaskDef, VpcSubnets = singleSubnets, SecurityGroups = new[] { sg }, PlatformVersion = FargatePlatformVersion.VERSION1_4, MinHealthyPercent = 0, AssignPublicIp = true, }); dframeMasterService.EnableCloudMap(new CloudMapOptions { CloudMapNamespace = ns, Name = dframeMapName, DnsRecordType = DnsRecordType.A, DnsTtl = Duration.Seconds(300), }); // output new CfnOutput(this, "ReportUrl", new CfnOutputProps { Value = $"https://{s3.BucketRegionalDomainName}/html/{stackProps.ReportId}/index.html" }); new CfnOutput(this, "EndPointStyle", new CfnOutputProps { Value = stackProps.BenchmarkEndpoint.ToString() }); new CfnOutput(this, "AsgName", new CfnOutputProps { Value = asg.AutoScalingGroupName }); new CfnOutput(this, "EcsClusterName", new CfnOutputProps { Value = cluster.ClusterName }); new CfnOutput(this, "DFrameWorkerEcsTaskdefImage", new CfnOutputProps { Value = dockerImage.ImageUri }); }
public SrcStack(Construct scope, string id, IStackProps props) : base(scope, id, props) { #region VPC, Subnets and Security Groups IVpc batchVpc = new Vpc(this, Constants.VPC_ID, new VpcProps { Cidr = Constants.CIDR_RANGE, MaxAzs = 4 }); var privateSubnetIds = new List <string>(); foreach (Subnet subnet in batchVpc.PrivateSubnets) { privateSubnetIds.Add(subnet.SubnetId); } var batchSecurityGroup = new SecurityGroup(this, Constants.BATCH_SECURITY_GROUP_ID, new SecurityGroupProps { Vpc = batchVpc, SecurityGroupName = Constants.BATCH_SECURITY_GROUP_NAME, Description = Constants.BATCH_SECURITY_GROUP_DESCRIPTION, } ); var batchSecurityGroupIds = new List <string>() { batchSecurityGroup.SecurityGroupId }; #endregion #region S3, DynamoDB, Lambda (to trigger the batch on file drop) s3.Bucket bucket = new s3.Bucket(this, Constants.S3_BUCKET_ID, new s3.BucketProps { BucketName = Constants.S3_BUCKET_NAME + this.Account }); var bucketName = bucket.BucketName; var gsi = new dynamo.GlobalSecondaryIndexProps() { IndexName = "GSI", PartitionKey = new dynamo.Attribute { Name = "CreatedTime", Type = dynamo.AttributeType.STRING }, ProjectionType = dynamo.ProjectionType.KEYS_ONLY }; dynamo.Table table = new dynamo.Table(this, Constants.DynamoDB_TABLE_ID, new dynamo.TableProps { TableName = Constants.DynamoDB_TABLE_ID, PartitionKey = new dynamo.Attribute { Name = "ProductId", Type = dynamo.AttributeType.STRING }, BillingMode = dynamo.BillingMode.PAY_PER_REQUEST }); table.AddGlobalSecondaryIndex(gsi); var lambdaBatchExecutionRoleProvider = new LambdaBatchExecutionRole(this, Constants.LAMBDA_BATCH_EXECUTION_ROLE_ID); #endregion DockerImageAsset imageAsset = new DockerImageAsset(this, "BatchProcessorImage", new DockerImageAssetProps { Directory = Constants.BATCH_PROCESSOR_PYTHON_CODE_PATH, RepositoryName = Constants.ECR_REPOSITORY_NAME }); #region Batch - ComputeEnvironment - Job Queue - Job Definition var batchServiceRole = new BatchServiceRole(this, Constants.BATCH_SERVICE_ROLE_ID); var ecsInstanceRole = new EcsInstanceRole(this, Constants.ECS_INSTANCE_ROLE_ID); var batchInstanceProfile = new InstanceProfile(this, Constants.BATCH_INSTANCE_PROFILE_ID, ecsInstanceRole); var computeEnvironment = new batch.CfnComputeEnvironment(this, Constants.BATCH_COMPUTE_ENVIRONMENT_ID, new batch.CfnComputeEnvironmentProps { ComputeEnvironmentName = Constants.BATCH_COMPUTE_ENVIRONMENT_NAME, Type = Constants.BATCH_COMPUTE_TYPE, ServiceRole = batchServiceRole.Role.RoleName, ComputeResources = new batch.CfnComputeEnvironment.ComputeResourcesProperty { Type = Constants.BATCH_COMPUTE_RESOURCE_TYPE, MinvCpus = 0, MaxvCpus = 32, DesiredvCpus = 0, InstanceRole = ecsInstanceRole.Role.RoleName, InstanceTypes = new string[] { Constants.BATCH_INSTANCE_TYPE }, SecurityGroupIds = batchSecurityGroupIds.ToArray(), Subnets = privateSubnetIds.ToArray() } }); var computeEnvironmentOrders = new List <batch.CfnJobQueue.ComputeEnvironmentOrderProperty>(); var computeEnvironmentOrder = new batch.CfnJobQueue.ComputeEnvironmentOrderProperty() { Order = 1, ComputeEnvironment = computeEnvironment.Ref }; computeEnvironmentOrders.Add(computeEnvironmentOrder); var batchProcessingJobQueue = new batch.CfnJobQueue(this, Constants.BATCH_JOB_QUEUE_ID, new batch.CfnJobQueueProps { JobQueueName = Constants.BATCH_PROCESSING_JOB_QUEUE, Priority = 1, ComputeEnvironmentOrder = computeEnvironmentOrders.ToArray() }); var batchProcessingJobDefinition = new batch.CfnJobDefinition(this, Constants.BATCH_JOB_DEFINITION_ID, new batch.CfnJobDefinitionProps { Type = Constants.CONTAINER, JobDefinitionName = Constants.BATCH_JOB_DEFINITION_NAME, ContainerProperties = new batch.CfnJobDefinition.ContainerPropertiesProperty() { Image = imageAsset.ImageUri, Vcpus = Constants.BATCH_JOB_DEFINITION_VCPU, Memory = Constants.BATCH_JOB_DEFINITION_MemoryLimitMiB, Command = new string[] { "python", "batch_processor.py" } } }); #endregion #region lambda s3 event trigger BatchTriggerLambdaInput batchTriggerLambdaInput = new BatchTriggerLambdaInput { BucketName = bucketName, FileName = "sample.csv", Region = "us-east-1", BatchJobDefinition = batchProcessingJobDefinition.JobDefinitionName, BatchJobName = Constants.BATCH_JOB_NAME, BatchJobQueue = batchProcessingJobQueue.JobQueueName, DBTable = table.TableName }; lambda.Function lambda = BatchTriggerLambda.BatchTriggerLambdaFunction(this, Constants.LAMBDA_NAME, batchTriggerLambdaInput, lambdaBatchExecutionRoleProvider); var batchLambdaFunction = new LambdaFunction( lambda ); var eventPutSource = new eventsources.S3EventSource(bucket, new eventsources.S3EventSourceProps { Events = new s3.EventType[] { s3.EventType.OBJECT_CREATED_PUT } }); lambda.AddEventSource(eventPutSource); #endregion #region OUTPUTS var cfnComputeOutput = new CfnOutput(this, Constants.BATCH_COMPUTE_ENVIRONMENT_NAME_OUTPUT_ID, new CfnOutputProps { Value = computeEnvironment.Ref } ); var cfnS3Output = new CfnOutput(this, Constants.S3_BUCKET_OUTPUT_ID, new CfnOutputProps { Value = bucket.BucketName } ); var cfnEcrRepositoryOutput = new CfnOutput(this, Constants.ECR_REPOSITORY_OUTPUT_ID, new CfnOutputProps { Value = imageAsset.Repository.RepositoryArn } ); var cfnDynamoTableOutput = new CfnOutput(this, Constants.DYNAMO_TABLE_OUTPUT_ID, new CfnOutputProps { Value = table.TableName } ); var cfnLambdaRepositoryOutput = new CfnOutput(this, Constants.LAMBDA_OUTPUT_ID, new CfnOutputProps { Value = lambda.FunctionArn } ); #endregion Tag.Add(this, "Name", Constants.APP_NAME); }