internal TheScalableWebhookStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
/*
* DynamoDB Table
* This is standing in for what is RDS on the diagram due to simpler/cheaper setup
*/
_dynamoDbTable = new DynamoDB.Table(this, "Messages", new DynamoDB.TableProps
{
PartitionKey = new DynamoDB.Attribute {
Name = "id", Type = DynamoDB.AttributeType.STRING
}
});
/*
* Queue Setup
*/
_queueRds = new SQS.Queue(this, "RDSPublishQueue", new SQS.QueueProps
{
VisibilityTimeout = Duration.Seconds(300)
});
/*
* Lambdas
* Both publisher and subscriber from pattern
*/
/*
* defines an AWS Lambda resource to publish to our sqs_queue
*/
_functionPublish = new Lambda.Function(this, "SQSPublishLambdaHandler", new Lambda.FunctionProps
{
Runtime = Lambda.Runtime.NODEJS_12_X, // execution environment
Handler = "lambda.handler", // file is "lambda", function is "handler"
Code = Lambda.Code.FromAsset("lambda_fns/publish"), // code loaded from the "lambda_fns/publish" directory
Environment = new Dictionary <string, string>
{
{ "queueURL", _queueRds.QueueUrl }
}
});
_queueRds.GrantSendMessages(_functionPublish);
/*
* defines an AWS Lambda resource to pull from our sqs_queue
*/
_functionSubscribe = new Lambda.Function(this, "SQSSubscribeLambdaHandler", new Lambda.FunctionProps
{
Runtime = Lambda.Runtime.NODEJS_12_X, // execution environment
Handler = "lambda.handler", // file is "lambda", function is "handler"
Code = Lambda.Code.FromAsset("lambda_fns/subscribe"), // code loaded from the "lambda_fns/subscribe" directory
Environment = new Dictionary <string, string>
{
{ "queueURL", _queueRds.QueueUrl },
{ "tableName", _dynamoDbTable.TableName }
},
ReservedConcurrentExecutions = 2 // throttle lambda to 2 concurrent invocations
});
_queueRds.GrantConsumeMessages(_functionSubscribe);
_functionSubscribe.AddEventSource(new LambdaEvents.SqsEventSource(_queueRds));
_dynamoDbTable.GrantReadWriteData(_functionSubscribe);
/**
* API Gateway Proxy
* Used to expose the webhook through a URL
*
* defines an API Gateway REST API resource backed by our "sqs_publish_lambda" function.
*/
new APIG.LambdaRestApi(this, "Endpoint", new APIG.LambdaRestApiProps
{
Handler = _functionPublish
});
}
public SrcStack(Construct scope, string id, IStackProps props) : base(scope, id, props)
{
#region VPC, Subnets and Security Groups
IVpc batchVpc = new Vpc(this, Constants.VPC_ID, new VpcProps {
Cidr = Constants.CIDR_RANGE,
MaxAzs = 4
});
var privateSubnetIds = new List <string>();
foreach (Subnet subnet in batchVpc.PrivateSubnets)
{
privateSubnetIds.Add(subnet.SubnetId);
}
var batchSecurityGroup = new SecurityGroup(this, Constants.BATCH_SECURITY_GROUP_ID,
new SecurityGroupProps {
Vpc = batchVpc,
SecurityGroupName = Constants.BATCH_SECURITY_GROUP_NAME,
Description = Constants.BATCH_SECURITY_GROUP_DESCRIPTION,
}
);
var batchSecurityGroupIds = new List <string>()
{
batchSecurityGroup.SecurityGroupId
};
#endregion
#region S3, DynamoDB, Lambda (to trigger the batch on file drop)
s3.Bucket bucket = new s3.Bucket(this, Constants.S3_BUCKET_ID, new s3.BucketProps {
BucketName = Constants.S3_BUCKET_NAME + this.Account
});
var bucketName = bucket.BucketName;
var gsi = new dynamo.GlobalSecondaryIndexProps()
{
IndexName = "GSI",
PartitionKey = new dynamo.Attribute {
Name = "CreatedTime", Type = dynamo.AttributeType.STRING
},
ProjectionType = dynamo.ProjectionType.KEYS_ONLY
};
dynamo.Table table = new dynamo.Table(this, Constants.DynamoDB_TABLE_ID, new dynamo.TableProps {
TableName = Constants.DynamoDB_TABLE_ID,
PartitionKey = new dynamo.Attribute {
Name = "ProductId", Type = dynamo.AttributeType.STRING
},
BillingMode = dynamo.BillingMode.PAY_PER_REQUEST
});
table.AddGlobalSecondaryIndex(gsi);
var lambdaBatchExecutionRoleProvider = new LambdaBatchExecutionRole(this, Constants.LAMBDA_BATCH_EXECUTION_ROLE_ID);
#endregion
DockerImageAsset imageAsset = new DockerImageAsset(this, "BatchProcessorImage", new DockerImageAssetProps {
Directory = Constants.BATCH_PROCESSOR_PYTHON_CODE_PATH,
RepositoryName = Constants.ECR_REPOSITORY_NAME
});
#region Batch - ComputeEnvironment - Job Queue - Job Definition
var batchServiceRole = new BatchServiceRole(this, Constants.BATCH_SERVICE_ROLE_ID);
var ecsInstanceRole = new EcsInstanceRole(this, Constants.ECS_INSTANCE_ROLE_ID);
var batchInstanceProfile = new InstanceProfile(this, Constants.BATCH_INSTANCE_PROFILE_ID, ecsInstanceRole);
var computeEnvironment = new batch.CfnComputeEnvironment(this, Constants.BATCH_COMPUTE_ENVIRONMENT_ID,
new batch.CfnComputeEnvironmentProps {
ComputeEnvironmentName = Constants.BATCH_COMPUTE_ENVIRONMENT_NAME,
Type = Constants.BATCH_COMPUTE_TYPE,
ServiceRole = batchServiceRole.Role.RoleName,
ComputeResources = new batch.CfnComputeEnvironment.ComputeResourcesProperty {
Type = Constants.BATCH_COMPUTE_RESOURCE_TYPE,
MinvCpus = 0,
MaxvCpus = 32,
DesiredvCpus = 0,
InstanceRole = ecsInstanceRole.Role.RoleName,
InstanceTypes = new string[] {
Constants.BATCH_INSTANCE_TYPE
},
SecurityGroupIds = batchSecurityGroupIds.ToArray(),
Subnets = privateSubnetIds.ToArray()
}
});
var computeEnvironmentOrders = new List <batch.CfnJobQueue.ComputeEnvironmentOrderProperty>();
var computeEnvironmentOrder = new batch.CfnJobQueue.ComputeEnvironmentOrderProperty()
{
Order = 1,
ComputeEnvironment = computeEnvironment.Ref
};
computeEnvironmentOrders.Add(computeEnvironmentOrder);
var batchProcessingJobQueue = new batch.CfnJobQueue(this, Constants.BATCH_JOB_QUEUE_ID, new batch.CfnJobQueueProps {
JobQueueName = Constants.BATCH_PROCESSING_JOB_QUEUE,
Priority = 1,
ComputeEnvironmentOrder = computeEnvironmentOrders.ToArray()
});
var batchProcessingJobDefinition = new batch.CfnJobDefinition(this, Constants.BATCH_JOB_DEFINITION_ID, new batch.CfnJobDefinitionProps {
Type = Constants.CONTAINER,
JobDefinitionName = Constants.BATCH_JOB_DEFINITION_NAME,
ContainerProperties = new batch.CfnJobDefinition.ContainerPropertiesProperty()
{
Image = imageAsset.ImageUri,
Vcpus = Constants.BATCH_JOB_DEFINITION_VCPU,
Memory = Constants.BATCH_JOB_DEFINITION_MemoryLimitMiB,
Command = new string[] {
"python",
"batch_processor.py"
}
}
});
#endregion
#region lambda s3 event trigger
BatchTriggerLambdaInput batchTriggerLambdaInput = new BatchTriggerLambdaInput {
BucketName = bucketName,
FileName = "sample.csv",
Region = "us-east-1",
BatchJobDefinition = batchProcessingJobDefinition.JobDefinitionName,
BatchJobName = Constants.BATCH_JOB_NAME,
BatchJobQueue = batchProcessingJobQueue.JobQueueName,
DBTable = table.TableName
};
lambda.Function lambda = BatchTriggerLambda.BatchTriggerLambdaFunction(this, Constants.LAMBDA_NAME,
batchTriggerLambdaInput, lambdaBatchExecutionRoleProvider);
var batchLambdaFunction = new LambdaFunction(
lambda
);
var eventPutSource = new eventsources.S3EventSource(bucket, new eventsources.S3EventSourceProps {
Events = new s3.EventType[] {
s3.EventType.OBJECT_CREATED_PUT
}
});
lambda.AddEventSource(eventPutSource);
#endregion
#region OUTPUTS
var cfnComputeOutput = new CfnOutput(this, Constants.BATCH_COMPUTE_ENVIRONMENT_NAME_OUTPUT_ID,
new CfnOutputProps {
Value = computeEnvironment.Ref
}
);
var cfnS3Output = new CfnOutput(this, Constants.S3_BUCKET_OUTPUT_ID,
new CfnOutputProps {
Value = bucket.BucketName
}
);
var cfnEcrRepositoryOutput = new CfnOutput(this, Constants.ECR_REPOSITORY_OUTPUT_ID,
new CfnOutputProps {
Value = imageAsset.Repository.RepositoryArn
}
);
var cfnDynamoTableOutput = new CfnOutput(this, Constants.DYNAMO_TABLE_OUTPUT_ID,
new CfnOutputProps {
Value = table.TableName
}
);
var cfnLambdaRepositoryOutput = new CfnOutput(this, Constants.LAMBDA_OUTPUT_ID,
new CfnOutputProps {
Value = lambda.FunctionArn
}
);
#endregion
Tag.Add(this, "Name", Constants.APP_NAME);
}