internal TheScheduledLambdaStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
// DynamoDB Table
_dynamoDbTable = new DynamoDB.Table(this, "RequestTable", new DynamoDB.TableProps
{
PartitionKey = new DynamoDB.Attribute
{
Name = "requestid", Type = DynamoDB.AttributeType.STRING
},
RemovalPolicy = RemovalPolicy.DESTROY
});
// Create the Lambda function we want to run on a schedule
_functionScheduled = new Lambda.Function(this, "ScheduledLambda", new Lambda.FunctionProps
{
Runtime = Lambda.Runtime.NODEJS_12_X, // execution environment
Handler = "index.handler", // file is "index", function is "handler"
Code = Lambda.Code.FromAsset("lambda_fns"), // code loaded from the "lambda_fns" directory,
Environment = new Dictionary <string, string>
{
{ "TABLE_NAME", _dynamoDbTable.TableName }
}
});
// Allow our lambda fn to write to the table
_dynamoDbTable.GrantReadWriteData(_functionScheduled);
// Create EventBridge rule that will execute our Lambda every 2 minutes
_ruleScheduled = new Events.Rule(this, "scheduledLambda-schedule", new Events.RuleProps
{
Schedule = Events.Schedule.Expression("rate(2 minutes)")
});
// Set the target of our EventBridge rule to our Lambda function
_ruleScheduled.AddTarget(new EventsTarget.LambdaFunction(_functionScheduled));
}
/// <summary>
/// Starting point: define stack in this method
/// </summary>
/// <param name="scope"></param>
/// <param name="id"></param>
/// <param name="props"></param>
/// <returns></returns>
internal AirQualityCdkStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
// Metadata to be used by internal construct methods - might need a bit of refactoring and better design
// this holds keyvaluePairs which can be used to configure environmentVariables
var metadata = new Dictionary <string, KeyValuePair <string, string> >();
// The code that defines your stack goes here
// SNS Topics
new Topic(this, "EmailNotificationPublisher");
// SQS queues
var cityFeedSqs = AirQualitySQS("CityFeedQueue");
// SNS Topics
var twitterPubSNS = new Topic(this, $"twitterPub-{Guid.NewGuid().ToString().Substring(0,10)}");
var generalPubSNS = new Topic(this, $"generalPub-{Guid.NewGuid().ToString().Substring(0,10)}");
// Dynamo DB table
var tableProps = new Dynamo.TableProps();
tableProps.TableName = Constants.DbAirQualityTableName;
var primaryKey = new Dynamo.Attribute {
Name = Constants.DbPartitionKeyName, Type = Dynamo.AttributeType.STRING
};
tableProps.BillingMode = Dynamo.BillingMode.PAY_PER_REQUEST; // on-demand pricing
tableProps.PartitionKey = primaryKey;
tableProps.Stream = Dynamo.StreamViewType.NEW_IMAGE;
var airqualityTable = new Dynamo.Table(this, Constants.DbAirQualityTableName, tableProps);
//Environment value metadata for lambda creation
metadata[Constants.EnvCityFeedSqsUrl] = KeyValuePair.Create(Constants.EnvCityFeedSqsUrl, cityFeedSqs.QueueUrl);
metadata[Constants.EnvTwitterSNS] = KeyValuePair.Create(Constants.EnvTwitterSNS, twitterPubSNS.TopicArn);
metadata[Constants.EnvTwitterAPIKey] = KeyValuePair.Create(Constants.EnvTwitterAPIKey, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterAPIKey));
metadata[Constants.EnvTwitterAPISecret] = KeyValuePair.Create(Constants.EnvTwitterAPISecret, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterAPISecret));
metadata[Constants.EnvTwitterAccessToken] = KeyValuePair.Create(Constants.EnvTwitterAccessToken, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterAccessToken));
metadata[Constants.EnvTwitterTokenSecret] = KeyValuePair.Create(Constants.EnvTwitterTokenSecret, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterTokenSecret));
metadata[Constants.EnvAirQualityTable] = KeyValuePair.Create(Constants.EnvAirQualityTable, airqualityTable.TableName);
metadata[Constants.EnvGeneralSNSTopic] = KeyValuePair.Create(Constants.EnvGeneralSNSTopic, generalPubSNS.TopicArn);
// AWS lambdas
// TODO: Lambda generation logic to be done using environment values. Config might also be pulled from S3
// GetCityFeed lambda
var functionsByName = ConstructLambdas(metadata);
// subscribe and configure
// find twitter publisher lambda
twitterPubSNS.AddSubscription(new LambdaSubscription(functionsByName
.Where(pair => pair.Key.Contains(Constants.DefaultAirQualityTwitterPublisherLambdaName))
.Single().Value));
// trigger feed processor lambda only on dynamo db stream change by coutry-city feed
functionsByName
.Where(pair => pair.Key.Contains(Constants.DefaultAirQualityFeedProcessorLambdaName))
.Single().Value.AddEventSource(new DynamoEventSource(airqualityTable, new DynamoEventSourceProps()));
// Event schedule every 15 mins - // architecture targets 5, change when tested and working
var waqiLambda = functionsByName
.Where(pair => pair.Key.Contains(Constants.DefaultWaqiGetCityFeedLambdaName))
.Single().Value;
var targetWaqiLambda = new LambdaFunction(waqiLambda);
new Rule(this, "Waqi5min", new RuleProps {
Schedule = Schedule.Rate(Duration.Minutes(5))
}).AddTarget(targetWaqiLambda);
}
internal TheScalableWebhookStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
/*
* DynamoDB Table
* This is standing in for what is RDS on the diagram due to simpler/cheaper setup
*/
_dynamoDbTable = new DynamoDB.Table(this, "Messages", new DynamoDB.TableProps
{
PartitionKey = new DynamoDB.Attribute {
Name = "id", Type = DynamoDB.AttributeType.STRING
}
});
/*
* Queue Setup
*/
_queueRds = new SQS.Queue(this, "RDSPublishQueue", new SQS.QueueProps
{
VisibilityTimeout = Duration.Seconds(300)
});
/*
* Lambdas
* Both publisher and subscriber from pattern
*/
/*
* defines an AWS Lambda resource to publish to our sqs_queue
*/
_functionPublish = new Lambda.Function(this, "SQSPublishLambdaHandler", new Lambda.FunctionProps
{
Runtime = Lambda.Runtime.NODEJS_12_X, // execution environment
Handler = "lambda.handler", // file is "lambda", function is "handler"
Code = Lambda.Code.FromAsset("lambda_fns/publish"), // code loaded from the "lambda_fns/publish" directory
Environment = new Dictionary <string, string>
{
{ "queueURL", _queueRds.QueueUrl }
}
});
_queueRds.GrantSendMessages(_functionPublish);
/*
* defines an AWS Lambda resource to pull from our sqs_queue
*/
_functionSubscribe = new Lambda.Function(this, "SQSSubscribeLambdaHandler", new Lambda.FunctionProps
{
Runtime = Lambda.Runtime.NODEJS_12_X, // execution environment
Handler = "lambda.handler", // file is "lambda", function is "handler"
Code = Lambda.Code.FromAsset("lambda_fns/subscribe"), // code loaded from the "lambda_fns/subscribe" directory
Environment = new Dictionary <string, string>
{
{ "queueURL", _queueRds.QueueUrl },
{ "tableName", _dynamoDbTable.TableName }
},
ReservedConcurrentExecutions = 2 // throttle lambda to 2 concurrent invocations
});
_queueRds.GrantConsumeMessages(_functionSubscribe);
_functionSubscribe.AddEventSource(new LambdaEvents.SqsEventSource(_queueRds));
_dynamoDbTable.GrantReadWriteData(_functionSubscribe);
/**
* API Gateway Proxy
* Used to expose the webhook through a URL
*
* defines an API Gateway REST API resource backed by our "sqs_publish_lambda" function.
*/
new APIG.LambdaRestApi(this, "Endpoint", new APIG.LambdaRestApiProps
{
Handler = _functionPublish
});
}
internal ApiGatewayCDKStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
var api = new RestApi(this, "MyApi2", new RestApiProps
{
RestApiName = "my-api2",
Deploy = true,
DeployOptions = new StageOptions
{
StageName = "preprod",
Variables = new Dictionary <string, string>
{
{ "lambdaAlias", "preprod" }
},
TracingEnabled = true
}
});
//var deployment = new Deployment(this, "preprd", new DeploymentProps
//{
// Api = api,
// Description = "Initial Deployment",
// RetainDeployments
//});
//deployment.AddStage("prod");
//deployment.AddStage("preprod");
var ddbTable = new DynamoDB.Table(this, "SampleTable", new DynamoDB.TableProps
{
TableName = "SampleTable",
PartitionKey = new DynamoDB.Attribute
{
Name = "PartitionKey",
Type = DynamoDB.AttributeType.STRING
},
SortKey = new DynamoDB.Attribute
{
Name = "SortKey",
Type = DynamoDB.AttributeType.STRING
}
});
var lambda = new Lambda.Function(this, "MyLambda", new Lambda.FunctionProps
{
FunctionName = "DemoProxyLambda",
Code = Lambda.Code.FromAsset("./LambdaSource"),
Handler = "WebAppLambda::WebAppLambda.LambdaEntryPoint::FunctionHandlerAsync",
Runtime = Lambda.Runtime.DOTNET_CORE_3_1,
MemorySize = 1536,
Timeout = Duration.Seconds(30),
CurrentVersionOptions = new Lambda.VersionOptions
{
RemovalPolicy = RemovalPolicy.RETAIN,
RetryAttempts = 1,
},
Tracing = Lambda.Tracing.ACTIVE,
Environment = new Dictionary <string, string>
{
{ "CodeVersionString", "0.0.13" }
}
});
lambda.Role.AttachInlinePolicy(new IAM.Policy(this, "lambdaiam", new IAM.PolicyProps
{
PolicyName = "DynamoDbAccess",
Document = new IAM.PolicyDocument(new IAM.PolicyDocumentProps
{
Statements = new[]
{
new IAM.PolicyStatement(new IAM.PolicyStatementProps
{
Effect = IAM.Effect.ALLOW,
Actions = new []
{
"dynamodb:BatchGetItem",
"dynamodb:BatchWriteItem",
"dynamodb:PutItem",
"dynamodb:DeleteItem",
"dynamodb:PartiQLUpdate",
"dynamodb:Scan",
"dynamodb:ListTagsOfResource",
"dynamodb:Query",
"dynamodb:UpdateItem",
"dynamodb:PartiQLSelect",
"dynamodb:DescribeTable",
"dynamodb:PartiQLInsert",
"dynamodb:GetItem",
"dynamodb:GetRecords",
"dynamodb:PartiQLDelete"
},
Resources = new []
{
ddbTable.TableArn
}
})
}
})
}));
lambda.CurrentVersion.AddAlias("preprod");
var awsIntegration = StageSpecificLambda(lambda);
api.Root.AddProxy(new ProxyResourceOptions
{
DefaultIntegration = awsIntegration
});
api.Root.AddMethod("GET", awsIntegration);
new RouteToS3(this, "s3route", new RouteToS3Props
{
Api = api,
PathInApiGateway = "content",
S3Bucket = "test-rfsship-content",
S3Prefix = "content",
});
}
public SrcStack(Construct scope, string id, IStackProps props) : base(scope, id, props)
{
#region VPC, Subnets and Security Groups
IVpc batchVpc = new Vpc(this, Constants.VPC_ID, new VpcProps {
Cidr = Constants.CIDR_RANGE,
MaxAzs = 4
});
var privateSubnetIds = new List <string>();
foreach (Subnet subnet in batchVpc.PrivateSubnets)
{
privateSubnetIds.Add(subnet.SubnetId);
}
var batchSecurityGroup = new SecurityGroup(this, Constants.BATCH_SECURITY_GROUP_ID,
new SecurityGroupProps {
Vpc = batchVpc,
SecurityGroupName = Constants.BATCH_SECURITY_GROUP_NAME,
Description = Constants.BATCH_SECURITY_GROUP_DESCRIPTION,
}
);
var batchSecurityGroupIds = new List <string>()
{
batchSecurityGroup.SecurityGroupId
};
#endregion
#region S3, DynamoDB, Lambda (to trigger the batch on file drop)
s3.Bucket bucket = new s3.Bucket(this, Constants.S3_BUCKET_ID, new s3.BucketProps {
BucketName = Constants.S3_BUCKET_NAME + this.Account
});
var bucketName = bucket.BucketName;
var gsi = new dynamo.GlobalSecondaryIndexProps()
{
IndexName = "GSI",
PartitionKey = new dynamo.Attribute {
Name = "CreatedTime", Type = dynamo.AttributeType.STRING
},
ProjectionType = dynamo.ProjectionType.KEYS_ONLY
};
dynamo.Table table = new dynamo.Table(this, Constants.DynamoDB_TABLE_ID, new dynamo.TableProps {
TableName = Constants.DynamoDB_TABLE_ID,
PartitionKey = new dynamo.Attribute {
Name = "ProductId", Type = dynamo.AttributeType.STRING
},
BillingMode = dynamo.BillingMode.PAY_PER_REQUEST
});
table.AddGlobalSecondaryIndex(gsi);
var lambdaBatchExecutionRoleProvider = new LambdaBatchExecutionRole(this, Constants.LAMBDA_BATCH_EXECUTION_ROLE_ID);
#endregion
DockerImageAsset imageAsset = new DockerImageAsset(this, "BatchProcessorImage", new DockerImageAssetProps {
Directory = Constants.BATCH_PROCESSOR_PYTHON_CODE_PATH,
RepositoryName = Constants.ECR_REPOSITORY_NAME
});
#region Batch - ComputeEnvironment - Job Queue - Job Definition
var batchServiceRole = new BatchServiceRole(this, Constants.BATCH_SERVICE_ROLE_ID);
var ecsInstanceRole = new EcsInstanceRole(this, Constants.ECS_INSTANCE_ROLE_ID);
var batchInstanceProfile = new InstanceProfile(this, Constants.BATCH_INSTANCE_PROFILE_ID, ecsInstanceRole);
var computeEnvironment = new batch.CfnComputeEnvironment(this, Constants.BATCH_COMPUTE_ENVIRONMENT_ID,
new batch.CfnComputeEnvironmentProps {
ComputeEnvironmentName = Constants.BATCH_COMPUTE_ENVIRONMENT_NAME,
Type = Constants.BATCH_COMPUTE_TYPE,
ServiceRole = batchServiceRole.Role.RoleName,
ComputeResources = new batch.CfnComputeEnvironment.ComputeResourcesProperty {
Type = Constants.BATCH_COMPUTE_RESOURCE_TYPE,
MinvCpus = 0,
MaxvCpus = 32,
DesiredvCpus = 0,
InstanceRole = ecsInstanceRole.Role.RoleName,
InstanceTypes = new string[] {
Constants.BATCH_INSTANCE_TYPE
},
SecurityGroupIds = batchSecurityGroupIds.ToArray(),
Subnets = privateSubnetIds.ToArray()
}
});
var computeEnvironmentOrders = new List <batch.CfnJobQueue.ComputeEnvironmentOrderProperty>();
var computeEnvironmentOrder = new batch.CfnJobQueue.ComputeEnvironmentOrderProperty()
{
Order = 1,
ComputeEnvironment = computeEnvironment.Ref
};
computeEnvironmentOrders.Add(computeEnvironmentOrder);
var batchProcessingJobQueue = new batch.CfnJobQueue(this, Constants.BATCH_JOB_QUEUE_ID, new batch.CfnJobQueueProps {
JobQueueName = Constants.BATCH_PROCESSING_JOB_QUEUE,
Priority = 1,
ComputeEnvironmentOrder = computeEnvironmentOrders.ToArray()
});
var batchProcessingJobDefinition = new batch.CfnJobDefinition(this, Constants.BATCH_JOB_DEFINITION_ID, new batch.CfnJobDefinitionProps {
Type = Constants.CONTAINER,
JobDefinitionName = Constants.BATCH_JOB_DEFINITION_NAME,
ContainerProperties = new batch.CfnJobDefinition.ContainerPropertiesProperty()
{
Image = imageAsset.ImageUri,
Vcpus = Constants.BATCH_JOB_DEFINITION_VCPU,
Memory = Constants.BATCH_JOB_DEFINITION_MemoryLimitMiB,
Command = new string[] {
"python",
"batch_processor.py"
}
}
});
#endregion
#region lambda s3 event trigger
BatchTriggerLambdaInput batchTriggerLambdaInput = new BatchTriggerLambdaInput {
BucketName = bucketName,
FileName = "sample.csv",
Region = "us-east-1",
BatchJobDefinition = batchProcessingJobDefinition.JobDefinitionName,
BatchJobName = Constants.BATCH_JOB_NAME,
BatchJobQueue = batchProcessingJobQueue.JobQueueName,
DBTable = table.TableName
};
lambda.Function lambda = BatchTriggerLambda.BatchTriggerLambdaFunction(this, Constants.LAMBDA_NAME,
batchTriggerLambdaInput, lambdaBatchExecutionRoleProvider);
var batchLambdaFunction = new LambdaFunction(
lambda
);
var eventPutSource = new eventsources.S3EventSource(bucket, new eventsources.S3EventSourceProps {
Events = new s3.EventType[] {
s3.EventType.OBJECT_CREATED_PUT
}
});
lambda.AddEventSource(eventPutSource);
#endregion
#region OUTPUTS
var cfnComputeOutput = new CfnOutput(this, Constants.BATCH_COMPUTE_ENVIRONMENT_NAME_OUTPUT_ID,
new CfnOutputProps {
Value = computeEnvironment.Ref
}
);
var cfnS3Output = new CfnOutput(this, Constants.S3_BUCKET_OUTPUT_ID,
new CfnOutputProps {
Value = bucket.BucketName
}
);
var cfnEcrRepositoryOutput = new CfnOutput(this, Constants.ECR_REPOSITORY_OUTPUT_ID,
new CfnOutputProps {
Value = imageAsset.Repository.RepositoryArn
}
);
var cfnDynamoTableOutput = new CfnOutput(this, Constants.DYNAMO_TABLE_OUTPUT_ID,
new CfnOutputProps {
Value = table.TableName
}
);
var cfnLambdaRepositoryOutput = new CfnOutput(this, Constants.LAMBDA_OUTPUT_ID,
new CfnOutputProps {
Value = lambda.FunctionArn
}
);
#endregion
Tag.Add(this, "Name", Constants.APP_NAME);
}