public static void ExecuteWithNoUi(LocalLambdaOptions localLambdaOptions, CommandLineOptions commandOptions, string lambdaAssemblyDirectory, RunConfiguration runConfiguration)
{
runConfiguration.OutputWriter.WriteLine("Executing Lambda function without web interface");
var lambdaProjectDirectory = Utils.FindLambdaProjectDirectory(lambdaAssemblyDirectory);
string configFile = DetermineConfigFile(commandOptions, lambdaAssemblyDirectory: lambdaAssemblyDirectory, lambdaProjectDirectory: lambdaProjectDirectory);
LambdaConfigInfo configInfo = LoadLambdaConfigInfo(configFile, commandOptions, lambdaAssemblyDirectory: lambdaAssemblyDirectory, lambdaProjectDirectory: lambdaProjectDirectory, runConfiguration);
LambdaFunction lambdaFunction = LoadLambdaFunction(configInfo, localLambdaOptions, commandOptions, lambdaAssemblyDirectory: lambdaAssemblyDirectory, lambdaProjectDirectory: lambdaProjectDirectory, runConfiguration);
string payload = DeterminePayload(localLambdaOptions, commandOptions, lambdaAssemblyDirectory: lambdaAssemblyDirectory, lambdaProjectDirectory: lambdaProjectDirectory, runConfiguration);
var awsProfile = commandOptions.AWSProfile ?? configInfo.AWSProfile;
if (!string.IsNullOrEmpty(awsProfile))
{
if (new Amazon.Runtime.CredentialManagement.CredentialProfileStoreChain().TryGetProfile(awsProfile, out _))
{
runConfiguration.OutputWriter.WriteLine($"... Setting AWS_PROFILE environment variable to {awsProfile}.");
}
else
{
runConfiguration.OutputWriter.WriteLine($"... Warning: Profile {awsProfile} not found in the aws credential store.");
awsProfile = null;
}
}
else
{
runConfiguration.OutputWriter.WriteLine("... No profile choosen for AWS credentials. The --profile switch can be used to configure an AWS profile.");
}
var awsRegion = commandOptions.AWSRegion ?? configInfo.AWSRegion;
if (!string.IsNullOrEmpty(awsRegion))
{
runConfiguration.OutputWriter.WriteLine($"... Setting AWS_REGION environment variable to {awsRegion}.");
}
else
{
runConfiguration.OutputWriter.WriteLine("... No default AWS region configured. The --region switch can be used to configure an AWS Region.");
}
// Create the execution request that will be sent into the LocalLambdaRuntime.
var request = new ExecutionRequest()
{
AWSProfile = awsProfile,
AWSRegion = awsRegion,
Payload = payload,
Function = lambdaFunction
};
ExecuteRequest(request, localLambdaOptions, runConfiguration);
if (runConfiguration.Mode == RunConfiguration.RunMode.Normal && commandOptions.PauseExit)
{
Console.WriteLine("Press any key to exist");
Console.ReadKey();
}
}
private async Task <APIGatewayProxyResponse> InvokeAPIGatewayRequest(TestLambdaContext context, string fileName)
{
var lambdaFunction = new LambdaFunction();
var filePath = Path.Combine(Path.GetDirectoryName(this.GetType().GetTypeInfo().Assembly.Location), fileName);
var requestStr = File.ReadAllText(filePath);
var request = JsonConvert.DeserializeObject <APIGatewayProxyRequest>(requestStr);
return(await lambdaFunction.FunctionHandlerAsync(request, context));
}
private async Task <APIGatewayProxyResponse> InvokeAPIGatewayRequest(string fileName)
{
var context = new TestLambdaContext();
var lambdaFunction = new LambdaFunction();
var requestStr = File.ReadAllText(fileName);
var request = JsonConvert.DeserializeObject <APIGatewayProxyRequest>(requestStr);
return(await lambdaFunction.FunctionHandlerAsync(request, context));
}
public void TestToUpperFunction()
{
// Invoke the lambda function and confirm the string was upper cased.
var function = new LambdaFunction();
var context = new TestLambdaContext();
var upperCase = function.FunctionHandler("hello world", context);
Assert.Equal("HELLO WORLD", upperCase);
}
public async Task TestGetQueryStringValue()
{
var lambdaFunction = new LambdaFunction();
var requestStr = File.ReadAllText("values-get-querystring-apigatway-request.json");
var request = JsonConvert.DeserializeObject <APIGatewayProxyRequest>(requestStr);
var response = await lambdaFunction.FunctionHandlerAsync(request, null);
Assert.Equal("Lewis, Meriwether", response.Body);
Assert.True(response.Headers.ContainsKey("Content-Type"));
Assert.Equal("text/plain; charset=utf-8", response.Headers["Content-Type"]);
}
public async Task TestGetAllValues()
{
var lambdaFunction = new LambdaFunction();
var requestStr = File.ReadAllText("values-get-all-apigatway-request.json");
var request = JsonConvert.DeserializeObject <APIGatewayProxyRequest>(requestStr);
var response = await lambdaFunction.FunctionHandlerAsync(request, null);
Assert.Equal("[\"value1\",\"value2\"]", response.Body);
Assert.True(response.Headers.ContainsKey("Content-Type"));
Assert.Equal("application/json; charset=utf-8", response.Headers["Content-Type"]);
}
internal LambdaStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
var config = new LambdaConfiguration()
{
FunctionName = id,
Resource = "./resources/function"
};
var lambda = new LambdaFunction(this);
lambda.Construct(config);
}
public void ProcessStartAndRun()
{
AutoResetEvent handle = new AutoResetEvent(false);
int count = 0;
IFunction function = new LambdaFunction(() => { count = 1; return(handle.Set()); });
Process process = new Process();
process.Start(function);
handle.WaitOne();
Assert.AreEqual(1, count);
}
public void ProcessStartAndRun()
{
AutoResetEvent handle = new AutoResetEvent(false);
int count = 0;
IFunction function = new LambdaFunction(() => { count = 1; return handle.Set(); });
Process process = new Process();
process.Start(function);
handle.WaitOne();
Assert.AreEqual(1, count);
}
public async Task TestPutWithBody()
{
var lambdaFunction = new LambdaFunction();
var requestStr = File.ReadAllText("values-put-withbody-apigatway-request.json");
var request = JsonConvert.DeserializeObject <APIGatewayProxyRequest>(requestStr);
var response = await lambdaFunction.FunctionHandlerAsync(request, null);
Assert.Equal(200, response.StatusCode);
Assert.Equal("Agent, Smith", response.Body);
Assert.True(response.Headers.ContainsKey("Content-Type"));
Assert.Equal("text/plain; charset=utf-8", response.Headers["Content-Type"]);
}
public void SpawnSimpleProcess()
{
AutoResetEvent handle = new AutoResetEvent(false);
int count = 0;
IFunction func = new SpawnFunction();
IFunction lambda = new LambdaFunction(() => { count = 1; return(handle.Set()); });
var result = func.Apply(null, new object[] { lambda });
Assert.IsNotNull(result);
Assert.IsInstanceOfType(result, typeof(Process));
handle.WaitOne();
Assert.AreEqual(1, count);
}
public void SpawnSimpleProcess()
{
AutoResetEvent handle = new AutoResetEvent(false);
int count = 0;
IFunction func = new SpawnFunction();
IFunction lambda = new LambdaFunction(() => { count = 1; return handle.Set(); });
var result = func.Apply(null, new object[] { lambda });
Assert.IsNotNull(result);
Assert.IsInstanceOfType(result, typeof(Process));
handle.WaitOne();
Assert.AreEqual(1, count);
}
/// <summary>
/// Starting point: define stack in this method
/// </summary>
/// <param name="scope"></param>
/// <param name="id"></param>
/// <param name="props"></param>
/// <returns></returns>
internal AirQualityCdkStack(Construct scope, string id, IStackProps props = null) : base(scope, id, props)
{
// Metadata to be used by internal construct methods - might need a bit of refactoring and better design
// this holds keyvaluePairs which can be used to configure environmentVariables
var metadata = new Dictionary <string, KeyValuePair <string, string> >();
// The code that defines your stack goes here
// SNS Topics
new Topic(this, "EmailNotificationPublisher");
// SQS queues
var cityFeedSqs = AirQualitySQS("CityFeedQueue");
// SNS Topics
var twitterPubSNS = new Topic(this, $"twitterPub-{Guid.NewGuid().ToString().Substring(0,10)}");
var generalPubSNS = new Topic(this, $"generalPub-{Guid.NewGuid().ToString().Substring(0,10)}");
// Dynamo DB table
var tableProps = new Dynamo.TableProps();
tableProps.TableName = Constants.DbAirQualityTableName;
var primaryKey = new Dynamo.Attribute {
Name = Constants.DbPartitionKeyName, Type = Dynamo.AttributeType.STRING
};
tableProps.BillingMode = Dynamo.BillingMode.PAY_PER_REQUEST; // on-demand pricing
tableProps.PartitionKey = primaryKey;
tableProps.Stream = Dynamo.StreamViewType.NEW_IMAGE;
var airqualityTable = new Dynamo.Table(this, Constants.DbAirQualityTableName, tableProps);
//Environment value metadata for lambda creation
metadata[Constants.EnvCityFeedSqsUrl] = KeyValuePair.Create(Constants.EnvCityFeedSqsUrl, cityFeedSqs.QueueUrl);
metadata[Constants.EnvTwitterSNS] = KeyValuePair.Create(Constants.EnvTwitterSNS, twitterPubSNS.TopicArn);
metadata[Constants.EnvTwitterAPIKey] = KeyValuePair.Create(Constants.EnvTwitterAPIKey, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterAPIKey));
metadata[Constants.EnvTwitterAPISecret] = KeyValuePair.Create(Constants.EnvTwitterAPISecret, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterAPISecret));
metadata[Constants.EnvTwitterAccessToken] = KeyValuePair.Create(Constants.EnvTwitterAccessToken, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterAccessToken));
metadata[Constants.EnvTwitterTokenSecret] = KeyValuePair.Create(Constants.EnvTwitterTokenSecret, System.Environment.GetEnvironmentVariable(Constants.EnvTwitterTokenSecret));
metadata[Constants.EnvAirQualityTable] = KeyValuePair.Create(Constants.EnvAirQualityTable, airqualityTable.TableName);
metadata[Constants.EnvGeneralSNSTopic] = KeyValuePair.Create(Constants.EnvGeneralSNSTopic, generalPubSNS.TopicArn);
// AWS lambdas
// TODO: Lambda generation logic to be done using environment values. Config might also be pulled from S3
// GetCityFeed lambda
var functionsByName = ConstructLambdas(metadata);
// subscribe and configure
// find twitter publisher lambda
twitterPubSNS.AddSubscription(new LambdaSubscription(functionsByName
.Where(pair => pair.Key.Contains(Constants.DefaultAirQualityTwitterPublisherLambdaName))
.Single().Value));
// trigger feed processor lambda only on dynamo db stream change by coutry-city feed
functionsByName
.Where(pair => pair.Key.Contains(Constants.DefaultAirQualityFeedProcessorLambdaName))
.Single().Value.AddEventSource(new DynamoEventSource(airqualityTable, new DynamoEventSourceProps()));
// Event schedule every 15 mins - // architecture targets 5, change when tested and working
var waqiLambda = functionsByName
.Where(pair => pair.Key.Contains(Constants.DefaultWaqiGetCityFeedLambdaName))
.Single().Value;
var targetWaqiLambda = new LambdaFunction(waqiLambda);
new Rule(this, "Waqi5min", new RuleProps {
Schedule = Schedule.Rate(Duration.Minutes(5))
}).AddTarget(targetWaqiLambda);
}
public bool TryLoadLambdaFuntion(LambdaConfigInfo configInfo, string functionHandler, out LambdaFunction lambdaFunction)
{
lambdaFunction = null;
try
{
lambdaFunction = LoadLambdaFuntion(configInfo, functionHandler);
return(true);
}
catch { }
return(false);
}
public SrcStack(Construct scope, string id, IStackProps props) : base(scope, id, props)
{
#region VPC, Subnets and Security Groups
IVpc batchVpc = new Vpc(this, Constants.VPC_ID, new VpcProps {
Cidr = Constants.CIDR_RANGE,
MaxAzs = 4
});
var privateSubnetIds = new List <string>();
foreach (Subnet subnet in batchVpc.PrivateSubnets)
{
privateSubnetIds.Add(subnet.SubnetId);
}
var batchSecurityGroup = new SecurityGroup(this, Constants.BATCH_SECURITY_GROUP_ID,
new SecurityGroupProps {
Vpc = batchVpc,
SecurityGroupName = Constants.BATCH_SECURITY_GROUP_NAME,
Description = Constants.BATCH_SECURITY_GROUP_DESCRIPTION,
}
);
var batchSecurityGroupIds = new List <string>()
{
batchSecurityGroup.SecurityGroupId
};
#endregion
#region S3, DynamoDB, Lambda (to trigger the batch on file drop)
s3.Bucket bucket = new s3.Bucket(this, Constants.S3_BUCKET_ID, new s3.BucketProps {
BucketName = Constants.S3_BUCKET_NAME + this.Account
});
var bucketName = bucket.BucketName;
var gsi = new dynamo.GlobalSecondaryIndexProps()
{
IndexName = "GSI",
PartitionKey = new dynamo.Attribute {
Name = "CreatedTime", Type = dynamo.AttributeType.STRING
},
ProjectionType = dynamo.ProjectionType.KEYS_ONLY
};
dynamo.Table table = new dynamo.Table(this, Constants.DynamoDB_TABLE_ID, new dynamo.TableProps {
TableName = Constants.DynamoDB_TABLE_ID,
PartitionKey = new dynamo.Attribute {
Name = "ProductId", Type = dynamo.AttributeType.STRING
},
BillingMode = dynamo.BillingMode.PAY_PER_REQUEST
});
table.AddGlobalSecondaryIndex(gsi);
var lambdaBatchExecutionRoleProvider = new LambdaBatchExecutionRole(this, Constants.LAMBDA_BATCH_EXECUTION_ROLE_ID);
#endregion
DockerImageAsset imageAsset = new DockerImageAsset(this, "BatchProcessorImage", new DockerImageAssetProps {
Directory = Constants.BATCH_PROCESSOR_PYTHON_CODE_PATH,
RepositoryName = Constants.ECR_REPOSITORY_NAME
});
#region Batch - ComputeEnvironment - Job Queue - Job Definition
var batchServiceRole = new BatchServiceRole(this, Constants.BATCH_SERVICE_ROLE_ID);
var ecsInstanceRole = new EcsInstanceRole(this, Constants.ECS_INSTANCE_ROLE_ID);
var batchInstanceProfile = new InstanceProfile(this, Constants.BATCH_INSTANCE_PROFILE_ID, ecsInstanceRole);
var computeEnvironment = new batch.CfnComputeEnvironment(this, Constants.BATCH_COMPUTE_ENVIRONMENT_ID,
new batch.CfnComputeEnvironmentProps {
ComputeEnvironmentName = Constants.BATCH_COMPUTE_ENVIRONMENT_NAME,
Type = Constants.BATCH_COMPUTE_TYPE,
ServiceRole = batchServiceRole.Role.RoleName,
ComputeResources = new batch.CfnComputeEnvironment.ComputeResourcesProperty {
Type = Constants.BATCH_COMPUTE_RESOURCE_TYPE,
MinvCpus = 0,
MaxvCpus = 32,
DesiredvCpus = 0,
InstanceRole = ecsInstanceRole.Role.RoleName,
InstanceTypes = new string[] {
Constants.BATCH_INSTANCE_TYPE
},
SecurityGroupIds = batchSecurityGroupIds.ToArray(),
Subnets = privateSubnetIds.ToArray()
}
});
var computeEnvironmentOrders = new List <batch.CfnJobQueue.ComputeEnvironmentOrderProperty>();
var computeEnvironmentOrder = new batch.CfnJobQueue.ComputeEnvironmentOrderProperty()
{
Order = 1,
ComputeEnvironment = computeEnvironment.Ref
};
computeEnvironmentOrders.Add(computeEnvironmentOrder);
var batchProcessingJobQueue = new batch.CfnJobQueue(this, Constants.BATCH_JOB_QUEUE_ID, new batch.CfnJobQueueProps {
JobQueueName = Constants.BATCH_PROCESSING_JOB_QUEUE,
Priority = 1,
ComputeEnvironmentOrder = computeEnvironmentOrders.ToArray()
});
var batchProcessingJobDefinition = new batch.CfnJobDefinition(this, Constants.BATCH_JOB_DEFINITION_ID, new batch.CfnJobDefinitionProps {
Type = Constants.CONTAINER,
JobDefinitionName = Constants.BATCH_JOB_DEFINITION_NAME,
ContainerProperties = new batch.CfnJobDefinition.ContainerPropertiesProperty()
{
Image = imageAsset.ImageUri,
Vcpus = Constants.BATCH_JOB_DEFINITION_VCPU,
Memory = Constants.BATCH_JOB_DEFINITION_MemoryLimitMiB,
Command = new string[] {
"python",
"batch_processor.py"
}
}
});
#endregion
#region lambda s3 event trigger
BatchTriggerLambdaInput batchTriggerLambdaInput = new BatchTriggerLambdaInput {
BucketName = bucketName,
FileName = "sample.csv",
Region = "us-east-1",
BatchJobDefinition = batchProcessingJobDefinition.JobDefinitionName,
BatchJobName = Constants.BATCH_JOB_NAME,
BatchJobQueue = batchProcessingJobQueue.JobQueueName,
DBTable = table.TableName
};
lambda.Function lambda = BatchTriggerLambda.BatchTriggerLambdaFunction(this, Constants.LAMBDA_NAME,
batchTriggerLambdaInput, lambdaBatchExecutionRoleProvider);
var batchLambdaFunction = new LambdaFunction(
lambda
);
var eventPutSource = new eventsources.S3EventSource(bucket, new eventsources.S3EventSourceProps {
Events = new s3.EventType[] {
s3.EventType.OBJECT_CREATED_PUT
}
});
lambda.AddEventSource(eventPutSource);
#endregion
#region OUTPUTS
var cfnComputeOutput = new CfnOutput(this, Constants.BATCH_COMPUTE_ENVIRONMENT_NAME_OUTPUT_ID,
new CfnOutputProps {
Value = computeEnvironment.Ref
}
);
var cfnS3Output = new CfnOutput(this, Constants.S3_BUCKET_OUTPUT_ID,
new CfnOutputProps {
Value = bucket.BucketName
}
);
var cfnEcrRepositoryOutput = new CfnOutput(this, Constants.ECR_REPOSITORY_OUTPUT_ID,
new CfnOutputProps {
Value = imageAsset.Repository.RepositoryArn
}
);
var cfnDynamoTableOutput = new CfnOutput(this, Constants.DYNAMO_TABLE_OUTPUT_ID,
new CfnOutputProps {
Value = table.TableName
}
);
var cfnLambdaRepositoryOutput = new CfnOutput(this, Constants.LAMBDA_OUTPUT_ID,
new CfnOutputProps {
Value = lambda.FunctionArn
}
);
#endregion
Tag.Add(this, "Name", Constants.APP_NAME);
}
