public KinesisFirehoseStack(Construct parent, string id, KinesisFirehoseStackProps props) : base(parent, id,
props)
{
var clicksDestinationBucket = new Bucket(this, "Bucket", new BucketProps
{
Versioned = true
});
var firehoseDeliveryRole = new Role(this, "FirehoseDeliveryRole", new RoleProps
{
RoleName = "FirehoseDeliveryRole",
AssumedBy = new ServicePrincipal("firehose.amazonaws.com"),
ExternalIds = new string[]
{
Aws.ACCOUNT_ID
}
});
var firehoseDeliveryPolicyS3Stm = new PolicyStatement();
firehoseDeliveryPolicyS3Stm.AddActions("s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject");
firehoseDeliveryPolicyS3Stm.AddResources(clicksDestinationBucket.BucketArn);
firehoseDeliveryPolicyS3Stm.AddResources(clicksDestinationBucket.ArnForObjects("*"));
var lambdaFunctionPolicy = new PolicyStatement();
lambdaFunctionPolicy.AddActions("dynamodb:GetItem");
lambdaFunctionPolicy.AddResources(props.TableArn);
var LambdaFunctionPolicyStmXRay = new PolicyStatement();
LambdaFunctionPolicyStmXRay.AddActions(
// Allows the Lambda function to interact with X-Ray
"xray:PutTraceSegments",
"xray:PutTelemetryRecords",
"xray:GetSamplingRules",
"xray:GetSamplingTargets",
"xray:GetSamplingStatisticSummaries"
);
LambdaFunctionPolicyStmXRay.AddAllResources();
var mysfitsClicksProcessor = new Function(this, "Function", new FunctionProps
{
Handler = "streaming_lambda::streaming_lambda.function::FunctionHandlerAsync",
Runtime = Runtime.DOTNET_CORE_2_1,
Description = "An Amazon Kinesis Firehose stream processor that enriches click records" +
" to not just include a mysfitId, but also other attributes that can be analyzed later.",
MemorySize = 128,
Code = Code.FromAsset("../lambda/stream/bin/Debug/netcoreapp2.1/Publish"),
Timeout = Duration.Seconds(30),
Tracing = Tracing.ACTIVE,
InitialPolicy = new PolicyStatement[]
{
lambdaFunctionPolicy,
LambdaFunctionPolicyStmXRay
},
Environment = new Dictionary <string, string>()
{
{
"mysfits_api_url",
string.Format($"https://${props.APIid}.execute-api.{Amazon.CDK.Aws.REGION}.amazonaws.com/prod/")
}
}
});
var firehoseDeliveryPolicyLambdaStm = new PolicyStatement();
firehoseDeliveryPolicyLambdaStm.AddActions("lambda:InvokeFunction");
firehoseDeliveryPolicyLambdaStm.AddResources(mysfitsClicksProcessor.FunctionArn);
firehoseDeliveryRole.AddToPolicy(firehoseDeliveryPolicyS3Stm);
firehoseDeliveryRole.AddToPolicy(firehoseDeliveryPolicyLambdaStm);
var mysfitsFireHoseToS3 = new CfnDeliveryStream(this, "DeliveryStream", new CfnDeliveryStreamProps
{
ExtendedS3DestinationConfiguration = new CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty()
{
BucketArn = clicksDestinationBucket.BucketArn,
BufferingHints = new CfnDeliveryStream.BufferingHintsProperty
{
IntervalInSeconds = 60,
SizeInMBs = 50
},
CompressionFormat = "UNCOMPRESSED",
Prefix = "firehose/",
RoleArn = firehoseDeliveryRole.RoleArn,
ProcessingConfiguration = new CfnDeliveryStream.ProcessingConfigurationProperty
{
Enabled = true,
Processors = new CfnDeliveryStream.ProcessorProperty[]
{
new CfnDeliveryStream.ProcessorProperty()
{
Type = "Lambda",
Parameters = new CfnDeliveryStream.ProcessorParameterProperty
{
ParameterName = "LambdaArn",
ParameterValue = mysfitsClicksProcessor.FunctionArn
}
}
}
}
}
});
new CfnPermission(this, "Permission", new CfnPermissionProps
{
Action = "lambda:InvokeFunction",
FunctionName = mysfitsClicksProcessor.FunctionArn,
Principal = "firehose.amazonaws.com",
SourceAccount = Amazon.CDK.Aws.ACCOUNT_ID,
SourceArn = mysfitsFireHoseToS3.AttrArn
});
var clickProcessingApiRole = new Role(this, "ClickProcessingApiRole", new RoleProps
{
AssumedBy = new ServicePrincipal("apigateway.amazonaws.com")
});
var apiPolicy = new PolicyStatement();
apiPolicy.AddActions("firehose:PutRecord");
apiPolicy.AddResources(mysfitsFireHoseToS3.AttrArn);
new Policy(this, "ClickProcessingApiPolicy", new PolicyProps
{
PolicyName = "api_gateway_firehose_proxy_role",
Statements = new PolicyStatement[]
{
apiPolicy
},
Roles = new[] { clickProcessingApiRole }
});
var api = new RestApi(this, "APIEndpoint", new RestApiProps
{
RestApiName = "ClickProcessing API Service",
CloudWatchRole = false,
EndpointTypes = new EndpointType[]
{
EndpointType.REGIONAL
}
});
var clicks = api.Root.AddResource("clicks");
clicks.AddMethod("PUT", new AwsIntegration(new AwsIntegrationProps
{
Service = "firehose",
IntegrationHttpMethod = "POST",
Action = "PutRecord",
Options = new IntegrationOptions
{
ConnectionType = ConnectionType.INTERNET,
CredentialsRole = clickProcessingApiRole,
IntegrationResponses = new IntegrationResponse[]
{
new IntegrationResponse()
{
StatusCode = "200",
ResponseTemplates =
{
{ "application/json", "{\"status\":\"OK\"}" }
},
ResponseParameters =
{
{ "method.response.header.Access-Control-Allow-Headers", "'Content-Type'" },
{ "method.response.header.Access-Control-Allow-Methods", "'OPTIONS,PUT'" },
{ "method.response.header.Access-Control-Allow-Origin", "'*'" }
}
}
},
RequestParameters =
{
{ "integration.request.header.Content-Type", "'application/x-amz-json-1.1'" }
},
RequestTemplates =
{
{
"application/json",
"{\"DeliveryStreamName\":\"" + mysfitsFireHoseToS3.Ref +
"\", \"Record\": { \"Data\": \"$util.base64Encode($input.json('$'))\"}"
}
}
}
}),
new MethodOptions
{
MethodResponses = new MethodResponse[]
{
new MethodResponse
{
StatusCode = "200",
ResponseParameters =
{
{ "method.response.header.Access-Control-Allow-Headers", true },
{ "method.response.header.Access-Control-Allow-Methods", true },
{ "method.response.header.Access-Control-Allow-Origin", true }
}
}
}
});
clicks.AddMethod("OPTIONS", new MockIntegration(new IntegrationOptions
{
IntegrationResponses = new IntegrationResponse[]
{
new IntegrationResponse
{
StatusCode = "200",
ResponseParameters = new Dictionary <string, string>
{
{
"method.response.header.Access-Control-Allow-Headers",
"'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token,X-Amz-User-Agent'"
},
{ "method.response.header.Access-Control-Allow-Origin", "'*'" },
{ "method.response.header.Access-Control-Allow-Credentials", "'false'" },
{ "method.response.header.Access-Control-Allow-Methods", "'OPTIONS,GET,PUT,POST,DELETE'" }
}
}
},
PassthroughBehavior = PassthroughBehavior.NEVER,
RequestTemplates = new Dictionary <string, string>
{
{ "application/json", "{\"statusCode\": 200}" }
}
}),
new MethodOptions
{
MethodResponses = new MethodResponse[]
{
new MethodResponse
{
StatusCode = "200",
ResponseParameters = new Dictionary <string, bool>
{
{ "method.response.header.Access-Control-Allow-Headers", true },
{ "method.response.header.Access-Control-Allow-Methods", true },
{ "method.response.header.Access-Control-Allow-Credentials", true },
{ "method.response.header.Access-Control-Allow-Origin", true }
}
}
}
});
}
public void CreateLogConsumerResources()
{
//LambdaRole
var firehoseLambdaRole = new Role(this, "FirehoseLambdaRole", new RoleProps
{
AssumedBy = new ServicePrincipal("lambda.amazonaws.com"),
Path = "/",
});
firehoseLambdaRole.AddToPolicy(new PolicyStatement(new PolicyStatementProps
{
Effect = Effect.ALLOW,
Resources = new string[] { "arn:aws:logs:*:*:*" },
Actions = new string[] { "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents" },
}));
//FirehoseDataProcessingFunction
var handler = new Function(this, "FirehoseDataProcessorFunction", new FunctionProps
{
FunctionName = "data-processor-function",
Runtime = Runtime.NODEJS_12_X,
Code = Code.FromAsset("resources"),
Handler = "index.handler",
Role = firehoseLambdaRole,
Timeout = Duration.Minutes(2)
});
//FirehoseDeliveryRole & Policies
var firehoseDeliveryRole = new Role(this, "FirehoseDeliveryRole", new RoleProps
{
AssumedBy = new ServicePrincipal("firehose.amazonaws.com"),
Path = "/"
});
//S3 permissions
firehoseDeliveryRole.AddToPolicy(new PolicyStatement(new PolicyStatementProps
{
Effect = Effect.ALLOW,
Resources = new string[] { _logsBucket.BucketArn, _logsBucket.BucketArn + "/*" },
Actions = new string[] { "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject"
, "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" },
}));
//Lambda permissions
firehoseDeliveryRole.AddToPolicy(new PolicyStatement(new PolicyStatementProps
{
Effect = Effect.ALLOW,
Resources = new string[] { handler.FunctionArn },
Actions = new string[] { "lambda:GetFunctionConfiguration", "lambda:InvokeFunction" },
}));
//Log group for Firehose logs.
var firehoseloggroup = new LogGroup(this, "firehoseloggroup", new LogGroupProps
{
LogGroupName = "central-logs-delivery-group"
});
var logstream = new LogStream(this, "logstream", new LogStreamProps
{
LogStreamName = "central-logs-delivery-stream",
LogGroup = firehoseloggroup
});
firehoseDeliveryRole.AddToPolicy(new PolicyStatement(new PolicyStatementProps
{
Effect = Effect.ALLOW,
Resources = new string[] { firehoseloggroup.LogGroupArn },
Actions = new string[] { "logs:PutLogEvents" },
}));
//FirehoseLoggingDeliveryStream - Start
CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty s3config = new CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty();
s3config.BucketArn = _logsBucket.BucketArn;
s3config.BufferingHints = new CfnDeliveryStream.BufferingHintsProperty
{
SizeInMBs = 50,
IntervalInSeconds = 300
};
s3config.CompressionFormat = "UNCOMPRESSED";
s3config.RoleArn = firehoseDeliveryRole.RoleArn;
s3config.Prefix = "CentralLogs/AWSLogs/";
s3config.ErrorOutputPrefix = "CentralLogs/AWSLogs/Error/";
var parameters = new CfnDeliveryStream.ProcessorParameterProperty();
parameters.ParameterName = "LambdaArn";
parameters.ParameterValue = handler.FunctionArn;
var paramsArray1 = new CfnDeliveryStream.ProcessorParameterProperty[] { parameters };
var processorProperty = new CfnDeliveryStream.ProcessorProperty();
processorProperty.Parameters = paramsArray1;
processorProperty.Type = "Lambda";
var paramsArray = new CfnDeliveryStream.ProcessorProperty[] { processorProperty };
s3config.ProcessingConfiguration = new CfnDeliveryStream.ProcessingConfigurationProperty
{
Enabled = true,
Processors = paramsArray
};
s3config.CloudWatchLoggingOptions = new CfnDeliveryStream.CloudWatchLoggingOptionsProperty
{
Enabled = true,
LogGroupName = firehoseloggroup.LogGroupName,
LogStreamName = logstream.LogStreamName
};
CfnDeliveryStream firehoseDeliveryStream = new CfnDeliveryStream(this, "FirehoseLoggingDeliveryStream", new CfnDeliveryStreamProps
{
DeliveryStreamType = "DirectPut",
ExtendedS3DestinationConfiguration = s3config
});
//FirehoseLoggingDeliveryStream - End
//Policy Statements for LogDestination- start
var policyStmt = new PolicyStatement(new PolicyStatementProps()
{
Actions = new string[] { "firehose:PutRecord" },
Resources = new string[] { "*" },
Effect = Effect.ALLOW
});
var policyDoc = new PolicyDocument();
policyDoc.AddStatements(new PolicyStatement[] { policyStmt });
var policyProp = new CfnRole.PolicyProperty();
policyProp.PolicyName = "logDestinationPolicy";
policyProp.PolicyDocument = policyDoc;
//Policy Statements - end
//AssumeRolePolicyDocument for LogDestination - start
var principal = new ServicePrincipal("logs.amazonaws.com");
var assumePolicyStatement = new PolicyStatement(new PolicyStatementProps
{
Actions = new string[] { "sts:AssumeRole" },
Effect = Effect.ALLOW,
Principals = new IPrincipal[] { principal }
});
var assumePolicyDoc = new PolicyDocument();
assumePolicyDoc.AddStatements(new PolicyStatement[] { assumePolicyStatement });
//AssumeRolePolicyDocument - end
var roleProps = new CfnRoleProps {
Path = "/",
AssumeRolePolicyDocument = assumePolicyDoc,
Policies = new CfnRole.PolicyProperty[] { policyProp }
};
CfnRole cfnRole = new CfnRole(this, "CfnRole", roleProps);
CfnDestination logDestination = new CfnDestination(this, "LogDestination", new CfnDestinationProps
{
DestinationName = "Central-Log-Destination",
RoleArn = cfnRole.AttrArn,
TargetArn = firehoseDeliveryStream.AttrArn,
DestinationPolicy = "{\"Version\" : \"2012-10-17\",\"Statement\" : [{\"Effect\" : \"Allow\", \"Principal\" : {\"AWS\" : [\"" + SourceLogAccountId + "\"]},\"Action\" : \"logs:PutSubscriptionFilter\", \"Resource\" : \"arn:aws:logs:" + this.Region + ":"
+ DestinationAccountId + ":destination:Central-Log-Destination\"}]}"
});
logDestination.AddDependsOn(firehoseDeliveryStream);
logDestination.AddDependsOn(cfnRole);
Console.WriteLine(logDestination.DestinationPolicy);
LogDestinationArn = logDestination.AttrArn;
CfnOutput output = new CfnOutput(this, "LogDestinationARN", new CfnOutputProps {
Description = "LogDestination ARN",
Value = logDestination.AttrArn
});
}
