/// <summary>
/// Check service health.
/// </summary>
/// <param name="client">Instance of <see cref="AmazonKinesisFirehoseClient"/> class.</param>
/// <returns>Success, RountTripTime.</returns>
public static async Task <(bool, double)> CheckServiceReachable(AmazonKinesisFirehoseClient client)
{
var stopwatch = new Stopwatch();
try
{
stopwatch.Start();
await client.DescribeDeliveryStreamAsync(new DescribeDeliveryStreamRequest
{
DeliveryStreamName = "KinesisTap"
});
stopwatch.Stop();
}
catch (AmazonKinesisFirehoseException)
{
stopwatch.Stop();
// Any exception is fine, we are currently only looking to
// check if the service is reachable and what is the RTT.
return(true, stopwatch.ElapsedMilliseconds);
}
catch (Exception)
{
stopwatch.Stop();
return(false, stopwatch.ElapsedMilliseconds);
}
return(true, stopwatch.ElapsedMilliseconds);
}
public FirehoseSender()
{
var config = new AmazonKinesisFirehoseConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.GetBySystemName(ConfigurationSettings.AppSettings[ConfigurationKeys.Region]);
_client = new AmazonKinesisFirehoseClient(ConfigurationSettings.AppSettings[ConfigurationKeys.AccessKeyId],
ConfigurationSettings.AppSettings[ConfigurationKeys.AccessKeySecret],
config);
}
public void Dispose()
{
try
{
_firehose?.Dispose();
_firehose = null;
}
catch
{
// ignored
}
}
public string PushToFirehose(string stuffToPush)
{
var retVal = string.Empty;
var config = new AmazonKinesisFirehoseConfig();
config.RegionEndpoint = RegionEndpoint.USEast2;
config.ServiceURL = string.Empty;
var client = new AmazonKinesisFirehoseClient(config);
return(retVal);
}
protected IAmazonKinesisFirehose CreateClient(AWSCredentials credentials, RegionEndpoint region)
{
var config = new AmazonKinesisFirehoseConfig {
RegionEndpoint = region
};
Amazon.PowerShell.Utils.Common.PopulateConfig(this, config);
this.CustomizeClientConfig(config);
var client = new AmazonKinesisFirehoseClient(credentials, config);
client.BeforeRequestEvent += RequestEventHandler;
client.AfterResponseEvent += ResponseEventHandler;
return(client);
}
private void updateAmazonKinesisFirehoseClient()
{
bool isOldClientNeedsRecreation = m_ClientCreatedDateTime == DateTime.MinValue ||
DateTime.Now.Subtract(m_ClientCreatedDateTime) >= TimeSpan.FromHours(1);
if (m_shouldInitializeClient && isOldClientNeedsRecreation)
{
try
{
SSMParameter arnParameter = SSMClient.Instance.GetParameter("/FSL/Firehose/ARN");
string firehoseArn = arnParameter?.Value;
SSMParameter regionParameter = SSMClient.Instance.GetParameter(SSMLibrary.Constants.FSL_FIREHOSE_REGION_PATH);
RegionEndpoint firehoseRegionEndpoint = RegionEndpoint.GetBySystemName(regionParameter.Value);
if (string.IsNullOrEmpty(firehoseArn))
{
// Create client without assume role, same account as firehose.
m_Client = new AmazonKinesisFirehoseClient(firehoseRegionEndpoint);
}
else
{
// Create client with assume role
SSMParameter STSRegion = SSMClient.Instance.GetParameter("/FSL/Firehose/STSRegion");
AmazonSecurityTokenServiceClient stsClient = new AmazonSecurityTokenServiceClient(RegionEndpoint.GetBySystemName(STSRegion.Value));
AssumeRoleRequest stsRequest = new AssumeRoleRequest
{
RoleArn = firehoseArn,
RoleSessionName = string.Format("{0}_{1}_Monitoring", m_Product, m_Feature)
};
AssumeRoleResponse stsResult = stsClient.AssumeRoleAsync(stsRequest).Result;
m_Client = new AmazonKinesisFirehoseClient(stsResult.Credentials, firehoseRegionEndpoint);
}
m_ClientCreatedDateTime = DateTime.Now;
}
catch (Exception ex)
{
m_logger.LogError(string.Format("Failed to initialize AWSLogger\n{0}", LogUtils.GetExceptionDetails(ex)));
}
}
}
static void Main()
{
var deliveryStreamName = ConfigurationManager.AppSettings["AWS.Kineses.DeliveryStreamName"];
var millisecondsToSleepBetweenPuts = Convert.ToInt32(ConfigurationManager.AppSettings["MillisecondsToSleepBetweenPuts"]);
var awsRegion = RegionEndpoint.GetBySystemName(ConfigurationManager.AppSettings["AWS.Region"]);
//spit out the delivery stream names available
var firehoseClient = new AmazonKinesisFirehoseClient(awsRegion);
var streams = firehoseClient.ListDeliveryStreams();
Console.WriteLine("Here are your delivery stream names:");
foreach (var stream in streams.DeliveryStreamNames)
{
Console.WriteLine($"{stream}");
}
Console.WriteLine("Starting PutRecord infinite loop:");
while (true)
{
//throw un some dummy data
var data = new FooDataPoint
{
Bar = "wut",
SomeInt = (int)(DateTime.UtcNow.Ticks % 100),
WhenWasIt = DateTime.UtcNow
};
using (var ms = new MemoryStream(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(data))))
{
var response = firehoseClient.PutRecord(new PutRecordRequest {
DeliveryStreamName = deliveryStreamName, Record = new Record {
Data = ms
}
});
Console.WriteLine($"{response.HttpStatusCode} {DateTime.Now:s}");
}
Thread.Sleep(TimeSpan.FromMilliseconds(millisecondsToSleepBetweenPuts));
}
}
static async Task Main(string[] args)
{
const string AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";
const string AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
var firehoseClient = new AmazonKinesisFirehoseClient(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, RegionEndpoint.USWest2);
var message = new Message
{
Name = "Gabriela Orselli",
Document = "22733611810",
Phone = "+5511953494000",
Metadata = new Dictionary <string, string> {
{ "amount", "250.00" }
}
};
var kinesisPublisher = new KinesisPublisherSample(firehoseClient);
var recordid = await kinesisPublisher.PutAsync(message, "poc-firehore-stream");
Console.WriteLine($"Record Id {recordid}");
//var s3Client = new AmazonS3Client(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, RegionEndpoint.USWest2);
//var s3Consumer = new S3ConsumerSample(s3Client);
//var s3ResponseMessage = await s3Consumer.ConsumeAsync("poc-kinesis-firehose", "pkfs-2019/06/14/01/poc-firehore-stream-2-2019-06-14-01-41-18-c4289cfc-b357-4b8f-9a59-28a4d667169c");
//Console.WriteLine($"Reponse document: {s3ResponseMessage.Document}");
//Console.WriteLine($"Reponse document: {s3ResponseMessage.Name}");
//Console.WriteLine($"Reponse document: {s3ResponseMessage.Phone}");
//foreach (var metadata in s3ResponseMessage.Metadata)
//{
// Console.WriteLine($"Reponse metadata: {metadata}");
//}
}
public KinesisPublisherSample(AmazonKinesisFirehoseClient client)
{
_client = client;
}
/// <summary>
/// Kinesis Firehoseへデータを書き込む関数です
/// ローカル開発環境として localStack を使用しています
/// </summary>
/// <param name="input"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task<string> FunctionHandler(FunctionInput input, ILambdaContext context)
{
#region ローカル開発用のコード
// S3 の作成
var creds = new BasicAWSCredentials("dummy", "dummy");
var s3Client = new AmazonS3Client(creds, new AmazonS3Config
{
ServiceURL = "http://localhost:4572",
UseHttp = true,
ForcePathStyle = true,
AuthenticationRegion = "us-east-1",
});
if (!await AmazonS3Util.DoesS3BucketExistAsync(s3Client, _bucket))
{
await s3Client.PutBucketAsync(new PutBucketRequest
{
BucketName = _bucket,
UseClientRegion = true
});
}
// Kinesis Firehose の作成
var request = new CreateDeliveryStreamRequest
{
DeliveryStreamName = _deliveryStream,
ExtendedS3DestinationConfiguration = new ExtendedS3DestinationConfiguration
{
BucketARN = $"arn:aws:s3:::{_bucket}",
Prefix = "firehose/",
RoleARN = "arn:aws:iam::dummy:role/dummy",
}
};
var client = new AmazonKinesisFirehoseClient(creds, new AmazonKinesisFirehoseConfig
{
ServiceURL = _serviceURL,
});
var response = await client.ListDeliveryStreamsAsync();
if (!response.DeliveryStreamNames.Any(_ => _ == _deliveryStream))
{
await client.CreateDeliveryStreamAsync(request);
}
#endregion
var data = JsonConvert.SerializeObject(input);
var record = new Record()
{
Data = new MemoryStream(Encoding.UTF8.GetBytes(data))
};
// Kinesis Firehose に対してデータを書き込む
var res = await client.PutRecordAsync(_deliveryStream, record);
if (res.HttpStatusCode == HttpStatusCode.OK)
{
return $"RecordId: {res.RecordId}";
}
else
{
return "Error";
}
}
public FirehoseClient(AmazonKinesisFirehoseClient client, DateTime validUntil)
{
_firehose = client;
_validUntil = validUntil;
}
public FirehoseClient(AmazonKinesisFirehoseClient client)
{
_firehose = client;
_validUntil = DateTime.MaxValue;
}
/// <summary>
/// A simple function that takes a string and does a ToUpper
/// </summary>
/// <param name="input"></param>
/// <param name="context"></param>
/// <returns></returns>
public APIGatewayProxyResponse FunctionHandler(APIGatewayProxyRequest request, ILambdaContext context)
{
APIGatewayProxyResponse APIResponse = new APIGatewayProxyResponse();
try
{
bool EVSendToFirehose = Environment.GetEnvironmentVariable("PUSH_TO_FIREHOSE").ToUpper() == "TRUE";
if (EVSendToFirehose) // Only Processes if SentToFirehose==True
{
/*Json to Telemetry List*/
TelemetryList telemetryList = JsonConvert.DeserializeObject <TelemetryList>(request.Body);
int ItemCount = telemetryList.Telemetry.Count;
/*Get Environment Vars*/
string EVDeliveryStream = Environment.GetEnvironmentVariable("FIREHOSE_DELIVERYSTREAM");
string EVRegion = Environment.GetEnvironmentVariable("REGION");
byte[] oByte;
Amazon.RegionEndpoint Region = Amazon.RegionEndpoint.GetBySystemName(EVRegion);
foreach (Telemetry telemetry in telemetryList.Telemetry)
{
telemetry.Instant = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
oByte = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(telemetry));
using (MemoryStream ms = new MemoryStream(oByte))
{
Record RecordToSend = new Record
{
Data = ms
};
PutRecordRequest DeliveryRequest = new PutRecordRequest
{
DeliveryStreamName = EVDeliveryStream, //DigitalPlatformTelemetry
Record = RecordToSend
};
AmazonKinesisFirehoseClient FirehoseClient = new AmazonKinesisFirehoseClient(Region);
PutRecordResponse Response = FirehoseClient.PutRecord(DeliveryRequest);
}
}
APIResponse.StatusCode = 200;
APIResponse.Body = "Sucess, processed " + ItemCount + " items " + DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
return(APIResponse);
}
APIResponse.StatusCode = 200;
APIResponse.Body = APIResponse.Body = "Sucess, no entries processed ( PUSH_TO_FIREHOSE = False ). " + DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
return(APIResponse);
}
catch (InvalidCastException e)
{
APIResponse.StatusCode = 500;
APIResponse.Body = e.Message;
return(APIResponse);
}
}
