public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
DescribeStreamResponse resp = new DescribeStreamResponse();
do
{
DescribeStreamRequest req = new DescribeStreamRequest
{
ExclusiveStartShardId = resp.StreamDescription.Shards[0].ShardId
,
Limit = maxItems
};
resp = client.DescribeStream(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamDescription.Shards)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.StreamDescription.Shards[0].ShardId));
}
private static void SendWebTransactionsToQueue(List <WebTransaction> transactions)
{
Amazon.Kinesis.AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
String kinesisStreamName = ConfigurationManager.AppSettings["KinesisStreamName"];
foreach (WebTransaction wt in transactions)
{
string dataAsJson = JsonConvert.SerializeObject(wt);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
//PutRecordsRequestEntry requestRecord = new PutRecordRequest();
Amazon.Kinesis.Model.PutRecordRequest requestRecord = new Amazon.Kinesis.Model.PutRecordRequest();
requestRecord.StreamName = kinesisStreamName;
requestRecord.PartitionKey = "url-response-times";
requestRecord.Data = memoryStream;
Amazon.Kinesis.Model.PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord);
Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record {0} to Kinesis. Exception: {1}", wt.Url, ex.Message);
}
}
}
}
public void Post([FromBody] dynamic value)
{
string dataAsJson = JsonConvert.SerializeObject(value);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
String kinesisStreamName = "click-stream";
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = kinesisStreamName;
requestRecord.PartitionKey = "temp";
requestRecord.Data = memoryStream;
kinesisClient.PutRecordAsync(requestRecord);
//Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message);
}
}
}
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = _regionEndpoint;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
foreach (DeviceData data in dataList)
{
string dataAsJson = JsonConvert.SerializeObject(data);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = _kinesisStreamName;
requestRecord.PartitionKey = data.DeviceId;
requestRecord.Data = memoryStream;
PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord);
Console.WriteLine("Successfully published. Record:{0},{1},{2} Seq:{3}",
data.DeviceId, data.Humidity, data.Temperature, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to publish. Exception: {0}", ex.Message);
}
}
}
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
ListStreamsResponse resp = new ListStreamsResponse();
string lastStream = "";
do
{
ListStreamsRequest req = new ListStreamsRequest
{
ExclusiveStartStreamName = lastStream,
Limit = maxItems
};
resp = client.ListStreams(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamNames)
{
AddObject(obj);
}
lastStream = resp.StreamNames[resp.StreamNames.Count];
}while (resp.HasMoreStreams);
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
ListStreamConsumersResponse resp = new ListStreamConsumersResponse();
do
{
ListStreamConsumersRequest req = new ListStreamConsumersRequest
{
NextToken = resp.NextToken
,
MaxResults = maxItems
};
resp = client.ListStreamConsumers(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.Consumers)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.NextToken));
}
public override IAmazonKinesis CreateKinesisClient()
{
var config = new AmazonKinesisConfig()
{
RegionEndpoint = AWSRegion
};
return(new AmazonKinesisClient(Credentials, config));
}
public AmazonKinesisClient GetClient()
{
var config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.GetBySystemName(ConfigurationSettings.AppSettings[ConfigurationKeys.Region]);
_streamName = ConfigurationSettings.AppSettings[ConfigurationKeys.KinesisStreamName];
return(new AmazonKinesisClient(ConfigurationSettings.AppSettings[ConfigurationKeys.AccessKeyId],
ConfigurationSettings.AppSettings[ConfigurationKeys.AccessKeySecret],
config));
}
protected IAmazonKinesis CreateClient(AWSCredentials credentials, RegionEndpoint region)
{
var config = new AmazonKinesisConfig {
RegionEndpoint = region
};
Amazon.PowerShell.Utils.Common.PopulateConfig(this, config);
this.CustomizeClientConfig(config);
var client = new AmazonKinesisClient(credentials, config);
client.BeforeRequestEvent += RequestEventHandler;
client.AfterResponseEvent += ResponseEventHandler;
return(client);
}
public KManager(string awsKey, string awsSecret, AmazonKinesisConfig config, string workerId = null)
{
if (workerId == null)
{
workerId = Environment.MachineName;
}
_client = new AmazonKinesisClient(awsKey, awsSecret, config);
_utilities = new Utilities(_client, workerId);
_dynamoDb = new DynamoDB(awsKey, awsSecret, config.RegionEndpoint, _utilities);
_producer = new Producer(_client, _utilities);
_consumer = new Consumer(_client, _utilities, _dynamoDb);
Log.Information("Instantiated KManager");
}
public /*async*/ static void Put(string key, string data)
{
byte[] bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(data));
using (MemoryStream ms = new MemoryStream(bytes)) {
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
//create client that pulls creds from web.config and takes in Kinesis config
AmazonKinesisClient client = new AmazonKinesisClient(config);
//create put request
PutRecordRequest requestRecord = new PutRecordRequest();
//list name of Kinesis stream
requestRecord.StreamName = "hubportal-test";
//give partition key that is used to place record in particular shard
requestRecord.PartitionKey = key;
//add record as memorystream
requestRecord.Data = ms;
//PutRecordResponse recordResponse = await client.PutRecordAsync(requestRecord);
//string response = $"Shard ID: {recordResponse.ShardId}, Metadata: {recordResponse.ResponseMetadata}";
}
}
public KManager(string awsKey, string awsSecret, string streamName, AmazonKinesisConfig config, string workerId = null) :
this(awsKey, awsSecret, config, workerId)
{
_utilities.SetStreamName(streamName);
}
public void WriteToKinesis(string data)
{
//Moved to Constructor
//Dictionary<string, string> kinesisConfigDict = LoadKinesisConfig();
int maxattempts = 5;
for (int attempts = 1; attempts <= maxattempts; attempts++)
{
try
{
//https://docs.aws.amazon.com/sdkfornet/v3/apidocs/items/Kinesis/TKinesisConfig.html
var kinesisConfig = new AmazonKinesisConfig();
// Use Dictionary to initialize kinesisConfig
// https://www.tutorialsteacher.com/csharp/csharp-dictionary
string resultTimeout;
if (kinesisConfigDict.TryGetValue("Timeout", out resultTimeout))
{
var timeout = new TimeSpan(0, 0, 5);
kinesisConfig.Timeout = timeout;
}
else
{
// This key does not exist
}
string resultRegionEndpoint;
if (kinesisConfigDict.TryGetValue("RegionEndpoint", out resultRegionEndpoint))
{
kinesisConfig.RegionEndpoint = Amazon.RegionEndpoint.GetBySystemName(resultRegionEndpoint);
}
else
{
// This key does not exist
}
string resultMaxErrorRetry;
if (kinesisConfigDict.TryGetValue("MaxErrorRetry", out resultMaxErrorRetry))
{
kinesisConfig.MaxErrorRetry = Int32.Parse(resultMaxErrorRetry);
}
else
{
// This key does not exist
}
string resultMaxIdleTimeMS;
if (kinesisConfigDict.TryGetValue("MaxIdleTime", out resultMaxIdleTimeMS))
{
kinesisConfig.MaxIdleTime = Int32.Parse(resultMaxIdleTimeMS);
}
else
{
// This key does not exist
}
// Use Dictionary to initialize awsCredentials
string resultAccessKey;
if (kinesisConfigDict.TryGetValue("AccessKey", out resultAccessKey))
{
// This key does exist
}
else
{
// This key does not exist
}
string resultSecretKey;
if (kinesisConfigDict.TryGetValue("SecretKey", out resultSecretKey))
{
// This key does exist
}
else
{
// This key does not exist
}
var awsCredentials = new Amazon.Runtime.BasicAWSCredentials(resultAccessKey, resultSecretKey);
//https://docs.aws.amazon.com/sdkfornet/v3/apidocs/items/Kinesis/TKinesisClient.html
//create client that pulls creds from web.config and takes in Kinesis config
var kinesisClient = new AmazonKinesisClient(awsCredentials, kinesisConfig);
// Use Dictionary get StreamName and PartitionKey for request
string resultStreamName;
if (kinesisConfigDict.TryGetValue("StreamName", out resultStreamName))
{
// This key does exist
}
else
{
// This key does not exist
MessageBox.Show("StreamName is required", "FATAL Kinesis Config Error");
System.Windows.Forms.Application.Exit();
}
string resultPartitionKey;
if (kinesisConfigDict.TryGetValue("PartitionKey", out resultPartitionKey))
{
// This key does exist
}
else
{
// This key does not exist
System.Diagnostics.Debug.WriteLine("Kinesis Config Error:" + "Kinesis PartitionKey is required" + "Setting Kinesis PartitionKey to default value GIADiamonds");
//MessageBox.Show("Kinesis Partition Key Required", "Kinesis Config Error");
resultPartitionKey = "GIADiamonds";
}
//https://csharp.hotexamples.com/examples/Amazon.Kinesis.Model/PutRecordRequest/-/php-putrecordrequest-class-examples.html
var request = new PutRecordRequest();
request.StreamName = resultStreamName;
request.Data = new System.IO.MemoryStream(Encoding.UTF8.GetBytes(data));
request.PartitionKey = resultPartitionKey;
var kinesisResponse = kinesisClient.PutRecord(request);
// PutRecord returns the shard ID of where the data record was placed and the sequence number that was assigned to the data record.
// Sequence numbers generally increase over time. To guarantee strictly increasing ordering, use the SequenceNumberForOrdering parameter. For more information, see the Amazon Kinesis Developer Guide .
// If a PutRecord request cannot be processed because of insufficient provisioned throughput on the shard involved in the request, PutRecord throws ProvisionedThroughputExceededException .
// Data records are accessible for only 24 hours from the time that they are added to an Amazon Kinesis stream.
System.Diagnostics.Debug.WriteLine("---vvv--- kinesisResponse Response ---vvv---:");
System.Diagnostics.Debug.WriteLine(kinesisResponse.HttpStatusCode.ToString());
System.Diagnostics.Debug.WriteLine(kinesisResponse.ResponseMetadata.ToString());
System.Diagnostics.Debug.WriteLine(kinesisResponse.SequenceNumber.ToString());
System.Diagnostics.Debug.WriteLine(kinesisResponse.ShardId.ToString());
System.Diagnostics.Debug.WriteLine("");
break;
}
catch (ProvisionedThroughputExceededException ptee)
{
if (attempts < maxattempts)
{
System.Diagnostics.Debug.WriteLine(ptee.Message);
//MessageBox.Show(ptee.Message, "Kinesis Provisioned Throughput Exceeded Exception");
Thread.Sleep(1000 * attempts * attempts);
}
else
{
System.Diagnostics.Debug.WriteLine(ptee.Message);
System.Diagnostics.Debug.WriteLine("Max Retries Exceeded - Exiting Retry Loop");
//MessageBox.Show(ptee.Message, "Kinesis Provisioned Throughput Exceeded Exception");
//MessageBox.Show("Max Retries Exceeded - Exiting Retry Loop", "Kinesis Provisioned Throughput Exceeded Exception");
break;
}
}
catch (AmazonKinesisException e)
{
System.Diagnostics.Debug.WriteLine(e.Message);
//MessageBox.Show(e.Message, "Kinesis Exception");
break;
}
}
}
private async void SubmitButton_Click(object sender, System.EventArgs e)
{
//Disable the submit button so the user can't accidentally submit
// the registration twice
FindViewById <Button>(Resource.Id.button1).Enabled = false;
//reset all of the error flags onClick
isError = false;
TextView errorFirstName = FindViewById <TextView>(Resource.Id.errorFirstName);
errorFirstName.Text = "";
TextView errorLastName = FindViewById <TextView>(Resource.Id.errorLastName);
errorLastName.Text = "";
TextView errorEmail = FindViewById <TextView>(Resource.Id.errorEmail);
errorEmail.Text = "";
TextView errorPhone = FindViewById <TextView>(Resource.Id.errorPhone);
errorPhone.Text = "";
EditText firstname = FindViewById <EditText>(Resource.Id.firstName);
//check that the information in the field is valid
//and that it isn't blank
if (firstname.Text != "" || Regex.IsMatch(firstname.Text, "^[A-Za-z]$"))
{
currentStudent.FirstName = firstname.Text;
}
else
{
errorFirstName.Text = "Please enter your first name";
isError = true;
}
EditText lastname = FindViewById <EditText>(Resource.Id.lastName);
//check that the information in the field is valid
//and that it isn't blank
if (lastname.Text != "" || Regex.IsMatch(lastname.Text, "^[A-Za-z]$"))
{
currentStudent.LastName = lastname.Text;
}
else
{
errorLastName.Text = "Please enter your last name";
isError = true;
}
EditText email = FindViewById <EditText>(Resource.Id.email);
//check that the information in the field is valid
//and that it isn't blank
if (email.Text != "" || Regex.IsMatch(email.Text, @"^([\w\.\-]+)@([\w\-]+)((\.(\w){2,3})+)$"))
{
currentStudent.Email = email.Text;
}
else
{
errorEmail.Text = "Please enter your email";
isError = true;
}
EditText phone = FindViewById <EditText>(Resource.Id.phoneNo);
//check that the information in the field is valid
//and that it isn't blank
if (phone.Text != "" || Regex.IsMatch(phone.Text, @"^\(? (\d{ 3})\)?[\s\-]? (\d{3})\-? (\d{4})$"))
{
currentStudent.Phone = phone.Text;
}
else
{
errorPhone.Text = "Please enter your phone number";
isError = true;
}
if (isError)
{
//Enable the submit button again so the user can press it
// since the validation failed
FindViewById <Button>(Resource.Id.button1).Enabled = true;
}
else
{
//add data to shared preferences
ISharedPreferencesEditor editor = prefs.Edit();
editor.PutString("firstname", currentStudent.FirstName);
editor.PutString("lastname", currentStudent.LastName);
editor.PutString("email", currentStudent.Email);
editor.PutString("school", currentStudent.School);
editor.PutString("phone", currentStudent.Phone);
DateTime time = DateTime.Now;
editor.PutString("timestamp", time.ToString("T"));
//write to SP
editor.Apply();
//information needs to be added to the database using the AWS Lambda function
var awsCredentials = new Amazon.Runtime.BasicAWSCredentials(WebApp.AWSKey, WebApp.AWSSecret);
try
{
string dataAsJson = JsonConvert.SerializeObject(currentStudent);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
//create config that points to AWS region
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
//create client that pulls creds from config
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(awsCredentials, Amazon.RegionEndpoint.USEast1);
//create put request
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = "exploreOTCData";
//give partition key that is used to place record in particular shard
requestRecord.PartitionKey = "registration";
//add record as memory stream
requestRecord.Data = memoryStream;
//PUT the record to Kinesis
Toast.MakeText(this, "Processing registration...", ToastLength.Long);
PutRecordResponse response = await kinesisClient.PutRecordAsync(requestRecord);
Console.WriteLine(response);
}
}
catch (Exception ex)
{
Console.Write(ex.Message);
}
//close this activity
Finish();
}
}