public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
DescribeStreamResponse resp = new DescribeStreamResponse();
do
{
DescribeStreamRequest req = new DescribeStreamRequest
{
ExclusiveStartShardId = resp.StreamDescription.Shards[0].ShardId
,
Limit = maxItems
};
resp = client.DescribeStream(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamDescription.Shards)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.StreamDescription.Shards[0].ShardId));
}
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList)
{
// note: this constructor relies on you having set up a credential profile
// named 'default', or have set credentials in environment variables
// AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY, or have an application settings
// file. See https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html
// for more details and other constructor options.
var kinesisClient = new AmazonKinesisClient(_regionEndpoint);
foreach (DeviceData data in dataList)
{
var dataAsJson = JsonConvert.SerializeObject(data);
var dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (var memoryStream = new MemoryStream(dataAsBytes))
{
try
{
var requestRecord = new PutRecordRequest
{
StreamName = _kinesisStreamName,
PartitionKey = data.DeviceId,
Data = memoryStream
};
var responseRecord = kinesisClient.PutRecordAsync(requestRecord).Result;
Console.WriteLine($"Successfully published. Record:{data.DeviceId},{data.Humidity},{data.Temperature} Seq:{responseRecord.SequenceNumber}");
}
catch (Exception ex)
{
Console.WriteLine($"Failed to publish. Exception: {ex.Message}");
}
}
}
}
public KinesisCommandChunkWriter(AmazonKinesisClient client,
string streamName, ISerializer serializer)
{
_client = client;
_streamName = streamName;
_serializer = serializer;
}
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = _regionEndpoint;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
foreach (DeviceData data in dataList)
{
string dataAsJson = JsonConvert.SerializeObject(data);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = _kinesisStreamName;
requestRecord.PartitionKey = data.DeviceId;
requestRecord.Data = memoryStream;
PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord);
Console.WriteLine("Successfully published. Record:{0},{1},{2} Seq:{3}",
data.DeviceId, data.Humidity, data.Temperature, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to publish. Exception: {0}", ex.Message);
}
}
}
}
public SimpleRecordSender(string regionName, string streamName, IMessageSerializer serializer, Func <TMessage, string> partitionKeyFunc)
{
RegionEndpoint endpoint = RegionEndpoint.USEast1;
switch (regionName)
{
//TODO: Implement the connection to more regions
case "useast1":
case "us-east-1":
endpoint = RegionEndpoint.USEast1;
break;
}
kinesisClient = new AmazonKinesisClient(endpoint);
this.serializer = serializer;
if (DoesKinesisStreamExists(kinesisClient, streamName))
{
}
else
{
throw new ArgumentException($"The Kinesis stream {streamName} does not exist");
}
this.streamName = streamName;
this.partitionKeyFunc = partitionKeyFunc;
}
public static async Task Main()
{
IAmazonKinesis client = new AmazonKinesisClient();
string streamName = "AmazonKinesisStream";
await ListTagsAsync(client, streamName);
}
private static void SendWebTransactionsToQueue(List <WebTransaction> transactions)
{
Amazon.Kinesis.AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
String kinesisStreamName = ConfigurationManager.AppSettings["KinesisStreamName"];
foreach (WebTransaction wt in transactions)
{
string dataAsJson = JsonConvert.SerializeObject(wt);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
//PutRecordsRequestEntry requestRecord = new PutRecordRequest();
Amazon.Kinesis.Model.PutRecordRequest requestRecord = new Amazon.Kinesis.Model.PutRecordRequest();
requestRecord.StreamName = kinesisStreamName;
requestRecord.PartitionKey = "url-response-times";
requestRecord.Data = memoryStream;
Amazon.Kinesis.Model.PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord);
Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record {0} to Kinesis. Exception: {1}", wt.Url, ex.Message);
}
}
}
}
KinesisLog(string AWSAccessKeyId, string AWSSecretAccessKey, string AWSRegion, string Stream, string Partition)
{
KinesisClient = new AmazonKinesisClient(AWSAccessKeyId, AWSSecretAccessKey,
RegionEndpoint.GetBySystemName(AWSRegion));
StreamName = Stream;
PartitionKey = Partition + "/hwmetrics";
}
public void run(ClientWithLoad client)
{
try
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "Kinesistask.run");
if (!started_)
{
started_ = true;
kinesisClient_ = client.ActualClient;
start_ = DateTime.Now;
end_deadline_ = start_.AddMilliseconds(timeout_);
StdErrorOut.Instance.StdOut(LogLevel.debug, "Kinesistask.run before Execute();");
Execute();
Interlocked.Decrement(ref client.Load);
}
else
{
Interlocked.Decrement(ref client.Load);
throw new Exception("The same task cannot be run more than once");
}
}
catch (Exception e)
{
StdErrorOut.Instance.StdError("Error in Kinesis Task run", e);
}
}
public async Task SendEvent(IHiarcEvent theEvent)
{
try
{
var accessKeyId = _kinesisSettings.AccessKeyId;
var secretAccessKey = _kinesisSettings.SecretAccessKey;
var streamName = _kinesisSettings.Stream;
var partitionKey = theEvent.Event;
var kinesisClient = new AmazonKinesisClient(accessKeyId, secretAccessKey, _region);
var serializedEvent = JsonSerializer.Serialize(theEvent);
using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(theEvent.Event));
var requestRecord = new PutRecordRequest()
{
StreamName = streamName, PartitionKey = partitionKey, Data = memoryStream
};
var responseRecord = await kinesisClient.PutRecordAsync(requestRecord);
_logger.LogDebug($"Successfully sent event '{theEvent.Event}' to '{this.Name}'. Payload: {serializedEvent}");
}
catch (Exception ex)
{
_logger.LogError($"Failed to send event '{theEvent.Event}' to '{this.Name}'. Exception: {ex.Message}");
}
}
public static List <string> GetDataStreamList()
{
List <string> streamList = null;
try
{
AmazonKinesisClient kinesisClient;
using (kinesisClient = new AmazonKinesisClient(Models.MyAWSConfigs.KinesisRegion))
{
ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
ListStreamsResponse listStreamsResponse = kinesisClient.ListStreams(listStreamsRequest);
if (listStreamsResponse.HttpStatusCode == System.Net.HttpStatusCode.OK)
{
streamList = listStreamsResponse.StreamNames;
}
else
{
Console.WriteLine("Error listing kinesis data streams");
}
}
}
catch (AmazonKinesisException e)
{
Console.WriteLine("AmazonKinesisException: " + e);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
return(streamList);
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
ListStreamsResponse resp = new ListStreamsResponse();
string lastStream = "";
do
{
ListStreamsRequest req = new ListStreamsRequest
{
ExclusiveStartStreamName = lastStream,
Limit = maxItems
};
resp = client.ListStreams(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.StreamNames)
{
AddObject(obj);
}
lastStream = resp.StreamNames[resp.StreamNames.Count];
}while (resp.HasMoreStreams);
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonKinesisClient client = new AmazonKinesisClient(creds, config);
ListStreamConsumersResponse resp = new ListStreamConsumersResponse();
do
{
ListStreamConsumersRequest req = new ListStreamConsumersRequest
{
NextToken = resp.NextToken
,
MaxResults = maxItems
};
resp = client.ListStreamConsumers(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.Consumers)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.NextToken));
}
private static void Main()
{
Console.WriteLine("Weather Stations Event Streamer" + Environment.NewLine);
AmazonKinesisClient kinesisClient = null;
try
{
//load config file
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json")
.Build();
var kinesisStreamName = configuration["kinesis-stream"];
int timeDelay = Convert.ToInt16(configuration["time-delay"]);
kinesisClient = new AmazonKinesisClient(region: Amazon.RegionEndpoint.USEast1);
// Load mock json data from local file then convert to json
var weatherEvents = JsonConvert.DeserializeObject <IList <Event> >(File.ReadAllText("weather-data.json"));
foreach (var weatherEvent in weatherEvents)
{
// Send each weather item as an event to kinesis
string dataAsJson = JsonConvert.SerializeObject(weatherEvent);
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(dataAsJson)))
{
var requestRecord = new PutRecordRequest
{
StreamName = kinesisStreamName,
PartitionKey = "weather",
Data = memoryStream
};
var response = kinesisClient.PutRecordAsync(requestRecord).Result;
if ((int)response.HttpStatusCode == 200)
{
Console.WriteLine($"Successfully sent record to Kinesis. Sequence number: {response.SequenceNumber}. Next event in {timeDelay}(s)");
}
else
{
Console.Write($"Failed on: {dataAsJson}");
}
}
Thread.Sleep(timeDelay * 1000);
}
}
catch (AmazonClientException amazonEx)
{
Console.WriteLine("Opps! Looks like you don't have an Amazon cli setup or incorrect Kinesis stream name. Update appsettings.json in this build directory. Check the exception below for more details.");
Console.Write(amazonEx.GetBaseException().ToString());
Console.ReadLine();
throw;
}
catch (Exception ex)
{
Console.Write(ex.GetBaseException().ToString());
Console.ReadLine();
throw;
}
finally {
kinesisClient?.Dispose();
}
}
public void Post([FromBody] dynamic value)
{
string dataAsJson = JsonConvert.SerializeObject(value);
byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson);
using (MemoryStream memoryStream = new MemoryStream(dataAsBytes))
{
try
{
AmazonKinesisConfig config = new AmazonKinesisConfig();
config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
String kinesisStreamName = "click-stream";
PutRecordRequest requestRecord = new PutRecordRequest();
requestRecord.StreamName = kinesisStreamName;
requestRecord.PartitionKey = "temp";
requestRecord.Data = memoryStream;
kinesisClient.PutRecordAsync(requestRecord);
//Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber);
}
catch (Exception ex)
{
Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message);
}
}
}
private static string PushData(string name, string userId)
{
try
{
var kinesis = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
var mo = new { Name = name, UserId = userId };
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(mo));
var ms = new MemoryStream(bytes);
Random rnd = new Random(100);
var request = new PutRecordRequest
{
StreamName = "your-kinesis-name",
PartitionKey = "first-my-partion",
Data = ms
};
var kinesisResponse = kinesis.PutRecord(request);
}
catch (Exception ex)
{
return(ex.Message);
}
return("OK");
}
public Actions(RegionEndpoint regionEndpoint, string streamName, DateTime timePeriod, string awsAccessKey, string awsSecretKey)
{
_streamName = streamName;
_regionEndpoint = regionEndpoint;
_timePeriod = timePeriod;
_amazonKinesisClient = new AmazonKinesisClient(new BasicAWSCredentials(awsAccessKey, awsSecretKey),
regionEndpoint);
}
/// <summary>
///
/// </summary>
/// <param name="serviceScopeFactory"></param>
/// <param name="busOptionsAccessor"></param>
/// <param name="transportOptionsAccessor"></param>
/// <param name="loggerFactory"></param>
public AmazonKinesisTransport(IServiceScopeFactory serviceScopeFactory,
IOptions <EventBusOptions> busOptionsAccessor,
IOptions <AmazonKinesisTransportOptions> transportOptionsAccessor,
ILoggerFactory loggerFactory)
: base(serviceScopeFactory, busOptionsAccessor, transportOptionsAccessor, loggerFactory)
{
kinesisClient = new AmazonKinesisClient(credentials: TransportOptions.Credentials,
clientConfig: TransportOptions.KinesisConfig);
}
public KinesisStreamConsumer(AWSCredentials credentials, RegionEndpoint region, IKinesisTracker tracker, IDistributedLockProvider lockManager, ILoggerFactory logFactory)
{
_logger = logFactory.CreateLogger(GetType());
_tracker = tracker;
_lockManager = lockManager;
_client = new AmazonKinesisClient(credentials, region);
_processTask = new Task(Process);
_processTask.Start();
}
public KinesisClient(string region, string streamName, string accessKey, string secretKey, string appName)
{
Region = region;
StreamName = streamName;
AccessKey = accessKey;
SecretKey = secretKey;
AppName = appName;
_client = new AmazonKinesisClient(new BasicAWSCredentials(AccessKey, SecretKey), RegionEndpoint.GetBySystemName(Region));
}
public KinesisProvider(AWSCredentials credentials, RegionEndpoint region, string appName, string streamName, IKinesisStreamConsumer consumer, ILoggerFactory logFactory)
{
_logger = logFactory.CreateLogger(GetType());
_appName = appName;
_streamName = streamName;
_consumer = consumer;
_serializer = new JsonSerializer();
_serializer.TypeNameHandling = TypeNameHandling.All;
_client = new AmazonKinesisClient(credentials, region);
}
public ConnatixKinesisUpload(string awsKey, string awsSecret, string region)
{
if (string.IsNullOrEmpty(awsKey) || string.IsNullOrEmpty(awsSecret))
{
m_client = new AmazonKinesisClient(RegionEndpoint.GetBySystemName(region));
}
else
{
m_client = new AmazonKinesisClient(awsKey, awsSecret, RegionEndpoint.GetBySystemName(region));
}
}
private static async Task SendMessageAsync(string streamName, string fileName)
{
IAmazonKinesis client = new AmazonKinesisClient();
var response = await client.PutRecordAsync(new PutRecordRequest
{
PartitionKey = $"{Guid.NewGuid()}",
StreamName = streamName,
Data = new MemoryStream(File.ReadAllBytes(fileName))
});
Console.WriteLine(JsonConvert.SerializeObject(response, Formatting.Indented));
}
public MessageProducerService()
{
var serverName = "localstack";
_kinesisClient = new AmazonKinesisClient(
"DUMMY_KEY",
"DUMMY_KEY",
new AmazonKinesisConfig
{
ServiceURL = $"http://{serverName}:4568",
});
}
public Consumer(AmazonKinesisClient client, IUtilities utilities, IDynamoDB dynamoDb)
{
_client = client;
_utilities = utilities;
_dynamoDb = dynamoDb;
_isRunning = false;
_cancellationTokenSource = null;
_saveProgressToDynamo = true;
_currentRecordsProcessing = 0;
}
private static void ReadFromStream()
{
/*Config example*/
//AmazonKinesisConfig config = new AmazonKinesisConfig();
//config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1;
//AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config);
/*instance example*/
var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1);
String kinesisStreamName = "your-kinesis-name";
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = kinesisStreamName;
DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = kinesisStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string json = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("Json string: " + json);
}
}
iteratorId = nextIterator;
}
}
}
public static async Task Main()
{
IAmazonKinesis client = new AmazonKinesisClient();
string consumerName = "NEW_CONSUMER_NAME";
string streamARN = "arn:aws:kinesis:us-east-2:000000000000:stream/AmazonKinesisStream";
var consumer = await RegisterConsumerAsync(client, consumerName, streamARN);
if (consumer is not null)
{
Console.WriteLine($"{consumer.ConsumerName}");
}
}
internal static void InitiateAWSClient()
{
try
{
if (kinesisClient == null)
{
kinesisClient = new AmazonKinesisClient("[[KIT_CLOUD_AWS_ACCESS_KEY]]", "[[KIT_CLOUD_AWS_SECRET_KEY]]", Amazon.RegionEndpoint.APSouth1);
}
}
catch (Exception ex)
{
//EventLogger.Write(ex, "FlowLayoutManager exception occured while processing the request for InitiateAWSClient");
}
}
public static async Task Main()
{
IAmazonKinesis client = new AmazonKinesisClient();
string streamName = "AmazonKinesisStream";
int shardCount = 1;
var success = await CreateNewStreamAsync(client, streamName, shardCount);
if (success)
{
Console.WriteLine($"The stream, {streamName} successfully created.");
}
}
protected IAmazonKinesis CreateClient(AWSCredentials credentials, RegionEndpoint region)
{
var config = new AmazonKinesisConfig {
RegionEndpoint = region
};
Amazon.PowerShell.Utils.Common.PopulateConfig(this, config);
this.CustomizeClientConfig(config);
var client = new AmazonKinesisClient(credentials, config);
client.BeforeRequestEvent += RequestEventHandler;
client.AfterResponseEvent += ResponseEventHandler;
return(client);
}
