public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonKinesisClient client = new AmazonKinesisClient(creds, config); DescribeStreamResponse resp = new DescribeStreamResponse(); do { DescribeStreamRequest req = new DescribeStreamRequest { ExclusiveStartShardId = resp.StreamDescription.Shards[0].ShardId , Limit = maxItems }; resp = client.DescribeStream(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.StreamDescription.Shards) { AddObject(obj); } }while (!string.IsNullOrEmpty(resp.StreamDescription.Shards[0].ShardId)); }
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList) { // note: this constructor relies on you having set up a credential profile // named 'default', or have set credentials in environment variables // AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY, or have an application settings // file. See https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html // for more details and other constructor options. var kinesisClient = new AmazonKinesisClient(_regionEndpoint); foreach (DeviceData data in dataList) { var dataAsJson = JsonConvert.SerializeObject(data); var dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (var memoryStream = new MemoryStream(dataAsBytes)) { try { var requestRecord = new PutRecordRequest { StreamName = _kinesisStreamName, PartitionKey = data.DeviceId, Data = memoryStream }; var responseRecord = kinesisClient.PutRecordAsync(requestRecord).Result; Console.WriteLine($"Successfully published. Record:{data.DeviceId},{data.Humidity},{data.Temperature} Seq:{responseRecord.SequenceNumber}"); } catch (Exception ex) { Console.WriteLine($"Failed to publish. Exception: {ex.Message}"); } } } }
public KinesisCommandChunkWriter(AmazonKinesisClient client, string streamName, ISerializer serializer) { _client = client; _streamName = streamName; _serializer = serializer; }
private static void PublishDeviceDataToKinesis(List <DeviceData> dataList) { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = _regionEndpoint; AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); foreach (DeviceData data in dataList) { string dataAsJson = JsonConvert.SerializeObject(data); byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = _kinesisStreamName; requestRecord.PartitionKey = data.DeviceId; requestRecord.Data = memoryStream; PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord); Console.WriteLine("Successfully published. Record:{0},{1},{2} Seq:{3}", data.DeviceId, data.Humidity, data.Temperature, responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to publish. Exception: {0}", ex.Message); } } } }
public SimpleRecordSender(string regionName, string streamName, IMessageSerializer serializer, Func <TMessage, string> partitionKeyFunc) { RegionEndpoint endpoint = RegionEndpoint.USEast1; switch (regionName) { //TODO: Implement the connection to more regions case "useast1": case "us-east-1": endpoint = RegionEndpoint.USEast1; break; } kinesisClient = new AmazonKinesisClient(endpoint); this.serializer = serializer; if (DoesKinesisStreamExists(kinesisClient, streamName)) { } else { throw new ArgumentException($"The Kinesis stream {streamName} does not exist"); } this.streamName = streamName; this.partitionKeyFunc = partitionKeyFunc; }
public static async Task Main() { IAmazonKinesis client = new AmazonKinesisClient(); string streamName = "AmazonKinesisStream"; await ListTagsAsync(client, streamName); }
private static void SendWebTransactionsToQueue(List <WebTransaction> transactions) { Amazon.Kinesis.AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1; AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); String kinesisStreamName = ConfigurationManager.AppSettings["KinesisStreamName"]; foreach (WebTransaction wt in transactions) { string dataAsJson = JsonConvert.SerializeObject(wt); byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { //PutRecordsRequestEntry requestRecord = new PutRecordRequest(); Amazon.Kinesis.Model.PutRecordRequest requestRecord = new Amazon.Kinesis.Model.PutRecordRequest(); requestRecord.StreamName = kinesisStreamName; requestRecord.PartitionKey = "url-response-times"; requestRecord.Data = memoryStream; Amazon.Kinesis.Model.PutRecordResponse responseRecord = kinesisClient.PutRecord(requestRecord); Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to send record {0} to Kinesis. Exception: {1}", wt.Url, ex.Message); } } } }
KinesisLog(string AWSAccessKeyId, string AWSSecretAccessKey, string AWSRegion, string Stream, string Partition) { KinesisClient = new AmazonKinesisClient(AWSAccessKeyId, AWSSecretAccessKey, RegionEndpoint.GetBySystemName(AWSRegion)); StreamName = Stream; PartitionKey = Partition + "/hwmetrics"; }
public void run(ClientWithLoad client) { try { StdErrorOut.Instance.StdOut(LogLevel.debug, "Kinesistask.run"); if (!started_) { started_ = true; kinesisClient_ = client.ActualClient; start_ = DateTime.Now; end_deadline_ = start_.AddMilliseconds(timeout_); StdErrorOut.Instance.StdOut(LogLevel.debug, "Kinesistask.run before Execute();"); Execute(); Interlocked.Decrement(ref client.Load); } else { Interlocked.Decrement(ref client.Load); throw new Exception("The same task cannot be run more than once"); } } catch (Exception e) { StdErrorOut.Instance.StdError("Error in Kinesis Task run", e); } }
public async Task SendEvent(IHiarcEvent theEvent) { try { var accessKeyId = _kinesisSettings.AccessKeyId; var secretAccessKey = _kinesisSettings.SecretAccessKey; var streamName = _kinesisSettings.Stream; var partitionKey = theEvent.Event; var kinesisClient = new AmazonKinesisClient(accessKeyId, secretAccessKey, _region); var serializedEvent = JsonSerializer.Serialize(theEvent); using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(theEvent.Event)); var requestRecord = new PutRecordRequest() { StreamName = streamName, PartitionKey = partitionKey, Data = memoryStream }; var responseRecord = await kinesisClient.PutRecordAsync(requestRecord); _logger.LogDebug($"Successfully sent event '{theEvent.Event}' to '{this.Name}'. Payload: {serializedEvent}"); } catch (Exception ex) { _logger.LogError($"Failed to send event '{theEvent.Event}' to '{this.Name}'. Exception: {ex.Message}"); } }
public static List <string> GetDataStreamList() { List <string> streamList = null; try { AmazonKinesisClient kinesisClient; using (kinesisClient = new AmazonKinesisClient(Models.MyAWSConfigs.KinesisRegion)) { ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); ListStreamsResponse listStreamsResponse = kinesisClient.ListStreams(listStreamsRequest); if (listStreamsResponse.HttpStatusCode == System.Net.HttpStatusCode.OK) { streamList = listStreamsResponse.StreamNames; } else { Console.WriteLine("Error listing kinesis data streams"); } } } catch (AmazonKinesisException e) { Console.WriteLine("AmazonKinesisException: " + e); } catch (Exception e) { Console.WriteLine("Error: " + e); } return(streamList); }
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonKinesisClient client = new AmazonKinesisClient(creds, config); ListStreamsResponse resp = new ListStreamsResponse(); string lastStream = ""; do { ListStreamsRequest req = new ListStreamsRequest { ExclusiveStartStreamName = lastStream, Limit = maxItems }; resp = client.ListStreams(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.StreamNames) { AddObject(obj); } lastStream = resp.StreamNames[resp.StreamNames.Count]; }while (resp.HasMoreStreams); }
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonKinesisClient client = new AmazonKinesisClient(creds, config); ListStreamConsumersResponse resp = new ListStreamConsumersResponse(); do { ListStreamConsumersRequest req = new ListStreamConsumersRequest { NextToken = resp.NextToken , MaxResults = maxItems }; resp = client.ListStreamConsumers(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.Consumers) { AddObject(obj); } }while (!string.IsNullOrEmpty(resp.NextToken)); }
private static void Main() { Console.WriteLine("Weather Stations Event Streamer" + Environment.NewLine); AmazonKinesisClient kinesisClient = null; try { //load config file var configuration = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json") .Build(); var kinesisStreamName = configuration["kinesis-stream"]; int timeDelay = Convert.ToInt16(configuration["time-delay"]); kinesisClient = new AmazonKinesisClient(region: Amazon.RegionEndpoint.USEast1); // Load mock json data from local file then convert to json var weatherEvents = JsonConvert.DeserializeObject <IList <Event> >(File.ReadAllText("weather-data.json")); foreach (var weatherEvent in weatherEvents) { // Send each weather item as an event to kinesis string dataAsJson = JsonConvert.SerializeObject(weatherEvent); using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(dataAsJson))) { var requestRecord = new PutRecordRequest { StreamName = kinesisStreamName, PartitionKey = "weather", Data = memoryStream }; var response = kinesisClient.PutRecordAsync(requestRecord).Result; if ((int)response.HttpStatusCode == 200) { Console.WriteLine($"Successfully sent record to Kinesis. Sequence number: {response.SequenceNumber}. Next event in {timeDelay}(s)"); } else { Console.Write($"Failed on: {dataAsJson}"); } } Thread.Sleep(timeDelay * 1000); } } catch (AmazonClientException amazonEx) { Console.WriteLine("Opps! Looks like you don't have an Amazon cli setup or incorrect Kinesis stream name. Update appsettings.json in this build directory. Check the exception below for more details."); Console.Write(amazonEx.GetBaseException().ToString()); Console.ReadLine(); throw; } catch (Exception ex) { Console.Write(ex.GetBaseException().ToString()); Console.ReadLine(); throw; } finally { kinesisClient?.Dispose(); } }
public void Post([FromBody] dynamic value) { string dataAsJson = JsonConvert.SerializeObject(value); byte[] dataAsBytes = Encoding.UTF8.GetBytes(dataAsJson); using (MemoryStream memoryStream = new MemoryStream(dataAsBytes)) { try { AmazonKinesisConfig config = new AmazonKinesisConfig(); config.RegionEndpoint = Amazon.RegionEndpoint.USEast1; AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); String kinesisStreamName = "click-stream"; PutRecordRequest requestRecord = new PutRecordRequest(); requestRecord.StreamName = kinesisStreamName; requestRecord.PartitionKey = "temp"; requestRecord.Data = memoryStream; kinesisClient.PutRecordAsync(requestRecord); //Console.WriteLine("Successfully sent record {0} to Kinesis. Sequence number: {1}", wt.Url, responseRecord.SequenceNumber); } catch (Exception ex) { Console.WriteLine("Failed to send record to Kinesis. Exception: {0}", ex.Message); } } }
private static string PushData(string name, string userId) { try { var kinesis = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1); var mo = new { Name = name, UserId = userId }; var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(mo)); var ms = new MemoryStream(bytes); Random rnd = new Random(100); var request = new PutRecordRequest { StreamName = "your-kinesis-name", PartitionKey = "first-my-partion", Data = ms }; var kinesisResponse = kinesis.PutRecord(request); } catch (Exception ex) { return(ex.Message); } return("OK"); }
public Actions(RegionEndpoint regionEndpoint, string streamName, DateTime timePeriod, string awsAccessKey, string awsSecretKey) { _streamName = streamName; _regionEndpoint = regionEndpoint; _timePeriod = timePeriod; _amazonKinesisClient = new AmazonKinesisClient(new BasicAWSCredentials(awsAccessKey, awsSecretKey), regionEndpoint); }
/// <summary> /// /// </summary> /// <param name="serviceScopeFactory"></param> /// <param name="busOptionsAccessor"></param> /// <param name="transportOptionsAccessor"></param> /// <param name="loggerFactory"></param> public AmazonKinesisTransport(IServiceScopeFactory serviceScopeFactory, IOptions <EventBusOptions> busOptionsAccessor, IOptions <AmazonKinesisTransportOptions> transportOptionsAccessor, ILoggerFactory loggerFactory) : base(serviceScopeFactory, busOptionsAccessor, transportOptionsAccessor, loggerFactory) { kinesisClient = new AmazonKinesisClient(credentials: TransportOptions.Credentials, clientConfig: TransportOptions.KinesisConfig); }
public KinesisStreamConsumer(AWSCredentials credentials, RegionEndpoint region, IKinesisTracker tracker, IDistributedLockProvider lockManager, ILoggerFactory logFactory) { _logger = logFactory.CreateLogger(GetType()); _tracker = tracker; _lockManager = lockManager; _client = new AmazonKinesisClient(credentials, region); _processTask = new Task(Process); _processTask.Start(); }
public KinesisClient(string region, string streamName, string accessKey, string secretKey, string appName) { Region = region; StreamName = streamName; AccessKey = accessKey; SecretKey = secretKey; AppName = appName; _client = new AmazonKinesisClient(new BasicAWSCredentials(AccessKey, SecretKey), RegionEndpoint.GetBySystemName(Region)); }
public KinesisProvider(AWSCredentials credentials, RegionEndpoint region, string appName, string streamName, IKinesisStreamConsumer consumer, ILoggerFactory logFactory) { _logger = logFactory.CreateLogger(GetType()); _appName = appName; _streamName = streamName; _consumer = consumer; _serializer = new JsonSerializer(); _serializer.TypeNameHandling = TypeNameHandling.All; _client = new AmazonKinesisClient(credentials, region); }
public ConnatixKinesisUpload(string awsKey, string awsSecret, string region) { if (string.IsNullOrEmpty(awsKey) || string.IsNullOrEmpty(awsSecret)) { m_client = new AmazonKinesisClient(RegionEndpoint.GetBySystemName(region)); } else { m_client = new AmazonKinesisClient(awsKey, awsSecret, RegionEndpoint.GetBySystemName(region)); } }
private static async Task SendMessageAsync(string streamName, string fileName) { IAmazonKinesis client = new AmazonKinesisClient(); var response = await client.PutRecordAsync(new PutRecordRequest { PartitionKey = $"{Guid.NewGuid()}", StreamName = streamName, Data = new MemoryStream(File.ReadAllBytes(fileName)) }); Console.WriteLine(JsonConvert.SerializeObject(response, Formatting.Indented)); }
public MessageProducerService() { var serverName = "localstack"; _kinesisClient = new AmazonKinesisClient( "DUMMY_KEY", "DUMMY_KEY", new AmazonKinesisConfig { ServiceURL = $"http://{serverName}:4568", }); }
public Consumer(AmazonKinesisClient client, IUtilities utilities, IDynamoDB dynamoDb) { _client = client; _utilities = utilities; _dynamoDb = dynamoDb; _isRunning = false; _cancellationTokenSource = null; _saveProgressToDynamo = true; _currentRecordsProcessing = 0; }
private static void ReadFromStream() { /*Config example*/ //AmazonKinesisConfig config = new AmazonKinesisConfig(); //config.RegionEndpoint = Amazon.RegionEndpoint.EUWest1; //AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config); /*instance example*/ var kinesisClient = new AmazonKinesisClient("XXXXX", "YYYYYY", RegionEndpoint.EUWest1); String kinesisStreamName = "your-kinesis-name"; DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = kinesisStreamName; DescribeStreamResponse describeResponse = kinesisClient.DescribeStream(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = kinesisStreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON; GetShardIteratorResponse iteratorResponse = kinesisClient.GetShardIterator(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(iteratorId)) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = kinesisClient.GetRecords(getRequest); string nextIterator = getResponse.NextShardIterator; List <Record> records = getResponse.Records; if (records.Count > 0) { Console.WriteLine("Received {0} records. ", records.Count); foreach (Record record in records) { string json = Encoding.UTF8.GetString(record.Data.ToArray()); Console.WriteLine("Json string: " + json); } } iteratorId = nextIterator; } } }
public static async Task Main() { IAmazonKinesis client = new AmazonKinesisClient(); string consumerName = "NEW_CONSUMER_NAME"; string streamARN = "arn:aws:kinesis:us-east-2:000000000000:stream/AmazonKinesisStream"; var consumer = await RegisterConsumerAsync(client, consumerName, streamARN); if (consumer is not null) { Console.WriteLine($"{consumer.ConsumerName}"); } }
internal static void InitiateAWSClient() { try { if (kinesisClient == null) { kinesisClient = new AmazonKinesisClient("[[KIT_CLOUD_AWS_ACCESS_KEY]]", "[[KIT_CLOUD_AWS_SECRET_KEY]]", Amazon.RegionEndpoint.APSouth1); } } catch (Exception ex) { //EventLogger.Write(ex, "FlowLayoutManager exception occured while processing the request for InitiateAWSClient"); } }
public static async Task Main() { IAmazonKinesis client = new AmazonKinesisClient(); string streamName = "AmazonKinesisStream"; int shardCount = 1; var success = await CreateNewStreamAsync(client, streamName, shardCount); if (success) { Console.WriteLine($"The stream, {streamName} successfully created."); } }
protected IAmazonKinesis CreateClient(AWSCredentials credentials, RegionEndpoint region) { var config = new AmazonKinesisConfig { RegionEndpoint = region }; Amazon.PowerShell.Utils.Common.PopulateConfig(this, config); this.CustomizeClientConfig(config); var client = new AmazonKinesisClient(credentials, config); client.BeforeRequestEvent += RequestEventHandler; client.AfterResponseEvent += ResponseEventHandler; return(client); }