//todo: handle multiple shards, aws sdk help says one thread per shard. Is _client thread safe? //todo: figure out algorithm to merge sequence in correct order private async Task <string[]> GetShardIds() { string lastShardId = null; var shardIds = new List <string>(); while (true) { var request = new DescribeStreamRequest { StreamName = _streamName, ExclusiveStartShardId = lastShardId }; var stream = (await _client.DescribeStreamAsync(request)).StreamDescription; shardIds.AddRange(stream.Shards.Select(shard => shard.ShardId)); if (!stream.HasMoreShards) { break; } lastShardId = shardIds.Last(); } return(shardIds.ToArray()); }
private void ProcessDescribeStream() { var req = new Amazon.Kinesis.Model.DescribeStreamRequest() { StreamName = dsRequest_.Stream, ExclusiveStartShardId = dsRequest_.ExclusiveStartShardId, Limit = dsRequest_.Limit }; CancellationTokenSource cts = new CancellationTokenSource(); CancellationToken ct = cts.Token; StdErrorOut.Instance.StdOut(LogLevel.debug, "Kinesis.Execute before kinesisClient_.DescribeStreamAsync(req, ct)"); var responseTask = kinesisClient_.DescribeStreamAsync(req, ct); responseTask.Wait(timeout_, ct); if (responseTask.IsCompleted) { response_ = new AwsKinesisResponse(responseTask.Result); succeed(); } if (!responseTask.IsCompleted) { cts.Cancel(); fail("Failed to describe Kinesis Stream"); } }
private async Task ReadFromStream() { var describeRequest = new DescribeStreamRequest { StreamName = _streamName, }; var describeStreamResponse = await _kinesisClient.DescribeStreamAsync(describeRequest); var shards = describeStreamResponse.StreamDescription.Shards; foreach (var shard in shards) { var getShardIteratorRequest = new GetShardIteratorRequest { StreamName = _streamName, ShardId = shard.ShardId, ShardIteratorType = ShardIteratorType.TRIM_HORIZON, }; var getShardIteratorResponse = await _kinesisClient.GetShardIteratorAsync(getShardIteratorRequest); var shardIterator = getShardIteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(shardIterator)) { var getRecordsRequest = new GetRecordsRequest { Limit = 100, ShardIterator = shardIterator, }; var getRecordsResponse = await _kinesisClient.GetRecordsAsync(getRecordsRequest); var nextIterator = getRecordsResponse.NextShardIterator; var records = getRecordsResponse.Records; if (records.Count > 0) { Console.WriteLine($"Received {records.Count} records."); foreach (var record in records) { var dataMessage = await JsonSerializer.DeserializeAsync <DataMessage>(record.Data); Console.WriteLine($"DataMessage Id={dataMessage.Id}, CreatedOn={dataMessage.CreatedOn.ToString("yyyy-MM-dd HH:mm")}"); } } shardIterator = nextIterator; } } }
public async Task <DescribeStreamResponse> GetStreamResponseAsync(string streamName = null) { if (string.IsNullOrEmpty(streamName) && string.IsNullOrEmpty(_streamName)) { throw new Exception("Please specify a stream name to get the stream response."); } var request = new DescribeStreamRequest() { StreamName = streamName ?? _streamName }; return(await _client.DescribeStreamAsync(request)); }
private async Task ReadFromStream() { DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.StreamName = myStreamName; DescribeStreamResponse describeResponse = await kinesisClient.DescribeStreamAsync(describeRequest); List <Shard> shards = describeResponse.StreamDescription.Shards; foreach (Shard shard in shards) { GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.StreamName = myStreamName; iteratorRequest.ShardId = shard.ShardId; iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON; GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest); string iteratorId = iteratorResponse.ShardIterator; while (!string.IsNullOrEmpty(iteratorId)) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.Limit = 1000; getRequest.ShardIterator = iteratorId; GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest); string nextIterator = getResponse.NextShardIterator; List <Record> records = getResponse.Records; if (records.Count > 0) { Console.WriteLine("Received {0} records. ", records.Count); foreach (Record record in records) { string theMessage = Encoding.UTF8.GetString(record.Data.ToArray()); Console.WriteLine("message string: " + theMessage); } } iteratorId = nextIterator; } } }
private bool DoesKinesisStreamExists(AmazonKinesisClient kinesisClient, string streamName) { var describeStreamReq = new DescribeStreamRequest(); describeStreamReq.StreamName = streamName; var describeResult = kinesisClient.DescribeStreamAsync(describeStreamReq).Result; string streamStatus = describeResult.StreamDescription.StreamStatus; if (streamStatus == StreamStatus.ACTIVE) { return(true); } else { return(false); } }
/// <summary> /// This method waits a maximum of 10 minutes for the specified stream to become active. /// <param name="myStreamName">Name of the stream whose active status is waited upon.</param> /// </summary> private static void WaitForStreamToBecomeAvailable(string myStreamName) { var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10); while (DateTime.UtcNow < deadline) { DescribeStreamRequest describeStreamReq = new DescribeStreamRequest(); describeStreamReq.StreamName = myStreamName; var describeResult = kinesisClient.DescribeStreamAsync(describeStreamReq).Result; string streamStatus = describeResult.StreamDescription.StreamStatus; Console.Error.WriteLine(" - current state: " + streamStatus); if (streamStatus == StreamStatus.ACTIVE) { return; } Thread.Sleep(TimeSpan.FromSeconds(20)); } throw new Exception("Stream " + myStreamName + " never went active."); }
public static async Task ReadFromKinesis() { dicPrefixo = new Dictionary <string, string>(); var file = "last_iterator_id.txt"; _client = new AmazonKinesisClient(workConfig["AccessKeyAWS"], workConfig["SecretKeyAWS"], RegionEndpoint.USEast1); try { var describer = await _client.DescribeStreamAsync(new DescribeStreamRequest { StreamName = workConfig["QueueNamedAWS"] }); var shards = describer.StreamDescription.Shards; foreach (var shard in shards) { var iterator_request = new GetShardIteratorRequest { StreamName = workConfig["QueueNamedAWS"], ShardId = shard.ShardId, ShardIteratorType = ShardIteratorType.LATEST, Timestamp = DateTime.MinValue }; var iterator = await _client.GetShardIteratorAsync(iterator_request); string curr_iterator_id = iterator.ShardIterator; string last_iterator_id = File.Exists(file) ? File.ReadAllText(file) : string.Empty; if (!string.IsNullOrEmpty(last_iterator_id)) { curr_iterator_id = last_iterator_id; } while (!string.IsNullOrEmpty(curr_iterator_id)) { var response = await _client.GetRecordsAsync(new GetRecordsRequest { ShardIterator = curr_iterator_id, Limit = 1 }); var next_iterator_id = response.NextShardIterator; var records = response.Records; //Console.Write("\r\n" + "Sequencial: " + curr_iterator_id + "\r\n"); if (records.Count > 0) { foreach (var record in records) { var strData = Encoding.UTF8.GetString(record.Data.ToArray()); var words = strData.Split(new string[] { "\\n" }, StringSplitOptions.None); foreach (var item in words) { Console.Write(item); var lines = item.Split('\n'); for (int i = 0; i < lines.Count() - 1; i++) { //GravaLog.GravaLogInformacao("Split linha registro - ServiceReader Linha: 194"); var msg_partes = lines[i].Split(','); //GravaLog.GravaLogInformacao("msg_partes - ServiceReader Linha: 197"); var addressAndTag = msg_partes[0].Replace("ns=2;s=", ""); //GravaLog.GravaLogInformacao("addressAndTag - ServiceReader Linha: 200"); string prefixoTagAtual; //como o codigo para ler o numero fila do tipo 1 foi separado do tipo 2 então agora o separador é constante. char separator = '.'; //Configuracoes.NumeroFila == "1" ? '\\' : '.'; if (addressAndTag.IndexOf(separator) == -1) { continue; } else { var tagPartes = addressAndTag.Split(separator); //GravaLog.GravaLogInformacao("tagPartes - ServiceReader Linha: 214"); var tagPartesPrefixo = tagPartes[0].Split(separator); //GravaLog.GravaLogInformacao("tagPartesPrefixo - ServiceReader Linha: 217"); prefixoTagAtual = tagPartes[3]; //GravaLog.GravaLogInformacao("prefixoTagAtual - ServiceReader Linha: 220"); nomeTagAtual = tagPartes[4].Split('\\').LastOrDefault(); //GravaLog.GravaLogInformacao("nomeTagAtual - ServiceReader Linha: 223"); } string strTagCombinada = prefixoTagAtual + "." + nomeTagAtual; CultureInfo culture = new CultureInfo("en-US"); DateTime dtRegistro; //GravaLog.GravaLogInformacao("dtRegistro - ServiceReader Linha: 231"); dtRegistro = Convert.ToDateTime(msg_partes[2].TrimEnd(), new CultureInfo("en-US")); //Grava Registro //GravaLog.GravaLogInformacao("Data.SetFila_Tabela - Grava fila completa - ServiceReader Linha: 236"); Data.SetFila_Tabela(Conexao.GetConnectionHist(workConfig), lines[i], dtRegistro, workConfig["QueueNamedAWS"], nomeTagAtual, msg_partes[1].TrimEnd(), prefixoTagAtual, curr_iterator_id); Int32 id_t_bitola = 0; GravaLog.GravaLogInformacao("Verifica se há nome da bitola no dicionario - ServiceReader Linha: 242"); if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SP_BITOLA", out string valueTagBitola)) { GravaLog.GravaLogInformacao("Retorna nome da bitola - ServiceReader Linha: 245"); id_t_bitola = Data.GetIdBitola(Conexao.GetConnectionProd(workConfig), valueTagBitola); } //Grava na Tabela de Mudança if (dicPrefixo.TryGetValue(strTagCombinada, out string valueTag)) { GravaLog.GravaLogInformacao("Verifica mudança de valores - ServiceReader Linha: 253"); if (valueTag != null && valueTag != String.Empty) { GravaLog.GravaLogInformacao("Troca o valor da Tag no dicionario - ServiceReader Linha: 256"); dicPrefixo.Remove(strTagCombinada); dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd()); if (msg_partes[1].TrimEnd() != valueTag) { GravaLog.GravaLogInformacao("Data.SetFila_Tabela_Mudanca - Grava mudanca na tabela - Tag: " + nomeTagAtual + " Valor Anterior: " + valueTag + " Valor Atual: " + msg_partes[1].TrimEnd() + " - ServiceReader Linha: 263"); Data.SetFila_Tabela_Mudanca(Conexao.GetConnectionHist(workConfig), workConfig["QueueNamedAWS"], nomeTagAtual, msg_partes[1].TrimEnd(), prefixoTagAtual, valueTag); GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - ServiceReader Linha: 268"); if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB") { if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT)) { GravaLog.GravaLogInformacao("BD1 desligada - ServiceReader Linha: 276"); if (valueTagBD1OUT == "1") { continue; } } GravaLog.GravaLogInformacao("Retorna ID da gaiola - ServiceReader Linha: 281"); Gaiola gaiola = null; if (prefixoTagAtual == "HH113") { if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2)) { if (Convert.ToInt32(valueBD2) == 0) { gaiola = RetornaGaiola("HF113"); } else { gaiola = RetornaGaiola(prefixoTagAtual); } } } else { gaiola = RetornaGaiola(prefixoTagAtual); } if (gaiola != null && gaiola.Num_Bloco > 0) { id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco); if (id_t_bitola > 0) { GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id + " Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 288"); Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco); } } } } } } else { dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd()); GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - dicionario limpo - ServiceReader Linha: 320"); if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB") { if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT)) { if (valueTagBD1OUT == "1") { continue; } } Gaiola gaiola = null; if (prefixoTagAtual == "HH113") { if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2)) { if (Convert.ToInt32(valueBD2) == 0) { gaiola = RetornaGaiola("HF113"); } else { gaiola = RetornaGaiola(prefixoTagAtual); } } } else { gaiola = RetornaGaiola(prefixoTagAtual); } if (gaiola != null && gaiola.Num_Bloco > 0) { id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco); if (id_t_bitola > 0) { GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id + " Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 319"); Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco); } } } } } } } } //GravaLog.GravaLogInformacao("Novo registro da fila"); File.WriteAllText(file, next_iterator_id); curr_iterator_id = next_iterator_id; } } } catch (Exception ex) { logger.LogCritical($"KinesisWorker.ServiceReader exceção em {DateTimeOffset.Now:dd/MM/yyy HH:mm:ss} ... {ex.Message}"); GravaLog.GravaLogErro(ex.Message); if (!stoppingToken.IsCancellationRequested) { await ReadFromKinesis(); } } finally { if (!stoppingToken.IsCancellationRequested) { await ReadFromKinesis(); } } }