private async Task ReadFromStream()
{
var describeRequest = new DescribeStreamRequest
{
StreamName = _streamName,
};
var describeStreamResponse = await _kinesisClient.DescribeStreamAsync(describeRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
foreach (var shard in shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamName = _streamName,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
};
var getShardIteratorResponse = await _kinesisClient.GetShardIteratorAsync(getShardIteratorRequest);
var shardIterator = getShardIteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(shardIterator))
{
var getRecordsRequest = new GetRecordsRequest
{
Limit = 100,
ShardIterator = shardIterator,
};
var getRecordsResponse = await _kinesisClient.GetRecordsAsync(getRecordsRequest);
var nextIterator = getRecordsResponse.NextShardIterator;
var records = getRecordsResponse.Records;
if (records.Count > 0)
{
Console.WriteLine($"Received {records.Count} records.");
foreach (var record in records)
{
var dataMessage = await JsonSerializer.DeserializeAsync <DataMessage>(record.Data);
Console.WriteLine($"DataMessage Id={dataMessage.Id}, CreatedOn={dataMessage.CreatedOn.ToString("yyyy-MM-dd HH:mm")}");
}
}
shardIterator = nextIterator;
}
}
}
static void Main(string[] args)
{
try
{
log = new KPLNETInterface.Log4NetLogger(typeof(Program));
string akid = Environment.GetEnvironmentVariable(AWS_ACCESS_KEY_ID);
string sKey = Environment.GetEnvironmentVariable(AWS_SECRET_ACCESS_KEY);
if (akid == null || sKey == null)
{
return;
}
c = new AmazonKinesisClient(akid, sKey, Amazon.RegionEndpoint.USWest2);
c.GetShardIteratorAsync(new GetShardIteratorRequest()
{
ShardId = "shardId-000000000000", StreamName = "KPLNETTest", ShardIteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER, StartingSequenceNumber = "49572131548342384703435782055007292317434442354572394498"
}).
ContinueWith((b) =>
{
if (b.Result.HttpStatusCode == System.Net.HttpStatusCode.OK)
{
string it = b.Result.ShardIterator;
if (it != null)
{
c.GetRecordsAsync(new GetRecordsRequest()
{
Limit = 1000, ShardIterator = it
}).ContinueWith(continuationAction);
}
else
{
Console.WriteLine("Done iterating all items in the shard");
}
}
else
{
Console.Error.WriteLine("GetShardIteratorRequest failed with Error code: " + b.Result.HttpStatusCode.ToString());
}
});
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.ToString());
}
finally
{
Console.ReadLine();
}
}
private async Task ReadFromStream()
{
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = myStreamName;
DescribeStreamResponse describeResponse =
await kinesisClient.DescribeStreamAsync(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = myStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string theMessage = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("message string: " + theMessage);
}
}
iteratorId = nextIterator;
}
}
}
private async void Read(string shardId)
{
var shardIterator = await GetShardIterator(shardId);
while (true)
{
if (_cancellationTokenSource.Token.IsCancellationRequested)
{
break;
}
var request = new GetRecordsRequest
{
Limit = 100,
ShardIterator = shardIterator
};
var response = await _client.GetRecordsAsync(request);
if (response.Records == null)
{
break; //todo: raise exception or event
}
if (response.Records.Any())
{
_lastSequenceNumber = response.Records.Last().SequenceNumber;
response.Records.ForEach(record =>
{
var chunk = (CommandChunk)_serializer.Deserialize(record.Data.ToArray());
_chunkHandler.Handle(chunk);
});
}
else
{
await Task.Delay(TimeSpan.FromMilliseconds(100));
}
shardIterator = response.NextShardIterator;
}
}
private async Task <RecordResponse> GetRecordResponse(KShard shard, CancellationToken ctx)
{
var request = new GetRecordsRequest {
ShardIterator = shard.ShardIterator
};
while (!ctx.IsCancellationRequested)
{
var record = await _client.GetRecordsAsync(request, ctx);
if (record.Records.Count > 0)
{
return(RecordResponse.Create(record, ctx));
}
request.ShardIterator = record.NextShardIterator;
await Task.Delay(1000, ctx);
}
return(RecordResponse.Empty);
}
public static async Task ReadFromKinesis()
{
dicPrefixo = new Dictionary <string, string>();
var file = "last_iterator_id.txt";
_client = new AmazonKinesisClient(workConfig["AccessKeyAWS"], workConfig["SecretKeyAWS"], RegionEndpoint.USEast1);
try
{
var describer = await _client.DescribeStreamAsync(new DescribeStreamRequest { StreamName = workConfig["QueueNamedAWS"] });
var shards = describer.StreamDescription.Shards;
foreach (var shard in shards)
{
var iterator_request = new GetShardIteratorRequest
{
StreamName = workConfig["QueueNamedAWS"],
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.LATEST,
Timestamp = DateTime.MinValue
};
var iterator = await _client.GetShardIteratorAsync(iterator_request);
string curr_iterator_id = iterator.ShardIterator;
string last_iterator_id = File.Exists(file) ? File.ReadAllText(file) : string.Empty;
if (!string.IsNullOrEmpty(last_iterator_id))
{
curr_iterator_id = last_iterator_id;
}
while (!string.IsNullOrEmpty(curr_iterator_id))
{
var response = await _client.GetRecordsAsync(new GetRecordsRequest { ShardIterator = curr_iterator_id, Limit = 1 });
var next_iterator_id = response.NextShardIterator;
var records = response.Records;
//Console.Write("\r\n" + "Sequencial: " + curr_iterator_id + "\r\n");
if (records.Count > 0)
{
foreach (var record in records)
{
var strData = Encoding.UTF8.GetString(record.Data.ToArray());
var words = strData.Split(new string[] { "\\n" }, StringSplitOptions.None);
foreach (var item in words)
{
Console.Write(item);
var lines = item.Split('\n');
for (int i = 0; i < lines.Count() - 1; i++)
{
//GravaLog.GravaLogInformacao("Split linha registro - ServiceReader Linha: 194");
var msg_partes = lines[i].Split(',');
//GravaLog.GravaLogInformacao("msg_partes - ServiceReader Linha: 197");
var addressAndTag = msg_partes[0].Replace("ns=2;s=", "");
//GravaLog.GravaLogInformacao("addressAndTag - ServiceReader Linha: 200");
string prefixoTagAtual;
//como o codigo para ler o numero fila do tipo 1 foi separado do tipo 2 então agora o separador é constante.
char separator = '.'; //Configuracoes.NumeroFila == "1" ? '\\' : '.';
if (addressAndTag.IndexOf(separator) == -1)
{
continue;
}
else
{
var tagPartes = addressAndTag.Split(separator);
//GravaLog.GravaLogInformacao("tagPartes - ServiceReader Linha: 214");
var tagPartesPrefixo = tagPartes[0].Split(separator);
//GravaLog.GravaLogInformacao("tagPartesPrefixo - ServiceReader Linha: 217");
prefixoTagAtual = tagPartes[3];
//GravaLog.GravaLogInformacao("prefixoTagAtual - ServiceReader Linha: 220");
nomeTagAtual = tagPartes[4].Split('\\').LastOrDefault();
//GravaLog.GravaLogInformacao("nomeTagAtual - ServiceReader Linha: 223");
}
string strTagCombinada = prefixoTagAtual + "." + nomeTagAtual;
CultureInfo culture = new CultureInfo("en-US");
DateTime dtRegistro;
//GravaLog.GravaLogInformacao("dtRegistro - ServiceReader Linha: 231");
dtRegistro = Convert.ToDateTime(msg_partes[2].TrimEnd(), new CultureInfo("en-US"));
//Grava Registro
//GravaLog.GravaLogInformacao("Data.SetFila_Tabela - Grava fila completa - ServiceReader Linha: 236");
Data.SetFila_Tabela(Conexao.GetConnectionHist(workConfig), lines[i], dtRegistro, workConfig["QueueNamedAWS"],
nomeTagAtual, msg_partes[1].TrimEnd(), prefixoTagAtual, curr_iterator_id);
Int32 id_t_bitola = 0;
GravaLog.GravaLogInformacao("Verifica se há nome da bitola no dicionario - ServiceReader Linha: 242");
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SP_BITOLA", out string valueTagBitola))
{
GravaLog.GravaLogInformacao("Retorna nome da bitola - ServiceReader Linha: 245");
id_t_bitola = Data.GetIdBitola(Conexao.GetConnectionProd(workConfig), valueTagBitola);
}
//Grava na Tabela de Mudança
if (dicPrefixo.TryGetValue(strTagCombinada, out string valueTag))
{
GravaLog.GravaLogInformacao("Verifica mudança de valores - ServiceReader Linha: 253");
if (valueTag != null && valueTag != String.Empty)
{
GravaLog.GravaLogInformacao("Troca o valor da Tag no dicionario - ServiceReader Linha: 256");
dicPrefixo.Remove(strTagCombinada);
dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd());
if (msg_partes[1].TrimEnd() != valueTag)
{
GravaLog.GravaLogInformacao("Data.SetFila_Tabela_Mudanca - Grava mudanca na tabela - Tag: " + nomeTagAtual +
" Valor Anterior: " + valueTag + " Valor Atual: " + msg_partes[1].TrimEnd() + " - ServiceReader Linha: 263");
Data.SetFila_Tabela_Mudanca(Conexao.GetConnectionHist(workConfig), workConfig["QueueNamedAWS"], nomeTagAtual,
msg_partes[1].TrimEnd(), prefixoTagAtual, valueTag);
GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - ServiceReader Linha: 268");
if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT))
{
GravaLog.GravaLogInformacao("BD1 desligada - ServiceReader Linha: 276");
if (valueTagBD1OUT == "1")
{
continue;
}
}
GravaLog.GravaLogInformacao("Retorna ID da gaiola - ServiceReader Linha: 281");
Gaiola gaiola = null;
if (prefixoTagAtual == "HH113")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2))
{
if (Convert.ToInt32(valueBD2) == 0)
{
gaiola = RetornaGaiola("HF113");
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
}
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
if (gaiola != null && gaiola.Num_Bloco > 0)
{
id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco);
if (id_t_bitola > 0)
{
GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id +
" Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 288");
Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco);
}
}
}
}
}
}
else
{
dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd());
GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - dicionario limpo - ServiceReader Linha: 320");
if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT))
{
if (valueTagBD1OUT == "1")
{
continue;
}
}
Gaiola gaiola = null;
if (prefixoTagAtual == "HH113")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2))
{
if (Convert.ToInt32(valueBD2) == 0)
{
gaiola = RetornaGaiola("HF113");
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
}
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
if (gaiola != null && gaiola.Num_Bloco > 0)
{
id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco);
if (id_t_bitola > 0)
{
GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id +
" Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 319");
Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco);
}
}
}
}
}
}
}
}
//GravaLog.GravaLogInformacao("Novo registro da fila");
File.WriteAllText(file, next_iterator_id);
curr_iterator_id = next_iterator_id;
}
}
}
catch (Exception ex)
{
logger.LogCritical($"KinesisWorker.ServiceReader exceção em {DateTimeOffset.Now:dd/MM/yyy HH:mm:ss} ... {ex.Message}");
GravaLog.GravaLogErro(ex.Message);
if (!stoppingToken.IsCancellationRequested)
{
await ReadFromKinesis();
}
}
finally
{
if (!stoppingToken.IsCancellationRequested)
{
await ReadFromKinesis();
}
}
}
