//todo: handle multiple shards, aws sdk help says one thread per shard. Is _client thread safe?
//todo: figure out algorithm to merge sequence in correct order
private async Task <string[]> GetShardIds()
{
string lastShardId = null;
var shardIds = new List <string>();
while (true)
{
var request = new DescribeStreamRequest
{
StreamName = _streamName,
ExclusiveStartShardId = lastShardId
};
var stream = (await _client.DescribeStreamAsync(request)).StreamDescription;
shardIds.AddRange(stream.Shards.Select(shard => shard.ShardId));
if (!stream.HasMoreShards)
{
break;
}
lastShardId = shardIds.Last();
}
return(shardIds.ToArray());
}
private void ProcessDescribeStream()
{
var req = new Amazon.Kinesis.Model.DescribeStreamRequest()
{
StreamName = dsRequest_.Stream,
ExclusiveStartShardId = dsRequest_.ExclusiveStartShardId,
Limit = dsRequest_.Limit
};
CancellationTokenSource cts = new CancellationTokenSource();
CancellationToken ct = cts.Token;
StdErrorOut.Instance.StdOut(LogLevel.debug, "Kinesis.Execute before kinesisClient_.DescribeStreamAsync(req, ct)");
var responseTask = kinesisClient_.DescribeStreamAsync(req, ct);
responseTask.Wait(timeout_, ct);
if (responseTask.IsCompleted)
{
response_ = new AwsKinesisResponse(responseTask.Result);
succeed();
}
if (!responseTask.IsCompleted)
{
cts.Cancel();
fail("Failed to describe Kinesis Stream");
}
}
private async Task ReadFromStream()
{
var describeRequest = new DescribeStreamRequest
{
StreamName = _streamName,
};
var describeStreamResponse = await _kinesisClient.DescribeStreamAsync(describeRequest);
var shards = describeStreamResponse.StreamDescription.Shards;
foreach (var shard in shards)
{
var getShardIteratorRequest = new GetShardIteratorRequest
{
StreamName = _streamName,
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.TRIM_HORIZON,
};
var getShardIteratorResponse = await _kinesisClient.GetShardIteratorAsync(getShardIteratorRequest);
var shardIterator = getShardIteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(shardIterator))
{
var getRecordsRequest = new GetRecordsRequest
{
Limit = 100,
ShardIterator = shardIterator,
};
var getRecordsResponse = await _kinesisClient.GetRecordsAsync(getRecordsRequest);
var nextIterator = getRecordsResponse.NextShardIterator;
var records = getRecordsResponse.Records;
if (records.Count > 0)
{
Console.WriteLine($"Received {records.Count} records.");
foreach (var record in records)
{
var dataMessage = await JsonSerializer.DeserializeAsync <DataMessage>(record.Data);
Console.WriteLine($"DataMessage Id={dataMessage.Id}, CreatedOn={dataMessage.CreatedOn.ToString("yyyy-MM-dd HH:mm")}");
}
}
shardIterator = nextIterator;
}
}
}
public async Task <DescribeStreamResponse> GetStreamResponseAsync(string streamName = null)
{
if (string.IsNullOrEmpty(streamName) && string.IsNullOrEmpty(_streamName))
{
throw new Exception("Please specify a stream name to get the stream response.");
}
var request = new DescribeStreamRequest()
{
StreamName = streamName ?? _streamName
};
return(await _client.DescribeStreamAsync(request));
}
private async Task ReadFromStream()
{
DescribeStreamRequest describeRequest = new DescribeStreamRequest();
describeRequest.StreamName = myStreamName;
DescribeStreamResponse describeResponse =
await kinesisClient.DescribeStreamAsync(describeRequest);
List <Shard> shards = describeResponse.StreamDescription.Shards;
foreach (Shard shard in shards)
{
GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
iteratorRequest.StreamName = myStreamName;
iteratorRequest.ShardId = shard.ShardId;
iteratorRequest.ShardIteratorType = ShardIteratorType.TRIM_HORIZON;
GetShardIteratorResponse iteratorResponse = await kinesisClient.GetShardIteratorAsync(iteratorRequest);
string iteratorId = iteratorResponse.ShardIterator;
while (!string.IsNullOrEmpty(iteratorId))
{
GetRecordsRequest getRequest = new GetRecordsRequest();
getRequest.Limit = 1000;
getRequest.ShardIterator = iteratorId;
GetRecordsResponse getResponse = await kinesisClient.GetRecordsAsync(getRequest);
string nextIterator = getResponse.NextShardIterator;
List <Record> records = getResponse.Records;
if (records.Count > 0)
{
Console.WriteLine("Received {0} records. ", records.Count);
foreach (Record record in records)
{
string theMessage = Encoding.UTF8.GetString(record.Data.ToArray());
Console.WriteLine("message string: " + theMessage);
}
}
iteratorId = nextIterator;
}
}
}
private bool DoesKinesisStreamExists(AmazonKinesisClient kinesisClient, string streamName)
{
var describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = streamName;
var describeResult = kinesisClient.DescribeStreamAsync(describeStreamReq).Result;
string streamStatus = describeResult.StreamDescription.StreamStatus;
if (streamStatus == StreamStatus.ACTIVE)
{
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// This method waits a maximum of 10 minutes for the specified stream to become active.
/// <param name="myStreamName">Name of the stream whose active status is waited upon.</param>
/// </summary>
private static void WaitForStreamToBecomeAvailable(string myStreamName)
{
var deadline = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (DateTime.UtcNow < deadline)
{
DescribeStreamRequest describeStreamReq = new DescribeStreamRequest();
describeStreamReq.StreamName = myStreamName;
var describeResult = kinesisClient.DescribeStreamAsync(describeStreamReq).Result;
string streamStatus = describeResult.StreamDescription.StreamStatus;
Console.Error.WriteLine(" - current state: " + streamStatus);
if (streamStatus == StreamStatus.ACTIVE)
{
return;
}
Thread.Sleep(TimeSpan.FromSeconds(20));
}
throw new Exception("Stream " + myStreamName + " never went active.");
}
public static async Task ReadFromKinesis()
{
dicPrefixo = new Dictionary <string, string>();
var file = "last_iterator_id.txt";
_client = new AmazonKinesisClient(workConfig["AccessKeyAWS"], workConfig["SecretKeyAWS"], RegionEndpoint.USEast1);
try
{
var describer = await _client.DescribeStreamAsync(new DescribeStreamRequest { StreamName = workConfig["QueueNamedAWS"] });
var shards = describer.StreamDescription.Shards;
foreach (var shard in shards)
{
var iterator_request = new GetShardIteratorRequest
{
StreamName = workConfig["QueueNamedAWS"],
ShardId = shard.ShardId,
ShardIteratorType = ShardIteratorType.LATEST,
Timestamp = DateTime.MinValue
};
var iterator = await _client.GetShardIteratorAsync(iterator_request);
string curr_iterator_id = iterator.ShardIterator;
string last_iterator_id = File.Exists(file) ? File.ReadAllText(file) : string.Empty;
if (!string.IsNullOrEmpty(last_iterator_id))
{
curr_iterator_id = last_iterator_id;
}
while (!string.IsNullOrEmpty(curr_iterator_id))
{
var response = await _client.GetRecordsAsync(new GetRecordsRequest { ShardIterator = curr_iterator_id, Limit = 1 });
var next_iterator_id = response.NextShardIterator;
var records = response.Records;
//Console.Write("\r\n" + "Sequencial: " + curr_iterator_id + "\r\n");
if (records.Count > 0)
{
foreach (var record in records)
{
var strData = Encoding.UTF8.GetString(record.Data.ToArray());
var words = strData.Split(new string[] { "\\n" }, StringSplitOptions.None);
foreach (var item in words)
{
Console.Write(item);
var lines = item.Split('\n');
for (int i = 0; i < lines.Count() - 1; i++)
{
//GravaLog.GravaLogInformacao("Split linha registro - ServiceReader Linha: 194");
var msg_partes = lines[i].Split(',');
//GravaLog.GravaLogInformacao("msg_partes - ServiceReader Linha: 197");
var addressAndTag = msg_partes[0].Replace("ns=2;s=", "");
//GravaLog.GravaLogInformacao("addressAndTag - ServiceReader Linha: 200");
string prefixoTagAtual;
//como o codigo para ler o numero fila do tipo 1 foi separado do tipo 2 então agora o separador é constante.
char separator = '.'; //Configuracoes.NumeroFila == "1" ? '\\' : '.';
if (addressAndTag.IndexOf(separator) == -1)
{
continue;
}
else
{
var tagPartes = addressAndTag.Split(separator);
//GravaLog.GravaLogInformacao("tagPartes - ServiceReader Linha: 214");
var tagPartesPrefixo = tagPartes[0].Split(separator);
//GravaLog.GravaLogInformacao("tagPartesPrefixo - ServiceReader Linha: 217");
prefixoTagAtual = tagPartes[3];
//GravaLog.GravaLogInformacao("prefixoTagAtual - ServiceReader Linha: 220");
nomeTagAtual = tagPartes[4].Split('\\').LastOrDefault();
//GravaLog.GravaLogInformacao("nomeTagAtual - ServiceReader Linha: 223");
}
string strTagCombinada = prefixoTagAtual + "." + nomeTagAtual;
CultureInfo culture = new CultureInfo("en-US");
DateTime dtRegistro;
//GravaLog.GravaLogInformacao("dtRegistro - ServiceReader Linha: 231");
dtRegistro = Convert.ToDateTime(msg_partes[2].TrimEnd(), new CultureInfo("en-US"));
//Grava Registro
//GravaLog.GravaLogInformacao("Data.SetFila_Tabela - Grava fila completa - ServiceReader Linha: 236");
Data.SetFila_Tabela(Conexao.GetConnectionHist(workConfig), lines[i], dtRegistro, workConfig["QueueNamedAWS"],
nomeTagAtual, msg_partes[1].TrimEnd(), prefixoTagAtual, curr_iterator_id);
Int32 id_t_bitola = 0;
GravaLog.GravaLogInformacao("Verifica se há nome da bitola no dicionario - ServiceReader Linha: 242");
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SP_BITOLA", out string valueTagBitola))
{
GravaLog.GravaLogInformacao("Retorna nome da bitola - ServiceReader Linha: 245");
id_t_bitola = Data.GetIdBitola(Conexao.GetConnectionProd(workConfig), valueTagBitola);
}
//Grava na Tabela de Mudança
if (dicPrefixo.TryGetValue(strTagCombinada, out string valueTag))
{
GravaLog.GravaLogInformacao("Verifica mudança de valores - ServiceReader Linha: 253");
if (valueTag != null && valueTag != String.Empty)
{
GravaLog.GravaLogInformacao("Troca o valor da Tag no dicionario - ServiceReader Linha: 256");
dicPrefixo.Remove(strTagCombinada);
dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd());
if (msg_partes[1].TrimEnd() != valueTag)
{
GravaLog.GravaLogInformacao("Data.SetFila_Tabela_Mudanca - Grava mudanca na tabela - Tag: " + nomeTagAtual +
" Valor Anterior: " + valueTag + " Valor Atual: " + msg_partes[1].TrimEnd() + " - ServiceReader Linha: 263");
Data.SetFila_Tabela_Mudanca(Conexao.GetConnectionHist(workConfig), workConfig["QueueNamedAWS"], nomeTagAtual,
msg_partes[1].TrimEnd(), prefixoTagAtual, valueTag);
GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - ServiceReader Linha: 268");
if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT))
{
GravaLog.GravaLogInformacao("BD1 desligada - ServiceReader Linha: 276");
if (valueTagBD1OUT == "1")
{
continue;
}
}
GravaLog.GravaLogInformacao("Retorna ID da gaiola - ServiceReader Linha: 281");
Gaiola gaiola = null;
if (prefixoTagAtual == "HH113")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2))
{
if (Convert.ToInt32(valueBD2) == 0)
{
gaiola = RetornaGaiola("HF113");
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
}
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
if (gaiola != null && gaiola.Num_Bloco > 0)
{
id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco);
if (id_t_bitola > 0)
{
GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id +
" Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 288");
Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco);
}
}
}
}
}
}
else
{
dicPrefixo.Add(strTagCombinada, msg_partes[1].TrimEnd());
GravaLog.GravaLogInformacao("Identificada Tag que retorna posicao do canal - dicionario limpo - ServiceReader Linha: 320");
if (nomeTagAtual == "GRVATUAL" || nomeTagAtual == "CC_CURPGVNB")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".SS_BD1OUT", out string valueTagBD1OUT))
{
if (valueTagBD1OUT == "1")
{
continue;
}
}
Gaiola gaiola = null;
if (prefixoTagAtual == "HH113")
{
if (dicPrefixo.TryGetValue(prefixoTagAtual + ".TB_Z13", out string valueBD2))
{
if (Convert.ToInt32(valueBD2) == 0)
{
gaiola = RetornaGaiola("HF113");
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
}
}
else
{
gaiola = RetornaGaiola(prefixoTagAtual);
}
if (gaiola != null && gaiola.Num_Bloco > 0)
{
id_t_bitola = Data.GetIdBitolaNumBloco(Conexao.GetConnectionProd(workConfig), gaiola.Num_Bloco);
if (id_t_bitola > 0)
{
GravaLog.GravaLogInformacao("Data.SetLaminacaoCambio_Canal - Grava dados do canal - " + "ID Bitola: - " + id_t_bitola + " ID Gaiola: " + gaiola.Id +
" Posicao: " + Convert.ToInt32(msg_partes[1].TrimEnd()) + " - ServiceReader Linha: 319");
Data.SetLaminacaoCambio_Canal(Conexao.GetConnectionProd(workConfig), id_t_bitola, Convert.ToInt32(msg_partes[1].TrimEnd()), gaiola.Id, gaiola.Num_Bloco);
}
}
}
}
}
}
}
}
//GravaLog.GravaLogInformacao("Novo registro da fila");
File.WriteAllText(file, next_iterator_id);
curr_iterator_id = next_iterator_id;
}
}
}
catch (Exception ex)
{
logger.LogCritical($"KinesisWorker.ServiceReader exceção em {DateTimeOffset.Now:dd/MM/yyy HH:mm:ss} ... {ex.Message}");
GravaLog.GravaLogErro(ex.Message);
if (!stoppingToken.IsCancellationRequested)
{
await ReadFromKinesis();
}
}
finally
{
if (!stoppingToken.IsCancellationRequested)
{
await ReadFromKinesis();
}
}
}
