/*
* this one will prepare some sort of message class that keeps content in certain format
* message body will be send over SQS
*/
public static ConversionRequestMessage ConstructMessage(int lastId, int rowcount, string pdfSource,
char delimiter, MySqlConnection conn, BankDataProcessingDynamoDbDAO bankDataProcessing, bool skipProcessedId)
{
string query = "select id, file_url from bank_data " +
"where id > @p1 " +
"and source = @p2 " +
"and file_url like '%.pdf' " +
"limit @p3; ";
MySqlCommand cmd = new MySqlCommand(query, conn);
cmd.Parameters.AddWithValue("@p1", lastId);
cmd.Parameters.AddWithValue("@p2", pdfSource);
cmd.Parameters.AddWithValue("@p3", rowcount);
logger.Debug(cmd.CommandText);
MySqlDataReader reader = cmd.ExecuteReader();
ConversionRequestMessage msg = new ConversionRequestMessage();
while (reader.Read())
{
int id = reader.GetInt32(0);
// this part will skip ids that are already in dynamo db table
// they will not be re-processed !!!!
if (skipProcessedId)
{
if (bankDataProcessing.IsIdPresent(id))
{
continue;
}
}
string fileUrl = reader.GetString(1);
msg.AddIdAndFile(id, fileUrl);
}
reader.Close();
return(msg);
}
static void Main(string[] args)
{
// read configuration
string toDebug = System.Configuration.ConfigurationManager.AppSettings["toDebug"];
string idFileName = System.Configuration.ConfigurationManager.AppSettings["idFileName"];
string notProcessedIdsFileName = System.Configuration.ConfigurationManager.AppSettings["notProcessedIdsFileName"];
// "server=liveboard.cjvgiw4swlyc.us-west-1.rds.amazonaws.com;database=sum_up;uid=yerem;pwd=sua.liveboard.2018;";
string connectionString = System.Configuration.ConfigurationManager.AppSettings["connectionString"];
string pdfSource = System.Configuration.ConfigurationManager.AppSettings["pdfSource"];
// 5
string rowsToProcessAsStr = System.Configuration.ConfigurationManager.AppSettings["rowsToProcessPerMessage"];
int rowsToProcess = 10;
if (!Int32.TryParse(rowsToProcessAsStr, out rowsToProcess))
{
logger.Fatal("Error conveting rowsToProces to int " + rowsToProcessAsStr);
System.Environment.Exit(1);
}
string numberOfMessagesToSendAsStr = System.Configuration.ConfigurationManager.AppSettings["numberOfMessagesToSend"];
int numberOfMessagesToSend = 5;
if (!Int32.TryParse(numberOfMessagesToSendAsStr, out numberOfMessagesToSend))
{
logger.Fatal("Error conveting numberOfMessagesToSend to int " + numberOfMessagesToSendAsStr);
System.Environment.Exit(1);
}
string sendingQueueUrl = System.Configuration.ConfigurationManager.AppSettings["sendingQueueUrl"];
string receivingQueueUrl = System.Configuration.ConfigurationManager.AppSettings["receivingQueueUrl"];
// sleep time between checking for messages
int sleepTimeMillis = 10 * 1000;
// max delay between two responses so every response within collector has to be < 40 min
double maxDelayInMinutes = 40.0;
// print configuration parameters
// skip processed ids
string skipProcessedIdAsString = System.Configuration.ConfigurationManager.AppSettings["skipProcessedId"];
bool skipProcessedId = skipProcessedIdAsString.Equals("T");
if (toDebug == "Y")
{
Console.WriteLine("Input parameters");
Console.WriteLine(idFileName);
Console.WriteLine(notProcessedIdsFileName);
Console.WriteLine(connectionString);
Console.WriteLine(pdfSource);
Console.WriteLine(rowsToProcessAsStr);
Console.WriteLine(numberOfMessagesToSendAsStr);
Console.WriteLine(sendingQueueUrl);
Console.WriteLine(skipProcessedId);
}
// open stream writer for non processes files
StreamWriter notProcessedStream = new StreamWriter(notProcessedIdsFileName, true);
// get last processed id
// int lastId = GetLastProcessedId(idFileName);
// new version
// establish connection to dynamo db table to exclude already processed rows
BankDataProcessingDynamoDbDAO bankDataProcessing =
new BankDataProcessingDynamoDbDAO("us-west-2");
bool isOk = bankDataProcessing.Connect();
if (!isOk)
{
logger.Fatal("Can not connect to dynamo db");
System.Environment.Exit(1);
}
int lastId = bankDataProcessing.GetMaxIdForSource(pdfSource);
Console.WriteLine("last:" + lastId.ToString());
// connect to sending queue
AmazonSQSConfig sqsConfig = new AmazonSQSConfig();
// sqsConfig.ServiceURL = sendingQueueUrl;
// this is needed in order to avoid error message
// The specified queue does not exist for this wsdl version
sqsConfig.RegionEndpoint = Amazon.RegionEndpoint.USWest2;
AmazonSQSClient sqsClient = new AmazonSQSClient(sqsConfig);
int imsg = 0;
char delimiter = '|';
// keep messages in memory for possible problem handling
Dictionary <string, ConversionRequestMessage> messagesTable = new Dictionary <string, ConversionRequestMessage>();
Dictionary <string, bool> isMessageProcessed = new Dictionary <string, bool>();
// loop over predefined number of messages
bool allSent = false;
bool allReceived = false;
int lastProcessedId = lastId;
ConversionResponseMessage responseMessageClass = new ConversionResponseMessage();
InstanceCollector instanceCollectior = new InstanceCollector();
while (!(allSent && allReceived))
{
// loop over predefined number of messages
// send message until the limit
if (!allSent)
{
// read from the database
// open and close connection to avoid long running connections
MySqlConnection conn = GetConnection(connectionString);
if (conn == null)
{
logger.Error("Error connecting to databse");
System.Environment.Exit(1);
}
ConversionRequestMessage msg = ConstructMessage(lastId, rowsToProcess, pdfSource, delimiter, conn, bankDataProcessing, skipProcessedId);
conn.Close();
if (msg.Size() > 0)
{
// prepare message
Console.WriteLine(msg.GetMessageBody(delimiter.ToString()));
// send message
imsg++;
logger.Debug("sending message: " + imsg.ToString());
SendMessageRequest request = new SendMessageRequest();
request.MessageBody = msg.GetMessageBody(delimiter.ToString());
request.QueueUrl = sendingQueueUrl;
SendMessageResponse response = sqsClient.SendMessage(request);
string messageId = response.MessageId;
logger.Info("message sent: " + messageId);
messagesTable.Add(messageId, msg);
isMessageProcessed.Add(messageId, false);
lastId = msg.getMaxId();
if (imsg == numberOfMessagesToSend)
{
allSent = true;
}
}
else
{
// no more data to send
allSent = true;
}
}
else
{
bankDataProcessing.Disconnect();
}
if (!allReceived)
{
ReceiveMessageRequest recRequest = new ReceiveMessageRequest();
recRequest.QueueUrl = receivingQueueUrl;
recRequest.MaxNumberOfMessages = 5;
logger.Debug("checking message from receiving queue ");
ReceiveMessageResponse response = sqsClient.ReceiveMessage(recRequest);
if (response.Messages.Count > 0)
{
foreach (var message in response.Messages)
{
string messageReceiptHandle = message.ReceiptHandle;
responseMessageClass.ParseMessage(message.Body);
string requestMessageId = responseMessageClass.RequestMessageId;
responseMessageClass.AppendNonProcessedIdsToFile(notProcessedStream);
// collect instance id and response time
instanceCollectior.collectResponseTime(responseMessageClass.InstanceId);
logger.Info("processing confirmation message for : " + requestMessageId);
// flag message as processed
if (isMessageProcessed.ContainsKey(requestMessageId))
{
isMessageProcessed[requestMessageId] = true;
if (messagesTable[requestMessageId].getMaxId() > lastProcessedId)
{
lastProcessedId = messagesTable[requestMessageId].getMaxId();
SaveLastProcessedId(idFileName, lastProcessedId);
}
}
else
{
logger.Warn("Inconsistent messager id" + requestMessageId);
}
// delete processed message
DeleteMessageRequest deleteMessageRequest = new DeleteMessageRequest();
deleteMessageRequest.QueueUrl = receivingQueueUrl;
deleteMessageRequest.ReceiptHandle = messageReceiptHandle;
// success is not tested !!!
sqsClient.DeleteMessage(deleteMessageRequest);
}
allReceived = verifyThatAllMessagesAreReceived(isMessageProcessed, imsg);
// todo: deal with situation when all messages are send but not all are received and there is a timeout !!!
}
// sleep a little before you go for next response message
if (allSent)
{
System.Threading.Thread.Sleep(sleepTimeMillis);
}
}
instanceCollectior.TotalMessages = imsg;
instanceCollectior.PrintAllInstances();
instanceCollectior.PrintNonResponsiveInstances(maxDelayInMinutes);
}
notProcessedStream.Close();
logger.Info("Done");
// Console.Read();
}
