public static void test_ConversionResponseMessage_GetMessageBody() { ConversionResponseMessage reqmsg = new ConversionResponseMessage("123456676"); reqmsg.AddIdAndFileUrlThatIsNotProcessed(1, "test1"); reqmsg.AddIdAndFileUrlThatIsNotProcessed(2, "test2"); reqmsg.AddIdAndFileUrlThatIsNotProcessed(3, "test3"); string body = reqmsg.GetMessageBody(); Console.WriteLine(body); }
static void Main(string[] args) { // imput parameters, read from config file App.config string toDebug = Config.toDebug; string pdfBucketName = Config.pdfBucketName; string docxBucketName = Config.docxBucketName; string outputDir = Config.outputDir; string licensePath = Config.licensePath; string listenerQueueName = Config.listenerQueueName; string confirmationQueueName = Config.confirmationQueueName; string extractWorkDir = Config.extractWorkDir; string tempDocDir = Config.tempDocDir; int sleepTimeMillis = Config.sleepTimeMillis; int maxIdleTime = Config.maxIdleTime; Config.printAllParams(); // start logging logger.Info("Starting program"); // try to get instance id string instanceId = "NOT_ON_AMAZON"; try { instanceId = Amazon.Util.EC2InstanceMetadata.InstanceId.ToString(); logger.Info("Instance id:" + instanceId); } catch (Exception) { logger.Info("Not on EC2 instance"); } // clear working directory RemoveFilesAndSubDirectories(tempDocDir); // initialize various objects that we need all the time RegionEndpoint bucketRegion = RegionEndpoint.USWest1; IAmazonS3 s3Client = new AmazonS3Client(bucketRegion); RegionEndpoint docxBucketRegion = RegionEndpoint.USWest2; IAmazonS3 s3DocxClient = new AmazonS3Client(docxBucketRegion); MetaDataHolderFactory.connectionString = Config.DbConnectionString; MetaDataHolderFactory.loadMaps(Config.languageMapFile, Config.threeMapFile, Config.twoMapFile, Config.nonMapFile); MetaDataHolderFactory.S3bucket = pdfBucketName; // text is needed like us-west-2 MetaDataHolderFactory.S3region = Amazon.RegionEndpoint.USWest1.SystemName; BankDataProcessingDynamoDbDAO bankDataProcessing = new BankDataProcessingDynamoDbDAO(Amazon.RegionEndpoint.USWest2.SystemName, pdfBucketName, docxBucketName); char delimiter = '|'; // open queues // connect to sending queue AmazonSQSConfig sqsConfig = new AmazonSQSConfig(); // this is needed as well sqsConfig.RegionEndpoint = Amazon.RegionEndpoint.USWest2; AmazonSQSClient sqsClient = new AmazonSQSClient(sqsConfig); ReceiveMessageRequest recRequest = new ReceiveMessageRequest(); recRequest.QueueUrl = listenerQueueName; recRequest.MaxNumberOfMessages = 1; // loop and read mails from controller int counter = 0; // doWork will be true until message with id = 0 is detected // this part is not implemented yet !!! bool doWork = true; // get message time, this will be the time of last message sent // if nothing happens for some time program will exit DateTime lastMessageTime = DateTime.Now; while (doWork) { TimeSpan idleTime = DateTime.Now.Subtract(lastMessageTime); if (idleTime.TotalMinutes > maxIdleTime) { logger.Info("Exiting, no message within last " + maxIdleTime.ToString() + " minutes"); break; } List <Tuple <int, string> > tuples; ConversionResponseMessage conversionResponseMessage; string requestMessageId; WaitForInputMessage(listenerQueueName, delimiter, sqsClient, recRequest, out requestMessageId, out tuples); // now process documents one after another int processedCounter = 0, totalCounter = 0; conversionResponseMessage = new ConversionResponseMessage(requestMessageId); conversionResponseMessage.InstanceId = instanceId; if (tuples == null) { logger.Debug("Sleeping"); System.Threading.Thread.Sleep(sleepTimeMillis); continue; } MetaDataHolderFactory.GetConnection(); bool ok = bankDataProcessing.Connect(); if (!ok) { logger.Error("Error in connecting to dynamo db: "); System.Environment.Exit(1); } foreach (Tuple <int, string> tup in tuples) { totalCounter++; int id = tup.Item1; string fileUrl = tup.Item2; if (id == 0) { doWork = false; break; } logger.Info(counter + " processing id: " + id + " " + fileUrl); // now do the processing of database data for id List <MetaDataHolder> mhlist = MetaDataHolderFactory.PopulateMetaDataHoldersFromDb(new int[] { id }); // it is always just one meta data holder for now so we can easily extract it // and pass it to the processing routine MetaDataHolder holder = mhlist[0]; string textFileS3Path = ""; ok = DoFileConversion(s3Client, pdfBucketName, s3DocxClient, docxBucketName, licensePath, outputDir, extractWorkDir, tempDocDir, fileUrl, holder, out textFileS3Path); if (!ok) { logger.Error("Error in processing id: " + id.ToString()); conversionResponseMessage.AddIdAndFileUrlThatIsNotProcessed(id, fileUrl); continue; } else { bankDataProcessing.Insert(id, holder.Bank, holder.Language, fileUrl, textFileS3Path); processedCounter++; } } MetaDataHolderFactory.CloseConnection(); bankDataProcessing.Disconnect(); // processing done see how successfull and report to the controller int badFiles = totalCounter - processedCounter; if (badFiles > 0) { logger.Info("Not all files are processed succesfully, failures:" + badFiles.ToString()); } SendMessageRequest request = new SendMessageRequest(); request.MessageBody = conversionResponseMessage.GetMessageBody(); request.QueueUrl = confirmationQueueName; SendMessageResponse confirmationResponse = sqsClient.SendMessage(request); if (confirmationResponse.HttpStatusCode == System.Net.HttpStatusCode.OK) { logger.Debug("Confirmation message sent "); // remember when last message with results is sent lastMessageTime = DateTime.Now; } else { logger.Error("Problem sending confirmation message"); } } System.Environment.Exit(0); }