public static void test_ConversionResponseMessage_parseMessage_2()
{
ConversionResponseMessage responseMessageClass = new ConversionResponseMessage();
responseMessageClass.ParseMessage("12345|");
string requestMessageId = responseMessageClass.RequestMessageId;
Console.WriteLine(requestMessageId);
}
public static void test_ConversionResponseMessage_AppendNonProcessedIdsToFile()
{
ConversionResponseMessage responseMessageClass = new ConversionResponseMessage();
responseMessageClass.ParseMessage("12345||1|abc|2|dfg");
StreamWriter sw = new System.IO.StreamWriter(@"C:\transfer\solid-conversions\test-non-processed.txt", true);
responseMessageClass.AppendNonProcessedIdsToFile(sw);
sw.Close();
}
public static void test_ConversionResponseMessage_GetMessageBody()
{
ConversionResponseMessage reqmsg = new ConversionResponseMessage("123456676");
reqmsg.AddIdAndFileUrlThatIsNotProcessed(1, "test1");
reqmsg.AddIdAndFileUrlThatIsNotProcessed(2, "test2");
reqmsg.AddIdAndFileUrlThatIsNotProcessed(3, "test3");
string body = reqmsg.GetMessageBody();
Console.WriteLine(body);
}
static void Main(string[] args)
{
// imput parameters, read from config file App.config
string toDebug = Config.toDebug;
string pdfBucketName = Config.pdfBucketName;
string docxBucketName = Config.docxBucketName;
string outputDir = Config.outputDir;
string licensePath = Config.licensePath;
string listenerQueueName = Config.listenerQueueName;
string confirmationQueueName = Config.confirmationQueueName;
string extractWorkDir = Config.extractWorkDir;
string tempDocDir = Config.tempDocDir;
int sleepTimeMillis = Config.sleepTimeMillis;
int maxIdleTime = Config.maxIdleTime;
Config.printAllParams();
// start logging
logger.Info("Starting program");
// try to get instance id
string instanceId = "NOT_ON_AMAZON";
try
{
instanceId = Amazon.Util.EC2InstanceMetadata.InstanceId.ToString();
logger.Info("Instance id:" + instanceId);
}
catch (Exception)
{
logger.Info("Not on EC2 instance");
}
// clear working directory
RemoveFilesAndSubDirectories(tempDocDir);
// initialize various objects that we need all the time
RegionEndpoint bucketRegion = RegionEndpoint.USWest1;
IAmazonS3 s3Client = new AmazonS3Client(bucketRegion);
RegionEndpoint docxBucketRegion = RegionEndpoint.USWest2;
IAmazonS3 s3DocxClient = new AmazonS3Client(docxBucketRegion);
MetaDataHolderFactory.connectionString = Config.DbConnectionString;
MetaDataHolderFactory.loadMaps(Config.languageMapFile, Config.threeMapFile, Config.twoMapFile, Config.nonMapFile);
MetaDataHolderFactory.S3bucket = pdfBucketName;
// text is needed like us-west-2
MetaDataHolderFactory.S3region = Amazon.RegionEndpoint.USWest1.SystemName;
BankDataProcessingDynamoDbDAO bankDataProcessing =
new BankDataProcessingDynamoDbDAO(Amazon.RegionEndpoint.USWest2.SystemName, pdfBucketName, docxBucketName);
char delimiter = '|';
// open queues
// connect to sending queue
AmazonSQSConfig sqsConfig = new AmazonSQSConfig();
// this is needed as well
sqsConfig.RegionEndpoint = Amazon.RegionEndpoint.USWest2;
AmazonSQSClient sqsClient = new AmazonSQSClient(sqsConfig);
ReceiveMessageRequest recRequest = new ReceiveMessageRequest();
recRequest.QueueUrl = listenerQueueName;
recRequest.MaxNumberOfMessages = 1;
// loop and read mails from controller
int counter = 0;
// doWork will be true until message with id = 0 is detected
// this part is not implemented yet !!!
bool doWork = true;
// get message time, this will be the time of last message sent
// if nothing happens for some time program will exit
DateTime lastMessageTime = DateTime.Now;
while (doWork)
{
TimeSpan idleTime = DateTime.Now.Subtract(lastMessageTime);
if (idleTime.TotalMinutes > maxIdleTime)
{
logger.Info("Exiting, no message within last " + maxIdleTime.ToString() + " minutes");
break;
}
List <Tuple <int, string> > tuples;
ConversionResponseMessage conversionResponseMessage;
string requestMessageId;
WaitForInputMessage(listenerQueueName, delimiter, sqsClient, recRequest, out requestMessageId, out tuples);
// now process documents one after another
int processedCounter = 0, totalCounter = 0;
conversionResponseMessage = new ConversionResponseMessage(requestMessageId);
conversionResponseMessage.InstanceId = instanceId;
if (tuples == null)
{
logger.Debug("Sleeping");
System.Threading.Thread.Sleep(sleepTimeMillis);
continue;
}
MetaDataHolderFactory.GetConnection();
bool ok = bankDataProcessing.Connect();
if (!ok)
{
logger.Error("Error in connecting to dynamo db: ");
System.Environment.Exit(1);
}
foreach (Tuple <int, string> tup in tuples)
{
totalCounter++;
int id = tup.Item1;
string fileUrl = tup.Item2;
if (id == 0)
{
doWork = false;
break;
}
logger.Info(counter + " processing id: " + id + " " + fileUrl);
// now do the processing of database data for id
List <MetaDataHolder> mhlist = MetaDataHolderFactory.PopulateMetaDataHoldersFromDb(new int[] { id });
// it is always just one meta data holder for now so we can easily extract it
// and pass it to the processing routine
MetaDataHolder holder = mhlist[0];
string textFileS3Path = "";
ok = DoFileConversion(s3Client, pdfBucketName, s3DocxClient, docxBucketName,
licensePath, outputDir, extractWorkDir, tempDocDir, fileUrl, holder, out textFileS3Path);
if (!ok)
{
logger.Error("Error in processing id: " + id.ToString());
conversionResponseMessage.AddIdAndFileUrlThatIsNotProcessed(id, fileUrl);
continue;
}
else
{
bankDataProcessing.Insert(id, holder.Bank, holder.Language, fileUrl, textFileS3Path);
processedCounter++;
}
}
MetaDataHolderFactory.CloseConnection();
bankDataProcessing.Disconnect();
// processing done see how successfull and report to the controller
int badFiles = totalCounter - processedCounter;
if (badFiles > 0)
{
logger.Info("Not all files are processed succesfully, failures:" + badFiles.ToString());
}
SendMessageRequest request = new SendMessageRequest();
request.MessageBody = conversionResponseMessage.GetMessageBody();
request.QueueUrl = confirmationQueueName;
SendMessageResponse confirmationResponse = sqsClient.SendMessage(request);
if (confirmationResponse.HttpStatusCode == System.Net.HttpStatusCode.OK)
{
logger.Debug("Confirmation message sent ");
// remember when last message with results is sent
lastMessageTime = DateTime.Now;
}
else
{
logger.Error("Problem sending confirmation message");
}
}
System.Environment.Exit(0);
}
