//public static async Task getWorkflowSuccessOperations(CancellationToken ct) //{ // string storageAccountString = CloudConfigurationManager.GetSetting(Configuration.ButlerStorageConnectionConfigurationKey); // await getWorkflowSuccessOperations(ct, storageAccountString); //} private static ButlerResponse DeserializeRsponseMessage(string txtMsg) { ButlerResponse messageResponse = null; try { messageResponse = JsonConvert.DeserializeObject <ButlerResponse>(txtMsg); } catch (Exception) { } return(messageResponse); }
/// <summary> /// Process message Back fro success and Fial queues /// </summary> /// <param name="storageAccountString">Storage account</param> /// <param name="queueName">Queue Name</param> /// <param name="status">Workflow status</param> private static void processMessageBack(string storageAccountString, string queueName, Configuration.WorkflowStatus status) { string erroMsg; //Trace.TraceInformation("reading from workflow results queue: " + status.ToString()); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageAccountString); CloudQueueClient inQueueClient = storageAccount.CreateCloudQueueClient(); CloudQueue inQueue = inQueueClient.GetQueueReference(queueName); inQueue.CreateIfNotExists(); CloudQueueMessage retrievedMessage = inQueue.GetMessage(); if (retrievedMessage != null) { try { if (retrievedMessage.DequeueCount > Configuration.maxDequeueCount) { //Poison Message erroMsg = String.Format("[0]Error: Max DequeueCount Messsage : {1}", "getWorkflowOperations_" + status.ToString(), retrievedMessage.AsString); SendMessageToDeadLetterQueue(erroMsg, inQueueClient, retrievedMessage); } else { ButlerResponse messageResponse = DeserializeRsponseMessage(retrievedMessage.AsString); if (messageResponse != null) { //Process the message Configuration.WorkflowStatus messageStatus = status; processFilesFromResponse(messageResponse, messageStatus, storageAccount); } else { //Message bad format ccopu to deadletter erroMsg = String.Format("[0]Error: invalid Response message : {1}", "getWorkflowOperations_" + status.ToString(), retrievedMessage.AsString); SendMessageToDeadLetterQueue(erroMsg, inQueueClient, retrievedMessage); } } // TODO: updateQueuemessage if it is taking longer //Process the message in less than 30 seconds, and then delete the message inQueue.DeleteMessage(retrievedMessage); } catch (Exception X) { //Trace the error but don't break the proccess or delete the message. //dequeue control is running too. Trace.TraceError(string.Format("[{0}] Error: {1}", "getWorkflowOperations_" + status.ToString(), X.Message)); } } }
/// <summary> /// Serialize Asset info and send message to success queue /// </summary> private void SendMessage() { string qName = MBFConfiguration.ButlerSuccessQueue; CloudStorageAccount storageAccount = CloudStorageAccount.Parse(myRequest.ProcessConfigConn); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); //TODO: queue info CloudQueue queue = queueClient.GetQueueReference(qName); ButlerResponse myButlerResponse = new ButlerResponse(); myButlerResponse.MezzanineFiles = myRequest.ButlerRequest.MezzanineFiles; //Add to Mezzamine Files the control File URL if it exist //Becouse it is needed to move/delete the control file from processing to succes or fail if (!string.IsNullOrEmpty(myRequest.ButlerRequest.ControlFileUri)) { myButlerResponse.MezzanineFiles.Add(myRequest.ButlerRequest.ControlFileUri); } myButlerResponse.TimeStampProcessingCompleted = DateTime.Now.ToString(); myButlerResponse.TimeStampProcessingStarted = myRequest.TimeStampProcessingStarted.ToString(); myButlerResponse.WorkflowName = myRequest.ProcessTypeId; myButlerResponse.MessageId = myRequest.ButlerRequest.MessageId; myButlerResponse.TimeStampRequestSubmitted = myRequest.ButlerRequest.TimeStampUTC; myButlerResponse.StorageConnectionString = myRequest.ButlerRequest.StorageConnectionString; CloudMediaContext _MediaServiceContext = new CloudMediaContext(myRequest.MediaAccountName, myRequest.MediaAccountKey); IAsset x = _MediaServiceContext.Assets.Where(xx => xx.Id == myRequest.AssetId).FirstOrDefault(); AssetInfo ai = new AssetInfo(x); StringBuilder AssetInfoResume = ai.GetStatsTxt(); AssetInfoResume.AppendLine(""); AssetInfoResume.AppendLine("Media Butler Process LOG " + DateTime.Now.ToString()); foreach (string txt in myRequest.Log) { AssetInfoResume.AppendLine(txt); } AssetInfoResume.AppendLine("-----------------------------"); myButlerResponse.Log = AssetInfoResume.ToString(); CloudQueueMessage responseMessae = new CloudQueueMessage(Newtonsoft.Json.JsonConvert.SerializeObject(myButlerResponse)); queue.AddMessage(responseMessae); Trace.TraceInformation("Return Butler Message sent to queue"); }
private static bool processFilesFromResponse(ButlerResponse jobResponse, Configuration.WorkflowStatus jobStatus, CloudStorageAccount account) { bool returnValue = true; CloudBlockBlob baseBlob = new CloudBlockBlob(new Uri(jobResponse.MezzanineFiles[0]), account.Credentials); CloudBlobContainer container = baseBlob.Container; string directoryTo = (jobStatus == Configuration.WorkflowStatus.Failed) ? Configuration.DirectoryFailed : Configuration.DirectoryCompleted; string timestampFileAppend = (string.IsNullOrEmpty(jobResponse.TimeStampProcessingStarted)) ? "" : jobResponse.TimeStampProcessingStarted.Replace(':', '-'); // fix: substitute / with - in date to avoid file being treated as series of dirs timestampFileAppend = timestampFileAppend.Replace('/', '-'); try { foreach (string videoUri in jobResponse.MezzanineFiles) { CloudBlockBlob fileToMove = new CloudBlockBlob(new Uri(videoUri), account.Credentials); var blobContinuationToken = new BlobContinuationToken(); string blobTarget = BlobUtilities.AdjustPath(fileToMove, directoryTo); int trimEnd = blobTarget.LastIndexOf('.'); string blobTargetFileExt = blobTarget.Substring(trimEnd, blobTarget.Length - trimEnd); blobTarget = string.Concat(blobTarget, ".", timestampFileAppend, ".", blobTargetFileExt); BlobUtilities.RenameBlobWithinContainer(container, BlobUtilities.ExtractBlobPath(fileToMove), blobTarget); } // write log file string blobUriString = BlobUtilities.AdjustPath(baseBlob, directoryTo); // remove file ext // append .log blobUriString = string.Concat(blobUriString, ".", timestampFileAppend, ".log"); CloudBlockBlob logBlob = container.GetBlockBlobReference(blobUriString); logBlob.Properties.ContentType = "text/plain"; logBlob.UploadText(jobResponse.Log); } catch (Exception) { throw; } return(returnValue); }
/// <summary> /// Send back to watcher a "Posion messsage" and delete from in queue /// </summary> /// <param name="poisonMessage">the poison message</param> /// <returns>Sueccess or not</returns> private bool SendPoisonMessage(CloudQueueMessage poisonMessage) { bool sw = false; try { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_Configuration.ProcessConfigConn); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); CloudQueue poisonQueue = queueClient.GetQueueReference(_Configuration.poisonQueue); poisonQueue.CreateIfNotExists(); Common.ResourceAccess.IButlerStorageManager storageManager = Common.ResourceAccess.BlobManagerFactory.CreateBlobManager(_Configuration.ProcessConfigConn); //1. Create Butler Response ButlerRequest myButlerRequest = Newtonsoft.Json.JsonConvert.DeserializeObject <ButlerRequest>(poisonMessage.AsString); ButlerResponse myButlerResponse = new ButlerResponse(); myButlerResponse.MezzanineFiles = myButlerRequest.MezzanineFiles; ////Add to Mezzamine Files the control File URL if it exist //Becouse it is needed to move/delete the control file from processing to succes or fail if (!string.IsNullOrEmpty(myButlerRequest.ControlFileUri)) { myButlerResponse.MezzanineFiles.Add(myButlerRequest.ControlFileUri); } myButlerResponse.TimeStampProcessingCompleted = DateTime.Now.ToString(); myButlerResponse.TimeStampProcessingStarted = DateTime.Now.ToString(); myButlerResponse.WorkflowName = myButlerRequest.WorkflowName; myButlerResponse.MessageId = myButlerRequest.MessageId; myButlerResponse.TimeStampRequestSubmitted = myButlerRequest.TimeStampUTC; myButlerResponse.StorageConnectionString = myButlerRequest.StorageConnectionString; myButlerResponse.Log = "Poison Message"; //2. Lookin for Errors in Table Status string processId = myProcessHandler.getProcessId(myButlerRequest.ControlFileUri, myButlerRequest.MessageId.ToString()); var processSnap = storageManager.readProcessSanpShot(myButlerRequest.WorkflowName, processId); if (processSnap != null) { //2.1 Load Erros dynamic dynObj = Newtonsoft.Json.JsonConvert.DeserializeObject((processSnap).jsonContext); var errorList = Newtonsoft.Json.JsonConvert.DeserializeObject <List <string> >((dynObj.Exceptions.ToString())); foreach (var errorTxt in errorList) { myButlerResponse.Log += "\r\n" + errorTxt; } //2.2 Update status processSnap.CurrentStep = MBFConfiguration.poisonFinishProcessStep; storageManager.PersistProcessStatus(processSnap); } else { //No process execution, fatal error //Update satus no process Status ProcessSnapShot psPoison = new ProcessSnapShot(myButlerResponse.WorkflowName, processId); psPoison.CurrentStep = MBFConfiguration.workflowFatalError; storageManager.PersistProcessStatus(psPoison); } //3. Send Poison Mesagge CloudQueueMessage poison = new CloudQueueMessage(Newtonsoft.Json.JsonConvert.SerializeObject(myButlerResponse)); poisonQueue.AddMessage(poison); sw = true; //4. Send On Error HTTP Callback Notification try { var StorageManager = ResourceAccess.BlobManagerFactory.CreateBlobManager(_Configuration.ProcessConfigConn); string jsonProcessConfiguration = StorageManager.GetButlerConfigurationValue( ProcessConfigKeys.DefualtPartitionKey, myButlerRequest.WorkflowName + ".config"); var processConfiguration = new ResourceAccess.jsonKeyValue(jsonProcessConfiguration); if (processConfiguration.Read(ProcessConfigKeys.MediaButlerHostHttpCallBackOnError) != "") { //POST string url = processConfiguration.Read(ProcessConfigKeys.MediaButlerHostHttpCallBackOnError); using (var client = new HttpClient()) { var content = new StringContent( StorageManager.readProcessSanpShot(myButlerRequest.WorkflowName, processId).jsonContext, Encoding.UTF8, "application/json"); var result = client.PostAsync(url, content).Result; Trace.TraceInformation("Http Post Notification Result: " + result.ToString()); } } } catch (Exception X) { Trace.TraceError("HTTP CALLBACK ERROR " + X.Message); } } catch (Exception X) { string txt = string.Format("[{0}] at {1} has an error: {2}", this.GetType().FullName, "GetNewMessage", X.Message); Trace.TraceError(txt); } return(sw); }