public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context) { try { Log.LogDebug("Reached AddAction of cloud" + action.GetFullName()); Log.LogDebug("Adding action " + action.GetFullName()); string actionGuid = Guid.NewGuid().ToString(); // Upload action directory to blob storage. string actionPath = Utility.GetActionDirectory(action); string zipPath = Path.Combine(Path.GetTempPath(), actionGuid); if (File.Exists(zipPath)) { File.Delete(zipPath); } ZipFile.CreateFromDirectory(actionPath, zipPath); Log.LogDebug("Created zip for " + actionPath); CloudBlockBlob blob = inputContainer.GetBlockBlobReference(actionGuid + ".zip"); blob.UploadFromFile(zipPath); File.Delete(zipPath); Log.LogDebug("Uploaded blob " + blob.Name); // Add entry to table storage. // TODO: Due to constraints on sizes of properties in Azure table entities, serializedAction cannot be larger // than 64kB. Fix this if this becomes an issue. byte[] serializedAction = Utility.ObjectToByteArray(action); string moduleHash = string.Empty; ActionsTableEntry entry = new ActionsTableEntry(action.name, actionGuid, schedulerInstanceGuid, serializedAction, Utility.version, null, moduleHash); tableDataSource.AddEntry(entry); Log.LogDebug("Added to table " + entry.PartitionKey + "," + entry.RowKey); CloudMessage cloudMessage = new CloudMessage(); cloudMessage.schedulerInstanceGuid = schedulerInstanceGuid; cloudMessage.actionGuid = actionGuid; cloudMessage.maxDequeueCount = maxDequeueCount; cloudMessage.useDb = Utility.useDb; cloudMessage.taskId = Utility.taskId; string messageString = JsonConvert.SerializeObject(cloudMessage); // Add message to queue. //Log.LogInfo("Executing: " + action.GetFullName() + " [cloud id:" + actionGuid + "]"); //string messageString = schedulerInstanceGuid + "," + actionGuid + "," + maxDequeueCount; var message = new CloudQueueMessage(messageString); actionsQueue.AddMessage(message); Log.LogDebug("Adding to queue " + message.Id); contextDictionary[actionGuid] = new CloudActionCompleteContext(action, callback, context); Log.LogDebug("Done adding."); } catch (Exception e) { Utility.scheduler.Dispose(); Log.LogFatalError(e.ToString()); } }
public void cloudExceptionCallback(BrokeredMessage message) { var actionGuid = (string)message.Properties["ActionGuid"]; CloudActionCompleteContext context = contextDictionary[actionGuid]; ActionsTableEntry entry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid); var action = (SMVAction)Utility.ByteArrayToObject(entry.SerializedAction); context.action.analysisProperty = action.analysisProperty; context.action.result = action.result; context.action.variables = action.variables; var results = new SMVActionResult[] { context.action.result }; context.callback(context.action, results, context.context); }
/// <summary> /// Callback that is called when SubscriptionClient.BeginReceive() receives a message. /// </summary> /// <param name="ar"></param> private void ActionComplete(BrokeredMessage message) { var actionGuid = (string)message.Properties["ActionGuid"]; var waitTime = (TimeSpan)message.Properties["WaitTime"]; // The amount of time the rule had to wait before it started being processed. var dequeueCount = (int)message.Properties["DequeueCount"]; // The number of times the message we sent was dequeued by the workers. message.Complete(); CloudActionCompleteContext context = contextDictionary[actionGuid]; ActionsTableEntry entry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid); var action = (SMVAction)Utility.ByteArrayToObject(entry.SerializedAction); Log.LogDebug("Reached ActionComplete of Cloud " + action.GetFullName()); if (action.result == null) { action.result = new SMVActionResult(action.name, "NO OUTPUT?", false, false, 0); Log.LogError(string.Format("Failed to complete action: {0} ({1})", actionGuid, context.action.name)); } Log.LogDebug("ActionComplete for " + action.GetFullName() + " [cloud id " + actionGuid + "]"); // Populate the original action object so that the master scheduler gets the changes to the action object. context.action.analysisProperty = action.analysisProperty; context.action.result = action.result; context.action.variables = action.variables; var results = new SMVActionResult[] { context.action.result }; if (entry.Status != (int)ActionStatus.Complete) { Log.LogError(string.Format("Failed to complete action: {0} ({1})", actionGuid, context.action.name)); context.callback(context.action, new SMVActionResult[] { context.action.result }, context.context); Utility.scheduler.errorsEncountered = true; return; } // Download and extract the results. string actionDirectory = Utility.GetActionDirectory(context.action); CloudBlockBlob resultsBlob = outputContainer.GetBlockBlobReference(actionGuid + ".zip"); string zipPath = Path.Combine(Path.GetTempPath(), actionGuid); if (File.Exists(zipPath)) { File.Delete(zipPath); } if (resultsBlob.Exists()) { resultsBlob.DownloadToFile(zipPath, FileMode.CreateNew); resultsBlob.Delete(); using (var archive = ZipFile.OpenRead(zipPath)) { foreach (var f in archive.Entries) { var toBeExtractedFilePath = Path.Combine(actionDirectory, f.FullName); if (File.Exists(toBeExtractedFilePath)) { File.Delete(toBeExtractedFilePath); } } archive.ExtractToDirectory(actionDirectory); } File.Delete(zipPath); // Write to the cloudstats.txt file. var contents = new string[] { "Wait Time: " + waitTime.ToString(), "Dequeue Count: " + dequeueCount, "Output" + Environment.NewLine + results.First().output }; File.AppendAllLines(Path.Combine(actionDirectory, "cloudstats.txt"), contents); Log.LogDebug("download results for " + action.GetFullName() + " [cloud id " + actionGuid + "]"); } else { Log.LogInfo("Results for " + action.GetFullName() + " [cloud id " + actionGuid + "] not available!"); } context.callback(context.action, results, context.context); }
public override void Run() { while (acceptingMessages) { try { currentMessage = inputQueue.GetMessage(TimeSpan.FromHours(1)); if (currentMessage != null) { string schedulerInstanceGuid = String.Empty; string actionGuid = String.Empty; int maxDequeueCount = 10; Boolean useDb = false; string taskId = String.Empty; try { CloudMessage message = JsonConvert.DeserializeObject <CloudMessage>(currentMessage.AsString); schedulerInstanceGuid = message.schedulerInstanceGuid; actionGuid = message.actionGuid; maxDequeueCount = message.maxDequeueCount; useDb = message.useDb; taskId = message.taskId; } catch (Exception e) { Log.LogInfo("Json message parsing failed. Trying old message parsing."); // Parse the message. string[] msgParts = currentMessage.AsString.Split(','); schedulerInstanceGuid = msgParts[0]; actionGuid = msgParts[1]; maxDequeueCount = 10; try { if (msgParts.Count() > 2) { maxDequeueCount = Convert.ToInt32(msgParts[2]); } } catch (Exception) { Log.LogError("Message parsing failed."); } } // Get the table entry. ActionsTableEntry tableEntry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid); tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.InProgress); CloudBlockBlob jobBlob = jobsContainer.GetBlockBlobReference(actionGuid + ".zip"); using (var outputMsg = new BrokeredMessage()) { outputMsg.Properties["SchedulerInstanceGuid"] = schedulerInstanceGuid; outputMsg.Properties["ActionGuid"] = actionGuid; outputMsg.Properties["DequeueCount"] = currentMessage.DequeueCount; outputMsg.Properties["WaitTime"] = DateTime.Now - currentMessage.InsertionTime; // Check if we have tried to process this message too many times. // If so, delete it and report an error back to the client. if (currentMessage.DequeueCount >= maxDequeueCount) { tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Error); SendMessageToTopic(outputMsg); inputQueue.DeleteMessage(currentMessage); jobBlob.Delete(); continue; } // Switch the version of SMV if required. if (!SetSmvVersion(tableEntry.Version)) { Trace.TraceError("Could not set SMV version."); tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Error); SendMessageToTopic(outputMsg); inputQueue.DeleteMessage(currentMessage); jobBlob.Delete(); continue; } // Download the job and extract it to the working directory. Utility.ClearDirectory(workingDirectory); string jobZipPath = Path.Combine(workingDirectory, "job.zip"); jobBlob.DownloadToFile(jobZipPath, FileMode.CreateNew); ZipFile.ExtractToDirectory(jobZipPath, workingDirectory); File.Delete(jobZipPath); // Deserialize the action. SMVAction action = (SMVAction)Utility.ByteArrayToObject(tableEntry.SerializedAction); // Get ready to execute the action. // We substitute the value of assemblyDir and workingDir with the values on this machine. string oldWorkingDir = action.variables["workingDir"].ToLower(); string oldAssemblyDir = action.variables["assemblyDir"].ToLower(); string newAssemblyDir = Path.Combine(smvDirectory, "bin").ToLower(); workingDirectory = workingDirectory.ToLower(); var keys = new List <string>(action.variables.Keys); foreach (var key in keys) { if (!string.IsNullOrEmpty(action.variables[key])) { if (action.variables[key].ToLower().StartsWith(oldAssemblyDir)) { action.variables[key] = action.variables[key].ToLower().Replace(oldAssemblyDir, newAssemblyDir); } else if (action.variables[key].ToLower().StartsWith(oldWorkingDir)) { action.variables[key] = action.variables[key].ToLower().Replace(oldWorkingDir, workingDirectory); } } } // NOTE: We set the Path attribute in the action to null because the action is always processed in the working directory. var path = action.Path; action.Path = null; Utility.SetSmvVar("workingDir", workingDirectory); // Execute the action. SMVActionResult result = Utility.ExecuteAction(action, true, useDb, taskId); // Change the paths back to their old values. foreach (var key in keys) { if (!string.IsNullOrEmpty(action.variables[key])) { if (action.variables[key].ToLower().StartsWith(newAssemblyDir)) { action.variables[key] = action.variables[key].ToLower().Replace(newAssemblyDir, oldAssemblyDir); } else if (action.variables[key].ToLower().StartsWith(workingDirectory)) { action.variables[key] = action.variables[key].ToLower().Replace(workingDirectory, oldWorkingDir); } } } // Now set the path attribute again because the client needs it. action.Path = path; action.result.output = action.result.output.Substring(1, 900) + "... (truncated)"; // Zip up the working directory and upload it as the result. string resultsZipPath = Path.Combine(resultsDirectory, actionGuid + ".zip"); ZipFile.CreateFromDirectory(workingDirectory, resultsZipPath); CloudBlockBlob resultsBlob = resultsContainer.GetBlockBlobReference(actionGuid + ".zip"); resultsBlob.UploadFromFile(resultsZipPath); File.Delete(resultsZipPath); // Job done! tableDataSource.UpdateAction(schedulerInstanceGuid, actionGuid, Utility.ObjectToByteArray(action)); tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Complete); SendMessageToTopic(outputMsg); if (currentMessage != null) { inputQueue.DeleteMessage(currentMessage); currentMessage = null; } jobBlob.DeleteIfExists(); Utility.ClearDirectory(workingDirectory); } } } catch (Exception e) { Trace.TraceError("Exception while processing queue item:" + e.ToString()); if (currentMessage != null) { inputQueue.UpdateMessage(currentMessage, TimeSpan.FromSeconds(5), MessageUpdateFields.Visibility); } } } }