public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context)
{
try
{
Log.LogDebug("Reached AddAction of cloud" + action.GetFullName());
Log.LogDebug("Adding action " + action.GetFullName());
string actionGuid = Guid.NewGuid().ToString();
// Upload action directory to blob storage.
string actionPath = Utility.GetActionDirectory(action);
string zipPath = Path.Combine(Path.GetTempPath(), actionGuid);
if (File.Exists(zipPath))
{
File.Delete(zipPath);
}
ZipFile.CreateFromDirectory(actionPath, zipPath);
Log.LogDebug("Created zip for " + actionPath);
CloudBlockBlob blob = inputContainer.GetBlockBlobReference(actionGuid + ".zip");
blob.UploadFromFile(zipPath);
File.Delete(zipPath);
Log.LogDebug("Uploaded blob " + blob.Name);
// Add entry to table storage.
// TODO: Due to constraints on sizes of properties in Azure table entities, serializedAction cannot be larger
// than 64kB. Fix this if this becomes an issue.
byte[] serializedAction = Utility.ObjectToByteArray(action);
string moduleHash = string.Empty;
ActionsTableEntry entry = new ActionsTableEntry(action.name, actionGuid, schedulerInstanceGuid, serializedAction,
Utility.version, null, moduleHash);
tableDataSource.AddEntry(entry);
Log.LogDebug("Added to table " + entry.PartitionKey + "," + entry.RowKey);
CloudMessage cloudMessage = new CloudMessage();
cloudMessage.schedulerInstanceGuid = schedulerInstanceGuid;
cloudMessage.actionGuid = actionGuid;
cloudMessage.maxDequeueCount = maxDequeueCount;
cloudMessage.useDb = Utility.useDb;
cloudMessage.taskId = Utility.taskId;
string messageString = JsonConvert.SerializeObject(cloudMessage);
// Add message to queue.
//Log.LogInfo("Executing: " + action.GetFullName() + " [cloud id:" + actionGuid + "]");
//string messageString = schedulerInstanceGuid + "," + actionGuid + "," + maxDequeueCount;
var message = new CloudQueueMessage(messageString);
actionsQueue.AddMessage(message);
Log.LogDebug("Adding to queue " + message.Id);
contextDictionary[actionGuid] = new CloudActionCompleteContext(action, callback, context);
Log.LogDebug("Done adding.");
} catch (Exception e)
{
Utility.scheduler.Dispose();
Log.LogFatalError(e.ToString());
}
}
public void cloudExceptionCallback(BrokeredMessage message)
{
var actionGuid = (string)message.Properties["ActionGuid"];
CloudActionCompleteContext context = contextDictionary[actionGuid];
ActionsTableEntry entry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid);
var action = (SMVAction)Utility.ByteArrayToObject(entry.SerializedAction);
context.action.analysisProperty = action.analysisProperty;
context.action.result = action.result;
context.action.variables = action.variables;
var results = new SMVActionResult[] { context.action.result };
context.callback(context.action, results, context.context);
}
/// <summary>
/// Callback that is called when SubscriptionClient.BeginReceive() receives a message.
/// </summary>
/// <param name="ar"></param>
private void ActionComplete(BrokeredMessage message)
{
var actionGuid = (string)message.Properties["ActionGuid"];
var waitTime = (TimeSpan)message.Properties["WaitTime"]; // The amount of time the rule had to wait before it started being processed.
var dequeueCount = (int)message.Properties["DequeueCount"]; // The number of times the message we sent was dequeued by the workers.
message.Complete();
CloudActionCompleteContext context = contextDictionary[actionGuid];
ActionsTableEntry entry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid);
var action = (SMVAction)Utility.ByteArrayToObject(entry.SerializedAction);
Log.LogDebug("Reached ActionComplete of Cloud " + action.GetFullName());
if (action.result == null)
{
action.result = new SMVActionResult(action.name, "NO OUTPUT?", false, false, 0);
Log.LogError(string.Format("Failed to complete action: {0} ({1})", actionGuid, context.action.name));
}
Log.LogDebug("ActionComplete for " + action.GetFullName() + " [cloud id " + actionGuid + "]");
// Populate the original action object so that the master scheduler gets the changes to the action object.
context.action.analysisProperty = action.analysisProperty;
context.action.result = action.result;
context.action.variables = action.variables;
var results = new SMVActionResult[] { context.action.result };
if (entry.Status != (int)ActionStatus.Complete)
{
Log.LogError(string.Format("Failed to complete action: {0} ({1})", actionGuid, context.action.name));
context.callback(context.action, new SMVActionResult[] { context.action.result }, context.context);
Utility.scheduler.errorsEncountered = true;
return;
}
// Download and extract the results.
string actionDirectory = Utility.GetActionDirectory(context.action);
CloudBlockBlob resultsBlob = outputContainer.GetBlockBlobReference(actionGuid + ".zip");
string zipPath = Path.Combine(Path.GetTempPath(), actionGuid);
if (File.Exists(zipPath))
{
File.Delete(zipPath);
}
if (resultsBlob.Exists())
{
resultsBlob.DownloadToFile(zipPath, FileMode.CreateNew);
resultsBlob.Delete();
using (var archive = ZipFile.OpenRead(zipPath))
{
foreach (var f in archive.Entries)
{
var toBeExtractedFilePath = Path.Combine(actionDirectory, f.FullName);
if (File.Exists(toBeExtractedFilePath))
{
File.Delete(toBeExtractedFilePath);
}
}
archive.ExtractToDirectory(actionDirectory);
}
File.Delete(zipPath);
// Write to the cloudstats.txt file.
var contents = new string[] { "Wait Time: " + waitTime.ToString(),
"Dequeue Count: " + dequeueCount,
"Output" + Environment.NewLine + results.First().output };
File.AppendAllLines(Path.Combine(actionDirectory, "cloudstats.txt"), contents);
Log.LogDebug("download results for " + action.GetFullName() + " [cloud id " + actionGuid + "]");
}
else
{
Log.LogInfo("Results for " + action.GetFullName() + " [cloud id " + actionGuid + "] not available!");
}
context.callback(context.action, results, context.context);
}
public override void Run()
{
while (acceptingMessages)
{
try
{
currentMessage = inputQueue.GetMessage(TimeSpan.FromHours(1));
if (currentMessage != null)
{
string schedulerInstanceGuid = String.Empty;
string actionGuid = String.Empty;
int maxDequeueCount = 10;
Boolean useDb = false;
string taskId = String.Empty;
try
{
CloudMessage message = JsonConvert.DeserializeObject <CloudMessage>(currentMessage.AsString);
schedulerInstanceGuid = message.schedulerInstanceGuid;
actionGuid = message.actionGuid;
maxDequeueCount = message.maxDequeueCount;
useDb = message.useDb;
taskId = message.taskId;
} catch (Exception e)
{
Log.LogInfo("Json message parsing failed. Trying old message parsing.");
// Parse the message.
string[] msgParts = currentMessage.AsString.Split(',');
schedulerInstanceGuid = msgParts[0];
actionGuid = msgParts[1];
maxDequeueCount = 10;
try
{
if (msgParts.Count() > 2)
{
maxDequeueCount = Convert.ToInt32(msgParts[2]);
}
}
catch (Exception) {
Log.LogError("Message parsing failed.");
}
}
// Get the table entry.
ActionsTableEntry tableEntry = tableDataSource.GetEntry(schedulerInstanceGuid, actionGuid);
tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.InProgress);
CloudBlockBlob jobBlob = jobsContainer.GetBlockBlobReference(actionGuid + ".zip");
using (var outputMsg = new BrokeredMessage())
{
outputMsg.Properties["SchedulerInstanceGuid"] = schedulerInstanceGuid;
outputMsg.Properties["ActionGuid"] = actionGuid;
outputMsg.Properties["DequeueCount"] = currentMessage.DequeueCount;
outputMsg.Properties["WaitTime"] = DateTime.Now - currentMessage.InsertionTime;
// Check if we have tried to process this message too many times.
// If so, delete it and report an error back to the client.
if (currentMessage.DequeueCount >= maxDequeueCount)
{
tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Error);
SendMessageToTopic(outputMsg);
inputQueue.DeleteMessage(currentMessage);
jobBlob.Delete();
continue;
}
// Switch the version of SMV if required.
if (!SetSmvVersion(tableEntry.Version))
{
Trace.TraceError("Could not set SMV version.");
tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Error);
SendMessageToTopic(outputMsg);
inputQueue.DeleteMessage(currentMessage);
jobBlob.Delete();
continue;
}
// Download the job and extract it to the working directory.
Utility.ClearDirectory(workingDirectory);
string jobZipPath = Path.Combine(workingDirectory, "job.zip");
jobBlob.DownloadToFile(jobZipPath, FileMode.CreateNew);
ZipFile.ExtractToDirectory(jobZipPath, workingDirectory);
File.Delete(jobZipPath);
// Deserialize the action.
SMVAction action = (SMVAction)Utility.ByteArrayToObject(tableEntry.SerializedAction);
// Get ready to execute the action.
// We substitute the value of assemblyDir and workingDir with the values on this machine.
string oldWorkingDir = action.variables["workingDir"].ToLower();
string oldAssemblyDir = action.variables["assemblyDir"].ToLower();
string newAssemblyDir = Path.Combine(smvDirectory, "bin").ToLower();
workingDirectory = workingDirectory.ToLower();
var keys = new List <string>(action.variables.Keys);
foreach (var key in keys)
{
if (!string.IsNullOrEmpty(action.variables[key]))
{
if (action.variables[key].ToLower().StartsWith(oldAssemblyDir))
{
action.variables[key] = action.variables[key].ToLower().Replace(oldAssemblyDir, newAssemblyDir);
}
else if (action.variables[key].ToLower().StartsWith(oldWorkingDir))
{
action.variables[key] = action.variables[key].ToLower().Replace(oldWorkingDir, workingDirectory);
}
}
}
// NOTE: We set the Path attribute in the action to null because the action is always processed in the working directory.
var path = action.Path;
action.Path = null;
Utility.SetSmvVar("workingDir", workingDirectory);
// Execute the action.
SMVActionResult result = Utility.ExecuteAction(action, true, useDb, taskId);
// Change the paths back to their old values.
foreach (var key in keys)
{
if (!string.IsNullOrEmpty(action.variables[key]))
{
if (action.variables[key].ToLower().StartsWith(newAssemblyDir))
{
action.variables[key] = action.variables[key].ToLower().Replace(newAssemblyDir, oldAssemblyDir);
}
else if (action.variables[key].ToLower().StartsWith(workingDirectory))
{
action.variables[key] = action.variables[key].ToLower().Replace(workingDirectory, oldWorkingDir);
}
}
}
// Now set the path attribute again because the client needs it.
action.Path = path;
action.result.output = action.result.output.Substring(1, 900) + "... (truncated)";
// Zip up the working directory and upload it as the result.
string resultsZipPath = Path.Combine(resultsDirectory, actionGuid + ".zip");
ZipFile.CreateFromDirectory(workingDirectory, resultsZipPath);
CloudBlockBlob resultsBlob = resultsContainer.GetBlockBlobReference(actionGuid + ".zip");
resultsBlob.UploadFromFile(resultsZipPath);
File.Delete(resultsZipPath);
// Job done!
tableDataSource.UpdateAction(schedulerInstanceGuid, actionGuid, Utility.ObjectToByteArray(action));
tableDataSource.UpdateStatus(schedulerInstanceGuid, actionGuid, ActionStatus.Complete);
SendMessageToTopic(outputMsg);
if (currentMessage != null)
{
inputQueue.DeleteMessage(currentMessage);
currentMessage = null;
}
jobBlob.DeleteIfExists();
Utility.ClearDirectory(workingDirectory);
}
}
}
catch (Exception e)
{
Trace.TraceError("Exception while processing queue item:" + e.ToString());
if (currentMessage != null)
{
inputQueue.UpdateMessage(currentMessage, TimeSpan.FromSeconds(5), MessageUpdateFields.Visibility);
}
}
}
}
