public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context)
        {
            Log.LogDebug("Reached Add Action of Local " + action.GetFullName());
            var entry = new ActionsQueueEntry(action, callback, context);

            actionsQueue.Enqueue(entry);
        }
        /// <summary>
        /// This function runs in its own thread, dequeueing actions from the actions queue and sending them to the
        /// appropriate scheduler.
        /// </summary>
        private void Execute()
        {
            while (!done)
            {
                ActionsQueueEntry entry;
                while (!done && actionsQueue.TryDequeue(out entry))
                {
                    SMVAction action        = entry.Action;
                    string    schedulerType = action.executeOn;

                    if (!schedulers.ContainsKey(schedulerType))
                    {
                        Log.LogFatalError("Could not find scheduler of type: " + schedulerType +
                                          " while executing action " + action.name);
                    }
                    else
                    {
                        Log.LogDebug("scheduler found for " + schedulerType);
                        ISMVActionScheduler scheduler = schedulers[schedulerType];
                        lock (Utility.lockObject)
                        {
                            Utility.result.Add(action.GetFullName(), "Skipped");
                        }
                        scheduler.AddAction(action, new SMVActionCompleteCallBack(ActionComplete), entry);
                    }
                }
                System.Threading.Thread.Sleep(2000);
            }
        }
Exemple #3
0
 public ActionsQueueEntry(SMVAction action, SMVActionCompleteCallBack callback, object context)
 {
     this.Action   = action;
     this.Callback = callback;
     this.Context  = context;
     this.Results  = new List <SMVActionResult>();
 }
Exemple #4
0
        public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context)
        {
            try
            {
                Log.LogDebug("Reached AddAction of cloud" + action.GetFullName());
                Log.LogDebug("Adding action " + action.GetFullName());
                string actionGuid = Guid.NewGuid().ToString();
                // Upload action directory to blob storage.

                string actionPath = Utility.GetActionDirectory(action);
                string zipPath    = Path.Combine(Path.GetTempPath(), actionGuid);
                if (File.Exists(zipPath))
                {
                    File.Delete(zipPath);
                }
                ZipFile.CreateFromDirectory(actionPath, zipPath);
                Log.LogDebug("Created zip for " + actionPath);

                CloudBlockBlob blob = inputContainer.GetBlockBlobReference(actionGuid + ".zip");
                blob.UploadFromFile(zipPath);
                File.Delete(zipPath);
                Log.LogDebug("Uploaded blob " + blob.Name);

                // Add entry to table storage.

                // TODO: Due to constraints on sizes of properties in Azure table entities, serializedAction cannot be larger
                // than 64kB. Fix this if this becomes an issue.
                byte[]            serializedAction = Utility.ObjectToByteArray(action);
                string            moduleHash       = string.Empty;
                ActionsTableEntry entry            = new ActionsTableEntry(action.name, actionGuid, schedulerInstanceGuid, serializedAction,
                                                                           Utility.version, null, moduleHash);
                tableDataSource.AddEntry(entry);

                Log.LogDebug("Added to table " + entry.PartitionKey + "," + entry.RowKey);
                CloudMessage cloudMessage = new CloudMessage();
                cloudMessage.schedulerInstanceGuid = schedulerInstanceGuid;
                cloudMessage.actionGuid            = actionGuid;
                cloudMessage.maxDequeueCount       = maxDequeueCount;
                cloudMessage.useDb  = Utility.useDb;
                cloudMessage.taskId = Utility.taskId;

                string messageString = JsonConvert.SerializeObject(cloudMessage);
                // Add message to queue.
                //Log.LogInfo("Executing: " + action.GetFullName() + " [cloud id:" + actionGuid + "]");
                //string messageString = schedulerInstanceGuid + "," + actionGuid + "," + maxDequeueCount;
                var message = new CloudQueueMessage(messageString);
                actionsQueue.AddMessage(message);

                Log.LogDebug("Adding to queue " + message.Id);

                contextDictionary[actionGuid] = new CloudActionCompleteContext(action, callback, context);

                Log.LogDebug("Done adding.");
            } catch (Exception e)
            {
                Utility.scheduler.Dispose();
                Log.LogFatalError(e.ToString());
            }
        }
Exemple #5
0
        /// <summary>
        /// Call back used by ExecuteActions().
        /// </summary>
        /// <param name="results"></param>
        /// <param name="context"></param>
        static void DoneExecuteAction(SMVAction action, IEnumerable <SMVActionResult> results, object context)
        {
            actionResults.AddRange(results);

            var countDownEvent = context as CountdownEvent;

            countDownEvent.Signal();
        }
Exemple #6
0
        /// <summary>
        /// Call back used by ExecuteActions().
        /// </summary>
        /// <param name="results"></param>
        /// <param name="context"></param>
        static void DoneExecuteAction(SMVAction action, IEnumerable <SMVActionResult> results, object context)
        {
            Log.LogDebug("Reached DoneExecuteAction for " + action.GetFullName());
            actionResults.AddRange(results);

            var countDownEvent = context as CountdownEvent;

            countDownEvent.Signal();
        }
Exemple #7
0
        /// <summary>
        /// Get the path to an action's working directory.
        /// </summary>
        /// <param name="action">The action object.</param>
        /// <returns>Full path to the action's working directory.</returns>
        public static string GetActionDirectory(SMVAction action)
        {
            string path = string.Empty;

            if (action.Path != null && action.Path.value != null)
            {
                path = action.Path.value;
            }
            return(Utility.ExpandVariables(path, action.variables));
        }
Exemple #8
0
        /// <summary>
        /// Gets the child action for an action.
        /// </summary>
        /// <param name="action">The parent action.</param>
        /// <returns>The child action if one exists, else null.</returns>
        public static SMVAction GetNextAction(SMVAction action)
        {
            if (action.nextAction == null || !actionsDictionary.ContainsKey(action.nextAction))
            {
                return(null);
            }
            SMVAction template   = actionsDictionary[action.nextAction];
            SMVAction nextAction = new SMVAction(template, string.Empty);

            return(nextAction);
        }
        public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context)
        {
            if (!action.executeOn.Equals(Utility.schedulerType))
            {
                action.executeOn = "local";
            }
            Log.LogDebug("Queuing action: " + action.GetFullName() + " on " + action.executeOn);
            var entry = new ActionsQueueEntry(action, callback, context);

            actionsQueue.Enqueue(entry);
            counters.AddOrUpdate("queued", 1, (k, v) => v + 1);
        }
Exemple #10
0
 public SMVAction(SMVAction orig, string analysisProperty)
 {
     this.breakOnError     = orig.breakOnError;
     this.name             = orig.name;
     this.Command          = orig.Command;
     this.Path             = orig.Path;
     this.Env              = orig.Env;
     this.CopyArtifact     = orig.CopyArtifact;
     this.executeOn        = orig.executeOn;
     this.nextAction       = orig.nextAction;
     this.variables        = orig.variables;
     this.analysisProperty = analysisProperty;
 }
Exemple #11
0
        public void AddAction(SMVAction action, SMVActionCompleteCallBack callback, object context)
        {
            var entry = new ActionsQueueEntry(action, callback, context);

            actionsQueue.Enqueue(entry);
        }
Exemple #12
0
 public CloudActionCompleteContext(SMVAction _action, SMVActionCompleteCallBack _callback, object _context)
 {
     action   = _action;
     callback = _callback;
     context  = _context;
 }
        /// <summary>
        /// This callback function is called once an action and all its children have executed.
        /// </summary>
        /// <param name="results">A list of results, one for each action (the action added to the queue and its children).</param>
        /// <param name="context">A context object.</param>
        private void ActionComplete(SMVAction a, IEnumerable <SMVActionResult> results, object context)
        {
            var       entry  = context as ActionsQueueEntry;
            SMVAction action = entry.Action;
            SMVActionCompleteCallBack callback = entry.Callback;

            try
            {
                if (action.result == null)
                {
                    action.result = new SMVActionResult(action.name, "NO OUTPUT?", false, false, 0);
                }

                entry.Results.AddRange(results);
                counters.AddOrUpdate("completed", 1, (k, v) => v + 1);
                times.Add(action.result.time);

                File.WriteAllText(Path.Combine(Utility.GetActionDirectory(action), "smvstats.txt"),
                                  string.Format("Time: {0}", action.result.time));

                // Add result to our global result set.
                string result = "Failed";
                if (action.result != null && action.result.isSuccessful)
                {
                    result = "Success";
                }
                // Otherwise, add the next action to the queue, if any.
                else
                {
                    errorsEncountered = true;
                }
                lock (Utility.lockObject)
                {
                    Utility.result[action.GetFullName()] = result;
                }

                // If there was an error, simply call the callback function with whatever results we have, the callback is
                // expected to handle the errors by looking at the list of results.
                if (action.result == null || action.result.breakExecution || !action.result.isSuccessful)
                {
                    entry.Callback(action, entry.Results, entry.Context);
                }
                // Otherwise, add the next action to the queue, if any.
                else
                {
                    SMVAction nextAction = Utility.GetNextAction(action);
                    if (nextAction != null)
                    {
                        nextAction.analysisProperty = action.analysisProperty;

                        DebugUtility.DumpVariables(entry.Action.variables, "entry.action");
                        DebugUtility.DumpVariables(Utility.smvVars, "smvvars");

                        nextAction.variables = Utility.smvVars.Union(entry.Action.variables).ToDictionary(g => g.Key, g => g.Value);
                        this.AddAction(nextAction, entry.Callback, entry.Context);
                    }
                    else
                    {
                        entry.Callback(action, entry.Results, entry.Context);
                    }
                }
            }
            catch (Exception e)
            {
                Log.LogError("Error processing finalization for action " + action.GetFullName());

                action.result.output = e.ToString();
                entry.Callback(action, entry.Results, entry.Context);
            }

            // upate status line
            if (updateStatusMode)
            {
                string resultString = string.Format("\r[INFO] {0} of {1} jobs remaining. Avg(s): {2}. Std.Dev(s): {3}",
                                                    counters["queued"] - counters["completed"], counters["queued"],
                                                    times.Average().ToString("F2"),
                                                    Math.Sqrt(times.Average(v => Math.Pow(v - times.Average(), 2))).ToString("F2"));
                Console.Write(resultString);
                if (counters["queued"] == counters["completed"])
                {
                    Console.WriteLine();
                }
            }
        }
Exemple #14
0
        /// <summary>
        /// Called by schedulers to execute an action.
        /// </summary>
        /// <param name="action">The action to be executed.</param>
        /// <returns>An SMVActionResult object representing the result of executing the action.</returns>
        public static SMVActionResult ExecuteAction(SMVAction action, bool fromWorker)
        {
            // NOTE: The code in this function must be thread safe.
            if (action == null)
            {
                return(null);
            }

            // If there is a plugin, call PreAction first.
            if (plugin != null)
            {
                plugin.PreAction(action);
            }

            using (MemoryStream stream = new MemoryStream())
            {
                // We use a logger for writing messages since we can't output to the console in this function (As this
                // may be running in multiple threads).
                StreamWriter logger = new StreamWriter(stream);
                IDictionary <string, string> variables = action.variables;
                DateTime startTime          = DateTime.Now;
                string   actionPath         = variables["workingDir"];
                string   actionOutput       = string.Empty;
                int      cumulativeExitCode = 0;

                // Get the name of the action.
                string name = action.name;
                if (variables.ContainsKey("analysisProperty"))
                {
                    name = action.name + " - " + variables["analysisProperty"];
                }
                variables["name"] = action.name;
                Log.LogInfo("Running action: " + name, logger);

                // Get the path to the action.
                if (action.Path != null)
                {
                    actionPath = action.Path.value;
                }
                actionPath = ExpandVariables(actionPath, variables);
                variables["actionPath"] = actionPath;

                // Launch a cmd.exe process to run commands in.
                if (Console.InputEncoding.Equals(Encoding.UTF8))
                {
                    Console.InputEncoding = new UTF8Encoding(false);
                }


                // Run the commands.
                if (action.Command != null)
                {
                    foreach (SMVCommand cmd in action.Command)
                    {
                        JobObject jobObject = null;
                        if (useJobObject)
                        {
                            //Update maxTime and maxMemory allowed
                            int maxMemory = int.MaxValue;
                            int maxTime   = int.MaxValue;
                            updateAttribute(ref maxTime, cmd.maxTime, "Time");
                            Log.LogDebug("Maximum time allowed for this command = " + maxTime);
                            updateAttribute(ref maxMemory, cmd.maxMemory, "Memory");

                            //Converting memory from MB to bytes, if input is valid
                            if (maxMemory < int.MaxValue)
                            {
                                maxMemory *= (1024 * 1024);
                            }
                            Log.LogDebug("Maximum memory allowed for this command = " + maxMemory);
                            jobObject = new JobObject();
                            jobObject.setConstraints(maxMemory, maxTime);
                        }
                        Process process = LaunchProcess("cmd.exe", "", actionPath, action.Env, logger, jobObject);
                        process.OutputDataReceived += (sender, e) => { Log.LogMessage(e.Data, logger); };
                        process.ErrorDataReceived  += (sender, e) => { Log.LogMessage(e.Data, logger); };

                        // Get the command and arguments, and expand all environment as well as SMV variables.
                        string cmdAttr       = ExpandVariables(Environment.ExpandEnvironmentVariables(cmd.value), variables);
                        string argumentsAttr = string.Empty;
                        if (!string.IsNullOrEmpty(cmd.arguments))
                        {
                            argumentsAttr = ExpandVariables(Environment.ExpandEnvironmentVariables(cmd.arguments), variables);
                        }

                        try
                        {
                            Log.LogInfo(String.Format(CultureInfo.InvariantCulture, "Launching {0} with arguments: {1}", cmdAttr, argumentsAttr), logger);
                            process.StandardInput.WriteLine(String.Join(" ", new String[] { cmdAttr, argumentsAttr }));
                            process.StandardInput.WriteLine("Exit %errorlevel%");
                            process.StandardInput.Close();
                            process.BeginOutputReadLine();
                            process.BeginErrorReadLine();
                            process.WaitForExit();
                            TimeSpan span = process.ExitTime - process.StartTime;
                            Log.LogMessage(string.Format("Command Exit code: {0}", process.ExitCode), logger);
                            cumulativeExitCode += Math.Abs(process.ExitCode);
                            if (useDb)
                            {
                                try
                                {
                                    using (var database = new SmvDbEntities())
                                    {
                                        var masterEntry = new TaskAction
                                        {
                                            ActionID         = Guid.NewGuid().ToString(),
                                            TaskID           = taskId,
                                            ActionName       = action.name,
                                            Success          = cumulativeExitCode.ToString(),
                                            ActionTime       = span.ToString(),
                                            WorkingDirectory = variables["workingDir"]
                                        };
                                        database.TaskActions.Add(masterEntry);
                                        database.SaveChanges();
                                    }
                                }
                                catch (Exception e)
                                {
                                    if (fromWorker)
                                    {
                                        Log.LogError("Exception while updating database " + e);
                                    }
                                    else
                                    {
                                        Log.LogFatalError("Exception while updating database " + e);
                                    }
                                }
                            }
                            if (useJobObject)
                            {
                                jobObject.QueryExtendedLimitInformation();
                                jobObject.Close();
                                jobObject.Dispose();
                            }
                        }
                        catch (Exception e)
                        {
                            Log.LogInfo(e.ToString(), logger);
                            Log.LogInfo("Could not start process: " + cmdAttr, logger);
                            if (useJobObject)
                            {
                                jobObject.Close();
                                jobObject.Dispose();
                            }
                            return(null);
                        }
                    }
                }

                logger.Flush();
                stream.Position = 0;
                string output = new StreamReader(stream).ReadToEnd();

                if (debugMode)
                {
                    Log.WriteToFile(Path.Combine(actionPath, string.Format("smvexecute-{0}.log", action.name)), output, false);
                }
                Log.LogDebug("cumulative exit code is " + cumulativeExitCode);

                DateTime endTime = DateTime.Now;

                action.result = new SMVActionResult(action.name, output, (cumulativeExitCode == 0),
                                                    cumulativeExitCode != 0 && action.breakOnError, (int)(endTime - startTime).TotalSeconds);

                // Call plugin post action only if we were successful in executing the action.
                if (cumulativeExitCode == 0)
                {
                    // get the output directory and set the output of the action from the build log.
                    if (action.name.Equals("NormalBuild"))
                    {
                        string logPath = Path.Combine(variables["workingDir"], variables["smvLogFileNamePrefix"] + ".log");
                        action.result.output = Utility.ReadFile(logPath);

                        variables["outputDir"] = ExtractBuildPath(variables["workingDir"], action.result.output, logger);
                        Utility.SetSmvVar("outputDir", variables["outputDir"]);
                    }

                    // Get the output directory and the analysis directory.
                    if (action.name.Equals("InterceptedBuild"))
                    {
                        string logPath = Path.Combine(variables["workingDir"], variables["smvLogFileNamePrefix"] + ".log");
                        action.result.output = Utility.ReadFile(logPath);
                    }

                    // Call the plugin's post action.
                    if (plugin != null)
                    {
                        plugin.PostAction(action);
                    }
                }
                else
                {
                    // are we sure we want to exit here... the cloud worker instance becomes
                    // unhealthy after exiting here...
                    if (action.breakOnError)
                    {
                        if (fromWorker)
                        {
                            Log.LogError(String.Format("Action: {0}, failed.", name));
                        }
                        else
                        {
                            Log.LogFatalError(String.Format("Action: {0}, failed.", name));
                        }
                    }
                    else
                    {
                        Log.LogError(String.Format("Action: {0}, failed.", name));
                    }
                }

                return(action.result);
            }
        }