/// <summary> /// Adds a droplet instance to the collection. /// </summary> /// <param name="instance">The droplet instance to add.</param> public void AddDropletInstance(DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } try { this.Lock.EnterWriteLock(); instance.Lock.EnterReadLock(); if (!this.Droplets.ContainsKey(instance.Properties.DropletId)) { this.Droplets.Add(instance.Properties.DropletId, new Droplet()); } this.Droplets[instance.Properties.DropletId].DropletInstances[instance.Properties.InstanceId] = instance; } finally { instance.Lock.ExitReadLock(); this.Lock.ExitWriteLock(); } this.ScheduleSnapshotAppState(); }
/// <summary> /// Removes a droplet instance from the collection. /// </summary> /// <param name="instance">The droplet instance to remove.</param> public void RemoveDropletInstance(DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } try { this.Lock.EnterWriteLock(); if (this.Droplets.ContainsKey(instance.Properties.DropletId)) { Droplet droplet = this.Droplets[instance.Properties.DropletId]; if (droplet.DropletInstances.ContainsKey(instance.Properties.InstanceId)) { droplet.DropletInstances.Remove(instance.Properties.InstanceId); if (droplet.DropletInstances.Count == 0) { this.Droplets.Remove(instance.Properties.DropletId); } } } } finally { this.Lock.ExitWriteLock(); } this.ScheduleSnapshotAppState(); }
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } DropletInstance instance = null; instance = new DropletInstance(); string instanceId = Guid.NewGuid().ToString("N"); instance.Properties.State = DropletInstanceState.Starting; instance.Properties.StateTimestamp = DateTime.Now; instance.Properties.Start = DateTime.Now; instance.Properties.InstanceId = instanceId; instance.Properties.DropletId = message.DropletId; instance.Properties.InstanceIndex = message.Index; instance.Properties.Name = message.Name; instance.Properties.Uris = message.Uris; instance.Properties.Users = message.Users; instance.Properties.Version = message.Version; instance.Properties.Framework = message.Framework; instance.Properties.Runtime = message.Runtime; instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index); this.AddDropletInstance(instance); return(instance); }
/// <summary> /// Untracks the memory used by the instance and flags it/ /// </summary> /// <param name="instance">The instance to be untracked.</param> public void RemoveInstanceResources(DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } try { this.Lock.EnterWriteLock(); instance.Lock.EnterWriteLock(); if (instance.Properties.ResourcesTracked) { instance.Properties.ResourcesTracked = false; this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024; this.Clients--; } } finally { instance.Lock.ExitWriteLock(); this.Lock.ExitWriteLock(); } }
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } DropletInstance instance = null; instance = new DropletInstance(); string instanceId = Credentials.GenerateSecureGuid().ToString("N"); string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N"); instance.Properties.State = DropletInstanceState.Starting; instance.Properties.StateTimestamp = DateTime.Now; instance.Properties.Start = DateTime.Now; instance.Properties.InstanceId = instanceId; instance.Properties.PrivateInstanceId = privateInstanceId; instance.Properties.DropletId = message.DropletId; instance.Properties.InstanceIndex = message.Index; instance.Properties.Name = message.Name; instance.Properties.Uris = message.Uris; instance.Properties.Users = message.Users; instance.Properties.Version = message.Version; instance.Properties.Stack = message.Stack; instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index); instance.Properties.Flapping = message.Flapping; instance.Properties.CloudControllerPartition = message.CloudControllerPartition; this.AddDropletInstance(instance); return(instance); }
/// <summary> /// Creates the application variable for an instance. Is is used for the plugin configuration. /// </summary> /// <param name="instance">The instance for which the application variable is to be generated.</param> /// <returns>The application variable.</returns> private string CreateInstanceVariable(DropletInstance instance) { List<string> whitelist = new List<string>() { "instance_id", "instance_index", "name", "uris", "users", "version", "start", "runtime", "state_timestamp", "port" }; Dictionary<string, object> result = new Dictionary<string, object>(); Dictionary<string, object> jsonInstance = instance.Properties.ToJsonIntermediateObject(); foreach (string key in whitelist) { if (jsonInstance.ContainsKey(key)) { // result[key] = JsonConvertibleObject.ObjectToValue<object>(jInstance[key]); result[key] = jsonInstance[key]; } } result["host"] = Host; result["limits"] = new Dictionary<string, object>() { { "fds", instance.Properties.FDSQuota }, { "mem", instance.Properties.MemoryQuotaBytes }, { "disk", instance.Properties.DiskQuotaBytes } }; return JsonConvertibleObject.SerializeToJson(result); }
private string CreateStartScript(DropletInstance instance) { string startCommand = StagingInfo.getStartCommand(Path.Combine(instance.Properties.Directory, "staging_info.yml")); var startScriptTemplate = @" set > {0}\logs\env.log cd {0}\app {1} > {0}\logs\stdout.log 2> {0}\logs\stderr.log "; string startScript = String.Format(startScriptTemplate, instance.Properties.Directory, startCommand); string scriptPath = Path.Combine(instance.Properties.Directory, "start.cmd"); File.WriteAllText(scriptPath, startScript, Encoding.ASCII); return scriptPath; }
/// <summary> /// Prepares the app directory. /// </summary> /// <param name="bitsFile">The bits file.</param> /// <param name="bitsUri">The bits URI.</param> /// <param name="hash">The sha1.</param> /// <param name="tarZipFile">The TGZ file.</param> /// <param name="instance">The instance.</param> public void PrepareAppDirectory(string bitsFile, string bitsUri, string hash, string tarZipFile, DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } // What we do here, in order of preference.. // 1. Check our own staged directory. // 2. Check shared directory from CloudController that could be mounted (bits_file) // 3. Pull from http if needed. string instanceDir = instance.Properties.Directory; lock (this.stagerLock) { // check before downloading if (instance.Properties.StopProcessed) { return; } if (File.Exists(tarZipFile)) { Logger.Debug(Strings.FoundStagedBitsInLocalCache); } else { // If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version. DateTime start = DateTime.Now; if (File.Exists(bitsFile)) { Logger.Debug(Strings.SharingCloudControllerStagingDirectory); File.Copy(bitsFile, tarZipFile); Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start); } else { Uri downloadUri = new Uri(bitsUri); Logger.Debug(Strings.Needtodownloadappbitsfrom, downloadUri); this.DownloadAppBits(downloadUri, hash, tarZipFile); Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start); } } // check before extracting if (instance.Properties.StopProcessed) { return; } DateTime startStageing = DateTime.Now; // Explode the app into its directory and optionally bind its local runtime. Directory.CreateDirectory(instance.Properties.Directory); DirectoryInfo deploymentDirInfo = new DirectoryInfo(instance.Properties.Directory); DirectorySecurity deploymentDirSecurity = deploymentDirInfo.GetAccessControl(); // Owner is important to account for disk quota deploymentDirSecurity.SetOwner(new NTAccount(instance.Properties.WindowsUserName)); deploymentDirSecurity.SetAccessRule( new FileSystemAccessRule( instance.Properties.WindowsUserName, FileSystemRights.Write | FileSystemRights.Read | FileSystemRights.Delete | FileSystemRights.Modify | FileSystemRights.FullControl, InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, PropagationFlags.None | PropagationFlags.InheritOnly, AccessControlType.Allow)); // Taking ownership of a file has to be executed with restore privilege elevated privilages using (new ProcessPrivileges.PrivilegeEnabler(Process.GetCurrentProcess(), ProcessPrivileges.Privilege.Restore)) { deploymentDirInfo.SetAccessControl(deploymentDirSecurity); } // Impersonate user to cascade the owernship to every file // Neccessary for windows disk quota using (new UserImpersonator(instance.Properties.WindowsUserName, ".", instance.Properties.WindowsPassword, true)) { DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip string tarFileName = Directory.GetFiles(instanceDir, "*.tar")[0]; DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar File.Delete(Path.Combine(instanceDir, tarFileName)); } Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing); } }
/// <summary> /// Registers the instance with the Vcap router. Called when the application is running and ready. /// </summary> /// <param name="instance">The instance to be registered.</param> private void RegisterInstanceWithRouter(DropletInstance instance) { RouterMessage response = new RouterMessage(); try { instance.Lock.EnterReadLock(); if (instance.Properties.Uris == null || instance.Properties.Uris.Length == 0) { return; } response.DeaId = UUID; response.Host = Host; response.Port = instance.Properties.Port; response.Uris = new List<string>(instance.Properties.Uris).ToArray(); response.Tags = new RouterMessage.TagsObject(); response.Tags.Framework = instance.Properties.Framework; response.Tags.Runtime = instance.Properties.Runtime; } finally { instance.Lock.ExitReadLock(); } this.deaReactor.SendRouterRegister(response.SerializeToJson()); }
/// <summary> /// Detects the if an app is ready and run the callback. /// </summary> /// <param name="instance">The instance to be checked.</param> /// <param name="callBack">The call back.</param> private static void DetectAppReady(DropletInstance instance, BoolStateBlockCallback callBack) { DetectPortReady(instance, callBack); }
private void CheckUsage(DropletInstance instance) { DropletInstanceUsage curUsage = instance.Properties.LastUsage; if (instance == null || curUsage == null) { return; } //// Check Memory //// Memory usage also enforced by windows job object //if (curUsage.MemoryBytes > (instance.Properties.MemoryQuotaBytes)) //{ // instance.ErrorLog.Fatal( // "Memory size usage exceeded the limit of {0} MiB. Memory size used: {1} MiB. Stopping the app instance.", // instance.Properties.MemoryQuotaBytes / 1024 / 1024, // curUsage.MemoryBytes / 1024 / 1024); // this.StopDroplet(instance); //} //// Check Disk //// Disk usage also enforced by windows disk quota //if (curUsage.DiskBytes > instance.Properties.DiskQuotaBytes * 1.05) //{ // instance.ErrorLog.Fatal( // "Disk size usage exceeded the limit of {0} MiB. Disk size used: {1} MiB. Stopping the app instance.", // instance.Properties.DiskQuotaBytes / 1024 / 1024, // curUsage.DiskBytes / 1024 / 1024); // this.StopDroplet(instance); //} // Check CPU if (instance.Usage.Count == 0) { return; } if (curUsage.Cpu > Monitoring.BeginReniceCpuThreshold) { int nice = instance.Properties.Nice + 1; if (nice <= Monitoring.MaxReniceValue) { instance.Properties.Nice = nice; ProcessPriorityClass priority = nice == 0 ? ProcessPriorityClass.Normal : nice == 1 ? ProcessPriorityClass.BelowNormal : ProcessPriorityClass.Idle; // TODO: instantiate ErrorLog // instance.ErrorLog.Warning(Strings.LoggerLoweringPriority, priority.ToString()); Logger.Info(Strings.LoweringPriorityOnCpuBound, instance.Properties.Name, priority); instance.Prison.jobObject.PriorityClass = priority; } } }
private void StartDropletInstance(DropletInstance instance, string sha1, string executableFile, string executableUri) { try { try { instance.Lock.EnterWriteLock(); var prisonInfo = new ProcessPrisonCreateInfo(); prisonInfo.Id = instance.Properties.InstanceId; prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes; if (this.useDiskQuota) { prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes; prisonInfo.DiskQuotaPath = instance.Properties.Directory; } if (this.uploadThrottleBitsps > 0) { prisonInfo.NetworkOutboundRateLimitBitsPerSecond = this.uploadThrottleBitsps; } prisonInfo.UrlPortAccess = instance.Properties.Port; Logger.Info("Creating Process Prisson: {0}", prisonInfo.Id); instance.Prison.Create(prisonInfo); Logger.Info("Opening firewall port {0} for instance {1}", instance.Properties.Port, instance.Properties.LoggingId); FirewallTools.OpenPort(instance.Properties.Port, instance.Properties.InstanceId); instance.Properties.WindowsPassword = instance.Prison.WindowsPassword; instance.Properties.WindowsUserName = instance.Prison.WindowsUsername; } finally { instance.Lock.ExitWriteLock(); } string tgzFile = Path.Combine(this.fileResources.StagedDir, sha1 + ".tgz"); this.fileResources.PrepareAppDirectory(executableFile, executableUri, sha1, tgzFile, instance); Logger.Debug(Strings.Downloadcompleate); string starting = string.Format(CultureInfo.InvariantCulture, Strings.StartingUpInstanceOnPort, instance.Properties.LoggingId, instance.Properties.Port); Logger.Info(starting); Logger.Debug(Strings.Clients, this.monitoring.Clients); Logger.Debug(Strings.ReservedMemoryUsageMb, this.monitoring.MemoryReservedMbytes, this.monitoring.MaxMemoryMbytes); try { instance.Lock.EnterWriteLock(); instance.Properties.EnvironmentVariables.Add(VcapWindowsUserVariable, instance.Properties.WindowsUserName); instance.Properties.EnvironmentVariables.Add(VcapWindowsUserPasswordVariable, instance.Properties.WindowsPassword); instance.Prison.SetUsersEnvironmentVariables(instance.Properties.EnvironmentVariables); } finally { instance.Lock.ExitWriteLock(); } DateTime start = DateTime.Now; string startSciprtPath = this.CreateStartScript(instance); var runInfo = new ProcessPrisonRunInfo(); runInfo.WorkingDirectory = Path.Combine(instance.Properties.Directory, "app"); runInfo.FileName = startSciprtPath; instance.Prison.RunProcess(runInfo); Logger.Debug(Strings.TookXTimeToLoadConfigureAndStartDebugMessage, (DateTime.Now - start).TotalSeconds); try { instance.Lock.EnterWriteLock(); if (!instance.Properties.StopProcessed) { this.droplets.ScheduleSnapshotAppState(); } } finally { instance.Lock.ExitWriteLock(); } this.DetectAppReady(instance); } catch (Exception ex) { Logger.Warning(Strings.FailedStagingAppDir, instance.Properties.Directory, instance.Properties.LoggingId, ex.ToString()); try { instance.Lock.EnterWriteLock(); instance.Properties.State = DropletInstanceState.Crashed; instance.Properties.ExitReason = DropletExitReason.Crashed; instance.Properties.StateTimestamp = DateTime.Now; this.StopDroplet(instance); } finally { instance.Lock.ExitWriteLock(); } } }
/// <summary> /// Unregisters the instance from the Vcap router. Called when the applicatoin is not in a running state any more. /// </summary> /// <param name="instance">The instance.</param> private void UnregisterInstanceFromRouter(DropletInstance instance) { RouterMessage response = new RouterMessage(); try { instance.Lock.EnterReadLock(); if (instance.Properties.Uris == null || instance.Properties.Uris.Length == 0) { return; } response.DeaId = UUID; response.Host = Host; response.Port = instance.Properties.Port; response.Uris = instance.Properties.Uris; response.Tags = new RouterMessage.TagsObject(); response.Tags.Component = "dea-" + this.Index.ToString(); } finally { instance.Lock.ExitReadLock(); } this.deaReactor.SendRouterUnregister(response.SerializeToJson()); }
/// <summary> /// Detects if an droplet instance is ready, so that it can be set to a Running state and registerd with the router. /// </summary> /// <param name="instance">The instance do be detected.</param> private void DetectAppReady(DropletInstance instance) { ThreadPool.QueueUserWorkItem( delegate { DetectAppReady( instance, delegate(bool detected) { try { instance.Lock.EnterWriteLock(); if (detected) { if (instance.Properties.State == DropletInstanceState.Starting) { Logger.Info(Strings.InstanceIsReadyForConnections, instance.Properties.LoggingId); instance.Properties.State = DropletInstanceState.Running; instance.Properties.StateTimestamp = DateTime.Now; this.deaReactor.SendDeaHeartbeat(instance.GenerateHeartbeat().SerializeToJson()); this.RegisterInstanceWithRouter(instance); this.droplets.ScheduleSnapshotAppState(); } } else { Logger.Warning(Strings.GivingUpOnConnectingApp); this.StopDroplet(instance); } } finally { instance.Lock.ExitWriteLock(); } }); }); }
/// <summary> /// Setups the instance environment variables to be passed when configuring the plugin of an instance. /// </summary> /// <param name="instance">The instance for which to generate the variables.</param> /// <param name="appVars">The user application variables.</param> /// <param name="services">The services to be bound to the instance.</param> /// <returns>The application variables.</returns> private Dictionary<string, string> SetupInstanceEnv(DropletInstance instance, string[] appVars, Dictionary<string, object>[] services) { Dictionary<string, string> env = new Dictionary<string, string>(); env.Add(HomeVariable, instance.Properties.Directory); env.Add(VcapApplicationVariable, this.CreateInstanceVariable(instance)); env.Add(VcapServicesVariable, CreateServicesApplicationVariable(services)); env.Add(VcapAppHostVariable, Host); env.Add(VcapAppPortVariable, instance.Properties.Port.ToString(CultureInfo.InvariantCulture)); env.Add(VcapAppDebugIpVariable, instance.Properties.DebugIP); env.Add(VcapAppDebugPortVariable, instance.Properties.DebugPort != null ? instance.Properties.DebugPort.ToString() : null); if (instance.Properties.DebugPort != null && this.stager.Runtimes[instance.Properties.Runtime].DebugEnvironmentVariables != null) { if (this.stager.Runtimes[instance.Properties.Runtime].DebugEnvironmentVariables.ContainsKey(instance.Properties.DebugMode)) { foreach (KeyValuePair<string, string> debugEnv in this.stager.Runtimes[instance.Properties.Runtime].DebugEnvironmentVariables[instance.Properties.DebugMode]) { env.Add(debugEnv.Key, debugEnv.Value); } } } // Do the runtime environment settings foreach (KeyValuePair<string, string> runtimeEnv in this.stager.Runtimes[instance.Properties.Runtime].Environment) { env.Add(runtimeEnv.Key, runtimeEnv.Value); } // User's environment settings if (appVars != null) { foreach (string appEnv in appVars) { string[] envVar = appEnv.Split(new char[] { '=' }, 2); env.Add(envVar[0], envVar[1]); } } return env; }
private void StartDropletInstance(DropletInstance instance, string sha1, string executableFile, Uri executableUri) { try { string tgzFile = Path.Combine(this.stager.StagedDir, sha1 + ".tgz"); this.stager.StageAppDirectory(executableFile, executableUri, sha1, tgzFile, instance); Logger.Debug(Strings.Downloadcompleate); string starting = string.Format(CultureInfo.InvariantCulture, Strings.StartingUpInstanceOnPort, instance.Properties.LoggingId, instance.Properties.Port); if (!string.IsNullOrEmpty(instance.Properties.DebugMode)) { Logger.Info(starting + Strings.WithDebuggerPort, instance.Properties.DebugPort); } else { Logger.Info(starting); } Logger.Debug(Strings.Clients, this.monitoring.Clients); Logger.Debug(Strings.ReservedMemoryUsageMb, this.monitoring.MemoryReservedMbytes, this.monitoring.MaxMemoryMbytes); List<ApplicationVariable> appVariables = new List<ApplicationVariable>(); try { instance.Lock.EnterWriteLock(); instance.Properties.WindowsPassword = "******" + Credentials.GenerateCredential(); instance.Properties.WindowsUserName = WindowsVCAPUsers.CreateUser(instance.Properties.InstanceId, instance.Properties.WindowsPassword); instance.Properties.EnvironmentVariables.Add(VcapWindowsUserVariable, instance.Properties.WindowsUserName); instance.Properties.EnvironmentVariables.Add(VcapWindowsUserPasswordVariable, instance.Properties.WindowsPassword); instance.Properties.EnvironmentVariables.Add(VcapPluginStagingInfoVariable, File.ReadAllText(Path.Combine(instance.Properties.Directory, "startup"))); foreach (KeyValuePair<string, string> appEnv in instance.Properties.EnvironmentVariables) { ApplicationVariable appVariable = new ApplicationVariable(); appVariable.Name = appEnv.Key; appVariable.Value = appEnv.Value; appVariables.Add(appVariable); } } finally { instance.Lock.ExitWriteLock(); } DateTime start = DateTime.Now; instance.LoadPlugin(); instance.Plugin.ConfigureApplication(appVariables.ToArray()); instance.Plugin.StartApplication(); int pid = instance.Plugin.GetApplicationProcessId(); Logger.Debug(Strings.TookXTimeToLoadConfigureAndStartDebugMessage, (DateTime.Now - start).TotalSeconds); try { instance.Lock.EnterWriteLock(); if (!instance.Properties.StopProcessed) { Logger.Info(Strings.PidAssignedToDroplet, pid, instance.Properties.LoggingId); instance.Properties.ProcessId = pid; this.droplets.ScheduleSnapshotAppState(); } } finally { instance.Lock.ExitWriteLock(); } this.DetectAppReady(instance); } catch (Exception ex) { Logger.Warning(Strings.FailedStagingAppDir, instance.Properties.Directory, instance.Properties.LoggingId, ex.ToString()); try { instance.Lock.EnterWriteLock(); instance.Properties.State = DropletInstanceState.Crashed; instance.Properties.ExitReason = DropletExitReason.Crashed; instance.Properties.StateTimestamp = DateTime.Now; this.StopDroplet(instance); } finally { instance.Lock.ExitWriteLock(); } } }
/// <summary> /// Checks the usage of the instance. If it has a usage above the quota and the DEA is in secure mode, the instance will be stopped. /// </summary> /// <param name="instance">The instance to checks.</param> private void CheckUsage(DropletInstance instance) { DropletInstanceUsage curUsage = instance.Properties.LastUsage; if (instance == null || curUsage == null) { return; } // Check Memory if (curUsage.MemoryKbytes > (instance.Properties.MemoryQuotaBytes / 1024)) { instance.ErrorLog.Fatal(Strings.MemoryLimitOfExceeded, instance.Properties.MemoryQuotaBytes / 1024 / 1024); instance.ErrorLog.Fatal(Strings.ActualUsageWasProcessTerminated, curUsage.MemoryKbytes / 1024); this.StopDroplet(instance); } // Check Disk if (curUsage.DiskBytes > instance.Properties.DiskQuotaBytes) { instance.ErrorLog.Fatal(Strings.DiskUsageLimitOf, instance.Properties.DiskQuotaBytes / 1024 / 1024); instance.ErrorLog.Fatal(Strings.ActualUsageWasProcessTerminated, curUsage.DiskBytes / 1024 / 1024); this.StopDroplet(instance); } // Check CPU if (instance.Usage.Count == 0) { return; } if (curUsage.Cpu > Monitoring.BeginReniceCpuThreshold) { int nice = instance.Properties.Nice + 1; if (nice <= Monitoring.MaxReniceValue) { instance.Properties.Nice = nice; ProcessPriorityClass priority = nice == 0 ? ProcessPriorityClass.Normal : nice == 1 ? ProcessPriorityClass.BelowNormal : ProcessPriorityClass.Idle; instance.ErrorLog.Warning(Strings.LoggerLoweringPriority, priority.ToString()); Logger.Info(Strings.LoweringPriorityOnCpuBound, instance.Properties.Name, priority); // Process.GetProcessById(instance.Properties.ProcessId).PriorityClass = priority; instance.JobObject.PriorityClass = priority; } } // TODO, Check for an attack, or what looks like one, and look at history? // pegged_cpus = @num_cores * 100 // also check for opened handles }
public void RecoverExistingDroplets() { if (!File.Exists(this.droplets.AppStateFile)) { this.droplets.RecoveredDroplets = true; return; } object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.droplets.AppStateFile)); foreach (object obj in instances) { DropletInstance instance = null; try { instance = new DropletInstance(); instance.Properties.FromJsonIntermediateObject(obj); instance.Properties.Orphaned = true; instance.Properties.ResourcesTracked = false; this.monitoring.AddInstanceResources(instance); instance.Properties.StopProcessed = false; instance.JobObject.JobMemoryLimit = instance.Properties.MemoryQuotaBytes; try { instance.LoadPlugin(); instance.Properties.EnvironmentVariables[VcapAppPidVariable] = instance.Properties.ProcessId.ToString(CultureInfo.InvariantCulture); List<ApplicationVariable> appVariables = new List<ApplicationVariable>(); foreach (KeyValuePair<string, string> appEnv in instance.Properties.EnvironmentVariables) { ApplicationVariable appVariable = new ApplicationVariable(); appVariable.Name = appEnv.Key; appVariable.Value = appEnv.Value; appVariables.Add(appVariable); } instance.Plugin.RecoverApplication(appVariables.ToArray()); } catch (Exception ex) { instance.ErrorLog.Error(ex.ToString()); } if (instance.Properties.State == DropletInstanceState.Starting) { this.DetectAppReady(instance); } this.droplets.AddDropletInstance(instance); instance = null; } catch (Exception ex) { Logger.Warning(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.InstanceId, ex.ToString()); } finally { if (instance != null) { instance.Dispose(); } } } this.droplets.RecoveredDroplets = true; if (this.monitoring.Clients > 0) { Logger.Info(Strings.DeaRecoveredApplications, this.monitoring.Clients); } this.MonitorApps(); this.droplets.ForEach(delegate(DropletInstance instance) { this.RegisterInstanceWithRouter(instance); }); this.SendHeartbeat(); this.droplets.ScheduleSnapshotAppState(); }
/// <summary> /// Setups the instance environment variables to be passed when configuring the plugin of an instance. /// </summary> /// <param name="instance">The instance for which to generate the variables.</param> /// <param name="appVars">The user application variables.</param> /// <param name="services">The services to be bound to the instance.</param> /// <returns>The application variables.</returns> private Dictionary<string, string> SetupInstanceEnv(DropletInstance instance, string[] appVars, Dictionary<string, object>[] services) { Dictionary<string, string> env = new Dictionary<string, string>(); env.Add(HomeVariable, Path.Combine(instance.Properties.Directory, "app")); env.Add(VcapApplicationVariable, this.CreateInstanceVariable(instance)); env.Add(VcapServicesVariable, CreateServicesApplicationVariable(services)); env.Add(VcapAppHostVariable, Host); env.Add(VcapAppPortVariable, instance.Properties.Port.ToString(CultureInfo.InvariantCulture)); env.Add("PORT", instance.Properties.Port.ToString(CultureInfo.InvariantCulture)); // User's environment settings if (appVars != null) { foreach (string appEnv in appVars) { string[] envVar = appEnv.Split(new char[] { '=' }, 2); env.Add(envVar[0], envVar[1]); } } return env; }
private void StopDroplet(DropletInstance instance) { try { instance.Lock.EnterWriteLock(); if (instance.Properties.StopProcessed) { return; } // Unplug us from the system immediately, both the routers and health managers. if (!instance.Properties.NotifiedExited) { this.UnregisterInstanceFromRouter(instance); if (instance.Properties.ExitReason == null) { instance.Properties.ExitReason = DropletExitReason.Crashed; instance.Properties.State = DropletInstanceState.Crashed; instance.Properties.StateTimestamp = DateTime.Now; if (!instance.IsProcessIdRunning) { instance.Properties.ProcessId = 0; } } this.deaReactor.SendDropletExited(instance.GenerateDropletExitedMessage().SerializeToJson()); instance.Properties.NotifiedExited = true; } Logger.Info(Strings.StoppingInstance, instance.Properties.LoggingId); // if system thinks this process is running, make sure to execute stop script if (instance.Properties.State == DropletInstanceState.Starting || instance.Properties.State == DropletInstanceState.Running) { instance.Properties.State = DropletInstanceState.Stopped; instance.Properties.StateTimestamp = DateTime.Now; } // this.monitoring.RemoveInstanceResources(instance); instance.Properties.StopProcessed = true; } catch (Exception ex) { Logger.Error(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.DropletId, instance.Properties.InstanceId, ex.ToString()); } finally { instance.Lock.ExitWriteLock(); } }
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } DropletInstance instance = null; instance = new DropletInstance(); string instanceId = Credentials.GenerateSecureGuid().ToString("N"); string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N"); instance.Properties.State = DropletInstanceState.Starting; instance.Properties.StateTimestamp = DateTime.Now; instance.Properties.Start = DateTime.Now; instance.Properties.InstanceId = instanceId; instance.Properties.PrivateInstanceId = privateInstanceId; instance.Properties.DropletId = message.DropletId; instance.Properties.InstanceIndex = message.Index; instance.Properties.Name = message.Name; instance.Properties.Uris = message.Uris; instance.Properties.Users = message.Users; instance.Properties.Version = message.Version; instance.Properties.Stack = message.Stack; instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index); instance.Properties.Flapping = message.Flapping; instance.Properties.CloudControllerPartition = message.CloudControllerPartition; this.AddDropletInstance(instance); return instance; }
public void RecoverExistingDroplets() { if (!File.Exists(this.droplets.AppStateFile)) { this.droplets.RecoveredDroplets = true; return; } object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.droplets.AppStateFile)); foreach (object obj in instances) { DropletInstance instance = null; try { instance = new DropletInstance(); instance.Properties.FromJsonIntermediateObject(obj); instance.Properties.Orphaned = true; instance.Properties.ResourcesTracked = false; this.monitoring.AddInstanceResources(instance); instance.Properties.StopProcessed = false; var prisonInfo = new ProcessPrisonCreateInfo(); prisonInfo.Id = instance.Properties.InstanceId; prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes; prisonInfo.WindowsPassword = instance.Properties.WindowsPassword; if (this.useDiskQuota) { prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes; prisonInfo.DiskQuotaPath = instance.Properties.Directory; } Logger.Info("Recovering Process Prisson: {0}", prisonInfo.Id); instance.Prison.Attach(prisonInfo); if (instance.Properties.State == DropletInstanceState.Starting) { this.DetectAppReady(instance); } this.droplets.AddDropletInstance(instance); instance = null; } catch (Exception ex) { Logger.Warning(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.InstanceId, ex.ToString()); } finally { if (instance != null) { instance.Dispose(); } } } this.droplets.RecoveredDroplets = true; if (this.monitoring.Clients > 0) { Logger.Info(Strings.DeaRecoveredApplications, this.monitoring.Clients); } this.MonitorApps(); this.droplets.ForEach(delegate(DropletInstance instance) { this.RegisterInstanceWithRouter(instance); }); this.SendHeartbeat(); this.droplets.ScheduleSnapshotAppState(); }
/// <summary> /// Detects if an application has the port ready and then invoke the call back. /// </summary> /// <param name="instance">The instance to be checked.</param> /// <param name="callBack">The call back.</param> private static void DetectPortReady(DropletInstance instance, BoolStateBlockCallback callBack) { int attempts = 0; bool keep_going = true; while (attempts <= 1000 && instance.Properties.State == DropletInstanceState.Starting && keep_going == true) { if (instance.IsPortReady(150)) { keep_going = false; callBack(true); } else { Thread.Sleep(100); attempts++; } } if (keep_going) { callBack(false); } }
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } DropletInstance instance = null; instance = new DropletInstance(); string instanceId = Guid.NewGuid().ToString("N"); instance.Properties.State = DropletInstanceState.Starting; instance.Properties.StateTimestamp = DateTime.Now; instance.Properties.Start = DateTime.Now; instance.Properties.InstanceId = instanceId; instance.Properties.DropletId = message.DropletId; instance.Properties.InstanceIndex = message.Index; instance.Properties.Name = message.Name; instance.Properties.Uris = message.Uris; instance.Properties.Users = message.Users; instance.Properties.Version = message.Version; instance.Properties.Framework = message.Framework; instance.Properties.Runtime = message.Runtime; instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index); this.AddDropletInstance(instance); return instance; }
/// <summary> /// Stages the app directory. /// </summary> /// <param name="bitsFile">The bits file.</param> /// <param name="bitsUri">The bits URI.</param> /// <param name="hash">The sha1.</param> /// <param name="tarZipFile">The TGZ file.</param> /// <param name="instance">The instance.</param> public void StageAppDirectory(string bitsFile, Uri bitsUri, string hash, string tarZipFile, DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } // What we do here, in order of preference.. // 1. Check our own staged directory. // 2. Check shared directory from CloudController that could be mounted (bits_file) // 3. Pull from http if needed. string instanceDir = instance.Properties.Directory; lock (this.stagerLock) { // check before downloading if (instance.Properties.StopProcessed) { return; } if (File.Exists(tarZipFile)) { Logger.Debug(Strings.FoundStagedBitsInLocalCache); } else { // If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version. DateTime start = DateTime.Now; if (!this.ForceHttpFileSharing && File.Exists(bitsFile)) { Logger.Debug(Strings.SharingCloudControllerStagingDirectory); File.Copy(bitsFile, tarZipFile); Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start); } else { Logger.Debug(Strings.Needtodownloadappbitsfrom, bitsUri); this.DownloadAppBits(bitsUri, hash, tarZipFile); Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start); } } // check before extracting if (instance.Properties.StopProcessed) { return; } DateTime startStageing = DateTime.Now; // Explode the app into its directory and optionally bind its local runtime. Directory.CreateDirectory(instanceDir); string tarFileName = Path.GetFileName(tarZipFile); tarFileName = Path.ChangeExtension(tarFileName, ".tar"); DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar File.Delete(Path.Combine(instanceDir, tarFileName)); Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing); } }