/// <summary> /// Removes a droplet instance from the collection. /// </summary> /// <param name="instance">The droplet instance to remove.</param> public void RemoveDropletInstance(DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } try { this.Lock.EnterWriteLock(); if (this.Droplets.ContainsKey(instance.Properties.DropletId)) { Droplet droplet = this.Droplets[instance.Properties.DropletId]; if (droplet.DropletInstances.ContainsKey(instance.Properties.InstanceId)) { droplet.DropletInstances.Remove(instance.Properties.InstanceId); if (droplet.DropletInstances.Count == 0) { this.Droplets.Remove(instance.Properties.DropletId); } } } } finally { this.Lock.ExitWriteLock(); } this.ScheduleSnapshotAppState(); }
/// <summary> /// Adds a droplet instance to the collection. /// </summary> /// <param name="instance">The droplet instance to add.</param> public void AddDropletInstance(DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } try { this.Lock.EnterWriteLock(); instance.Lock.EnterReadLock(); if (!this.Droplets.ContainsKey(instance.Properties.DropletId)) { this.Droplets.Add(instance.Properties.DropletId, new Droplet()); } this.Droplets[instance.Properties.DropletId].DropletInstances[instance.Properties.InstanceId] = instance; } finally { instance.Lock.ExitReadLock(); this.Lock.ExitWriteLock(); } this.ScheduleSnapshotAppState(); }
/// <summary> /// Untracks the memory used by the instance and flags it/ /// </summary> /// <param name="instance">The instance to be untracked.</param> public void RemoveInstanceResources(DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } try { this.Lock.EnterWriteLock(); instance.Lock.EnterWriteLock(); if (instance.Properties.ResourcesTracked) { instance.Properties.ResourcesTracked = false; this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024; this.Clients--; } } finally { instance.Lock.ExitWriteLock(); this.Lock.ExitWriteLock(); } }
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } DropletInstance instance = null; instance = new DropletInstance(); string instanceId = Credentials.GenerateSecureGuid().ToString("N"); string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N"); instance.Properties.State = DropletInstanceState.Starting; instance.Properties.StateTimestamp = DateTime.Now; instance.Properties.Start = DateTime.Now; instance.Properties.InstanceId = instanceId; instance.Properties.PrivateInstanceId = privateInstanceId; instance.Properties.DropletId = message.DropletId; instance.Properties.InstanceIndex = message.Index; instance.Properties.Name = message.Name; instance.Properties.Uris = message.Uris; instance.Properties.Users = message.Users; instance.Properties.Version = message.Version; instance.Properties.Stack = message.Stack; instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index); instance.Properties.Flapping = message.Flapping; instance.Properties.CloudControllerPartition = message.CloudControllerPartition; this.AddDropletInstance(instance); return(instance); }
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } DropletInstance instance = null; instance = new DropletInstance(); string instanceId = Credentials.GenerateSecureGuid().ToString("N"); string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N"); instance.Properties.State = DropletInstanceState.Starting; instance.Properties.StateTimestamp = DateTime.Now; instance.Properties.Start = DateTime.Now; instance.Properties.InstanceId = instanceId; instance.Properties.PrivateInstanceId = privateInstanceId; instance.Properties.DropletId = message.DropletId; instance.Properties.InstanceIndex = message.Index; instance.Properties.Name = message.Name; instance.Properties.Uris = message.Uris; instance.Properties.Users = message.Users; instance.Properties.Version = message.Version; instance.Properties.Stack = message.Stack; instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index); instance.Properties.Flapping = message.Flapping; instance.Properties.CloudControllerPartition = message.CloudControllerPartition; this.AddDropletInstance(instance); return instance; }
/// <summary> /// Detects if an droplet instance is ready, so that it can be set to a Running state and registerd with the router. /// </summary> /// <param name="instance">The instance do be detected.</param> private void DetectAppReady(DropletInstance instance) { ThreadPool.QueueUserWorkItem( delegate { DetectAppReady( instance, delegate(bool detected) { try { instance.Lock.EnterWriteLock(); if (detected) { if (instance.Properties.State == DropletInstanceState.Starting) { Logger.Info(Strings.InstanceIsReadyForConnections, instance.Properties.LoggingId); instance.Properties.State = DropletInstanceState.Running; instance.Properties.StateTimestamp = DateTime.Now; this.SendHeartbeat(); this.RegisterInstanceWithRouter(instance); this.droplets.ScheduleSnapshotAppState(); } } else { Logger.Warning(Strings.GivingUpOnConnectingApp); this.StopDroplet(instance); } } finally { instance.Lock.ExitWriteLock(); } }); }); }
/// <summary> /// Prepares the app directory. /// </summary> /// <param name="bitsFile">The bits file.</param> /// <param name="bitsUri">The bits URI.</param> /// <param name="hash">The sha1.</param> /// <param name="tarZipFile">The TGZ file.</param> /// <param name="instance">The instance.</param> public void PrepareAppDirectory(string bitsFile, string bitsUri, string hash, string tarZipFile, DropletInstance instance) { if (instance == null) { throw new ArgumentNullException("instance"); } // What we do here, in order of preference.. // 1. Check our own staged directory. // 2. Check shared directory from CloudController that could be mounted (bits_file) // 3. Pull from http if needed. string instanceDir = instance.Properties.Directory; lock (this.stagerLock) { // check before downloading if (instance.Properties.StopProcessed) { return; } if (File.Exists(tarZipFile)) { Logger.Debug(Strings.FoundStagedBitsInLocalCache); } else { // If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version. DateTime start = DateTime.Now; if (File.Exists(bitsFile)) { Logger.Debug(Strings.SharingCloudControllerStagingDirectory); File.Copy(bitsFile, tarZipFile); Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start); } else { Uri downloadUri = new Uri(bitsUri); Logger.Debug(Strings.Needtodownloadappbitsfrom, downloadUri); this.DownloadAppBits(downloadUri, hash, tarZipFile); Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start); } } // create log files string logDir = Path.Combine(instance.Properties.Directory, "logs"); Directory.CreateDirectory(logDir); using (File.Create(Path.Combine(logDir, "stdout.log"))) { } using (File.Create(Path.Combine(logDir, "stderr.log"))) { } // check before extracting if (instance.Properties.StopProcessed) { return; } DateTime startStageing = DateTime.Now; // Explode the app into its directory and optionally bind its local runtime. Directory.CreateDirectory(instance.Properties.Directory); DirectoryInfo deploymentDirInfo = new DirectoryInfo(instance.Properties.Directory); DirectorySecurity deploymentDirSecurity = deploymentDirInfo.GetAccessControl(); // Owner is important to account for disk quota deploymentDirSecurity.SetOwner(new NTAccount(instance.Properties.WindowsUserName)); deploymentDirSecurity.SetAccessRule( new FileSystemAccessRule( instance.Properties.WindowsUserName, FileSystemRights.Write | FileSystemRights.Read | FileSystemRights.Delete | FileSystemRights.Modify | FileSystemRights.FullControl, InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, PropagationFlags.None | PropagationFlags.InheritOnly, AccessControlType.Allow)); // Taking ownership of a file has to be executed with restore privilege elevated privilages using (new ProcessPrivileges.PrivilegeEnabler(Process.GetCurrentProcess(), ProcessPrivileges.Privilege.Restore)) { deploymentDirInfo.SetAccessControl(deploymentDirSecurity); } // Impersonate user to cascade the owernship to every file // Neccessary for windows disk quota using (new UserImpersonator(instance.Properties.WindowsUserName, ".", instance.Properties.WindowsPassword, true)) { DEAUtilities.ExtractArchive(tarZipFile, instanceDir); } Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing); } }
/// <summary> /// Detects if an application has the port ready and then invoke the call back. /// </summary> /// <param name="instance">The instance to be checked.</param> /// <param name="callBack">The call back.</param> private static void DetectPortReady(DropletInstance instance, BoolStateBlockCallback callBack) { int attempts = 0; bool keep_going = true; while (attempts <= 1000 && instance.Properties.State == DropletInstanceState.Starting && keep_going == true) { if (instance.IsPortReady(150)) { keep_going = false; callBack(true); } else { Thread.Sleep(100); attempts++; } } if (keep_going) { callBack(false); } }
/// <summary> /// Detects the if an app is ready and run the callback. /// </summary> /// <param name="instance">The instance to be checked.</param> /// <param name="callBack">The call back.</param> private static void DetectAppReady(DropletInstance instance, BoolStateBlockCallback callBack) { DetectPortReady(instance, callBack); }
public void RecoverExistingDroplets() { if (!File.Exists(this.droplets.AppStateFile)) { this.droplets.RecoveredDroplets = true; return; } object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.droplets.AppStateFile)); foreach (object obj in instances) { DropletInstance instance = null; try { instance = new DropletInstance(); instance.Properties.FromJsonIntermediateObject(obj); instance.Properties.Orphaned = true; instance.Properties.ResourcesTracked = false; this.monitoring.AddInstanceResources(instance); instance.Properties.StopProcessed = false; Logger.Info("Recovering Instance: {0}", instance.Properties.ContainerId); CloudFoundry.WindowsPrison.PrisonManager.LoadPrisonAndAttach(Guid.Parse(instance.Properties.ContainerId)); if (instance.Properties.State == DropletInstanceState.Starting) { this.DetectAppReady(instance); } this.droplets.AddDropletInstance(instance); instance = null; } catch (Exception ex) { Logger.Warning(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.InstanceId, ex.ToString()); } finally { if (instance != null) { instance.Dispose(); } } } this.droplets.RecoveredDroplets = true; if (this.monitoring.Clients > 0) { Logger.Info(Strings.DeaRecoveredApplications, this.monitoring.Clients); } this.MonitorApps(); this.droplets.ForEach(delegate(DropletInstance instance) { this.RegisterInstanceWithRouter(instance); }); this.SendHeartbeat(); this.droplets.ScheduleSnapshotAppState(); }
/// <summary> /// Unregisters the instance from the Vcap router. Called when the applicatoin is not in a running state any more. /// </summary> /// <param name="instance">The instance.</param> private void UnregisterInstanceFromRouter(DropletInstance instance) { RouterMessage response = new RouterMessage(); try { instance.Lock.EnterReadLock(); if (instance.Properties.Uris == null || instance.Properties.Uris.Length == 0) { return; } response.DeaId = UUID; response.Host = Host; response.Port = instance.Properties.Port; response.Uris = instance.Properties.Uris; response.Tags = new RouterMessage.TagsObject(); response.Tags.Component = "dea-" + this.Index.ToString(); } finally { instance.Lock.ExitReadLock(); } this.deaReactor.SendRouterUnregister(response.SerializeToJson()); }
private void StopDroplet(DropletInstance instance) { try { instance.Lock.EnterWriteLock(); if (instance.Properties.StopProcessed) { return; } // Unplug us from the system immediately, both the routers and health managers. if (!instance.Properties.NotifiedExited) { this.UnregisterInstanceFromRouter(instance); if (instance.Properties.ExitReason == null) { instance.Properties.ExitReason = DropletExitReason.Crashed; instance.Properties.State = DropletInstanceState.Crashed; instance.Properties.StateTimestamp = DateTime.Now; if (!instance.IsProcessIdRunning) { instance.Properties.ProcessId = 0; } } this.deaReactor.SendDropletExited(instance.GenerateDropletExitedMessage().SerializeToJson()); instance.Properties.NotifiedExited = true; } Logger.Info(Strings.StoppingInstance, instance.Properties.LoggingId); // if system thinks this process is running, make sure to execute stop script if (instance.Properties.State == DropletInstanceState.Starting || instance.Properties.State == DropletInstanceState.Running) { instance.Properties.State = DropletInstanceState.Stopped; instance.Properties.StateTimestamp = DateTime.Now; } // this.monitoring.RemoveInstanceResources(instance); instance.Properties.StopProcessed = true; } catch (Exception ex) { Logger.Error(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.DropletId, instance.Properties.InstanceId, ex.ToString()); } finally { instance.Lock.ExitWriteLock(); } }
private void StartDropletInstance(DropletInstance instance, string sha1, string executableFile, string executableUri) { try { try { instance.Lock.EnterWriteLock(); var containerRules = new CloudFoundry.WindowsPrison.PrisonConfiguration(); containerRules.PrisonHomeRootPath = instance.Properties.Directory; containerRules.Rules |= CloudFoundry.WindowsPrison.RuleTypes.WindowStation; containerRules.Rules |= CloudFoundry.WindowsPrison.RuleTypes.IISGroup; containerRules.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes; containerRules.PriorityClass = ProcessPriorityClass.BelowNormal; containerRules.ActiveProcessesLimit = 10; if (this.uploadThrottleBitsps > 0) { containerRules.Rules |= CloudFoundry.WindowsPrison.RuleTypes.Network; containerRules.NetworkOutboundRateLimitBitsPerSecond = this.uploadThrottleBitsps; containerRules.AppPortOutboundRateLimitBitsPerSecond = this.uploadThrottleBitsps; } containerRules.Rules |= CloudFoundry.WindowsPrison.RuleTypes.Httpsys; containerRules.UrlPortAccess = instance.Properties.Port; if (this.useDiskQuota) { containerRules.Rules |= CloudFoundry.WindowsPrison.RuleTypes.Disk; containerRules.DiskQuotaBytes = instance.Properties.DiskQuotaBytes; } //var prisonInfo = new ProcessPrisonCreateInfo(); //prisonInfo.Id = instance.Properties.InstanceId; //prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes; //if (this.useDiskQuota) //{ // prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes; // prisonInfo.DiskQuotaPath = instance.Properties.Directory; //} //if (this.uploadThrottleBitsps > 0) //{ // prisonInfo.NetworkOutboundRateLimitBitsPerSecond = this.uploadThrottleBitsps; //} //prisonInfo.UrlPortAccess = instance.Properties.Port; instance.Prison.Tag = "dea"; instance.Properties.ContainerId = instance.Prison.Id.ToString(); Logger.Info("Creating Process Prisson: {0}", instance.Properties.ContainerId); instance.Prison.Lockdown(containerRules); //instance.Prison.Create(prisonInfo); Logger.Info("Opening firewall port {0} for instance {1}", instance.Properties.Port, instance.Properties.LoggingId); FirewallTools.OpenPort(instance.Properties.Port, instance.Properties.InstanceId); instance.Properties.WindowsUserName = instance.Prison.User.UserName; instance.Properties.WindowsPassword = instance.Prison.User.Password; //instance.Properties.WindowsPassword = instance.Prison.WindowsPassword; //instance.Properties.WindowsUserName = instance.Prison.WindowsUsername; } finally { instance.Lock.ExitWriteLock(); } string tgzFile = Path.Combine(this.fileResources.StagedDir, sha1 + ".tgz"); this.fileResources.PrepareAppDirectory(executableFile, executableUri, sha1, tgzFile, instance); Logger.Debug(Strings.Downloadcompleate); string starting = string.Format(CultureInfo.InvariantCulture, Strings.StartingUpInstanceOnPort, instance.Properties.LoggingId, instance.Properties.Port); Logger.Info(starting); Logger.Debug(Strings.Clients, this.monitoring.Clients); Logger.Debug(Strings.ReservedMemoryUsageMb, this.monitoring.MemoryReservedMbytes, this.monitoring.MaxMemoryMbytes); try { instance.Lock.EnterWriteLock(); instance.Properties.EnvironmentVariables.Add(VcapWindowsUserVariable, instance.Properties.WindowsUserName); instance.Properties.EnvironmentVariables.Add(VcapWindowsUserPasswordVariable, instance.Properties.WindowsPassword); instance.Prison.User.SetUserEnvironmentVariables(instance.Properties.EnvironmentVariables); } finally { instance.Lock.ExitWriteLock(); } DateTime start = DateTime.Now; string startSciprtPath = this.CreateStartScript(instance); instance.Prison.Execute(null, startSciprtPath, Path.Combine(instance.Properties.Directory, "app"), false, null, null, null, null); Logger.Debug(Strings.TookXTimeToLoadConfigureAndStartDebugMessage, (DateTime.Now - start).TotalSeconds); try { instance.Lock.EnterWriteLock(); if (!instance.Properties.StopProcessed) { this.droplets.ScheduleSnapshotAppState(); } } finally { instance.Lock.ExitWriteLock(); } if (File.Exists(this.logyardUidPath)) { LogyardInstanceRequest logyardMsg = new LogyardInstanceRequest(); logyardMsg.AppGUID = instance.Properties.DropletId; logyardMsg.AppName = instance.Properties.Name; logyardMsg.AppSpace = ""; logyardMsg.DockerId = instance.Properties.InstanceId; logyardMsg.Index = -1; Dictionary<string, string> logfiles = new Dictionary<string, string>(); logfiles["stdout"] = @"logs\stdout.log"; logfiles["stderr"] = @"logs\stderr.log"; logyardMsg.LogFiles = logfiles; logyardMsg.Type = "app"; logyardMsg.RootPath = instance.Properties.Directory; string logyardId = File.ReadAllText(this.logyardUidPath).Trim(); this.deaReactor.SendLogyardNotification(logyardId, logyardMsg.SerializeToJson()); } this.DetectAppReady(instance); } catch (Exception ex) { Logger.Warning(Strings.FailedStagingAppDir, instance.Properties.Directory, instance.Properties.LoggingId, ex.ToString()); try { instance.Lock.EnterWriteLock(); instance.Properties.State = DropletInstanceState.Crashed; instance.Properties.ExitReason = DropletExitReason.Crashed; instance.Properties.StateTimestamp = DateTime.Now; this.StopDroplet(instance); } finally { instance.Lock.ExitWriteLock(); } } }
/// <summary> /// Setups the instance environment variables to be passed when configuring the plugin of an instance. /// </summary> /// <param name="instance">The instance for which to generate the variables.</param> /// <param name="appVars">The user application variables.</param> /// <param name="services">The services to be bound to the instance.</param> /// <returns>The application variables.</returns> private Dictionary<string, string> SetupInstanceEnv(DropletInstance instance, string[] appVars, Dictionary<string, object>[] services) { Dictionary<string, string> env = new Dictionary<string, string>(); env.Add(HomeVariable, Path.Combine(instance.Properties.Directory, "app")); env.Add(VcapApplicationVariable, this.CreateInstanceVariable(instance)); env.Add(VcapServicesVariable, CreateServicesApplicationVariable(services)); env.Add(VcapAppHostVariable, Host); env.Add(VcapAppPortVariable, instance.Properties.Port.ToString(CultureInfo.InvariantCulture)); env.Add("PORT", instance.Properties.Port.ToString(CultureInfo.InvariantCulture)); env.Add("HOMEPATH", Path.Combine(instance.Properties.Directory)); // User's environment settings if (appVars != null) { var parsedAppVards = ParseEnvironmnetVariables(appVars); foreach (var appEnv in parsedAppVards) { env.Add(appEnv.Key, appEnv.Value); } } return env; }
/// <summary> /// Creates the application variable for an instance. Is is used for the plugin configuration. /// </summary> /// <param name="instance">The instance for which the application variable is to be generated.</param> /// <returns>The application variable.</returns> private string CreateInstanceVariable(DropletInstance instance) { List<string> whitelist = new List<string>() { "instance_id", "instance_index", "name", "uris", "users", "version", "start", "runtime", "state_timestamp", "port" }; Dictionary<string, object> result = new Dictionary<string, object>(); Dictionary<string, object> jsonInstance = instance.Properties.ToJsonIntermediateObject(); foreach (string key in whitelist) { if (jsonInstance.ContainsKey(key)) { // result[key] = JsonConvertibleObject.ObjectToValue<object>(jInstance[key]); result[key] = jsonInstance[key]; } } result["host"] = Host; result["limits"] = new Dictionary<string, object>() { { "fds", instance.Properties.FDSQuota }, { "mem", instance.Properties.MemoryQuotaBytes }, { "disk", instance.Properties.DiskQuotaBytes } }; return JsonConvertibleObject.SerializeToJson(result); }
private string CreateStartScript(DropletInstance instance) { string startCommand = StagingInfo.getStartCommand(Path.Combine(instance.Properties.Directory, "staging_info.yml")); var startScriptTemplate = @" set > {0}\logs\env.log cd {0}\app {1} > {0}\logs\stdout.log 2> {0}\logs\stderr.log "; string startScript = String.Format(startScriptTemplate, instance.Properties.Directory, startCommand); string scriptPath = Path.Combine(instance.Properties.Directory, "start.cmd"); File.WriteAllText(scriptPath, startScript, Encoding.ASCII); return scriptPath; }