/// <summary>
/// Adds a droplet instance to the collection.
/// </summary>
/// <param name="instance">The droplet instance to add.</param>
public void AddDropletInstance(DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterReadLock();
if (!this.Droplets.ContainsKey(instance.Properties.DropletId))
{
this.Droplets.Add(instance.Properties.DropletId, new Droplet());
}
this.Droplets[instance.Properties.DropletId].DropletInstances[instance.Properties.InstanceId] = instance;
}
finally
{
instance.Lock.ExitReadLock();
this.Lock.ExitWriteLock();
}
this.ScheduleSnapshotAppState();
}
/// <summary>
/// Removes a droplet instance from the collection.
/// </summary>
/// <param name="instance">The droplet instance to remove.</param>
public void RemoveDropletInstance(DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
if (this.Droplets.ContainsKey(instance.Properties.DropletId))
{
Droplet droplet = this.Droplets[instance.Properties.DropletId];
if (droplet.DropletInstances.ContainsKey(instance.Properties.InstanceId))
{
droplet.DropletInstances.Remove(instance.Properties.InstanceId);
if (droplet.DropletInstances.Count == 0)
{
this.Droplets.Remove(instance.Properties.DropletId);
}
}
}
}
finally
{
this.Lock.ExitWriteLock();
}
this.ScheduleSnapshotAppState();
}
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
DropletInstance instance = null;
instance = new DropletInstance();
string instanceId = Guid.NewGuid().ToString("N");
instance.Properties.State = DropletInstanceState.Starting;
instance.Properties.StateTimestamp = DateTime.Now;
instance.Properties.Start = DateTime.Now;
instance.Properties.InstanceId = instanceId;
instance.Properties.DropletId = message.DropletId;
instance.Properties.InstanceIndex = message.Index;
instance.Properties.Name = message.Name;
instance.Properties.Uris = message.Uris;
instance.Properties.Users = message.Users;
instance.Properties.Version = message.Version;
instance.Properties.Framework = message.Framework;
instance.Properties.Runtime = message.Runtime;
instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index);
this.AddDropletInstance(instance);
return(instance);
}
/// <summary>
/// Untracks the memory used by the instance and flags it/
/// </summary>
/// <param name="instance">The instance to be untracked.</param>
public void RemoveInstanceResources(DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterWriteLock();
if (instance.Properties.ResourcesTracked)
{
instance.Properties.ResourcesTracked = false;
this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024;
this.Clients--;
}
}
finally
{
instance.Lock.ExitWriteLock();
this.Lock.ExitWriteLock();
}
}
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
DropletInstance instance = null;
instance = new DropletInstance();
string instanceId = Credentials.GenerateSecureGuid().ToString("N");
string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");
instance.Properties.State = DropletInstanceState.Starting;
instance.Properties.StateTimestamp = DateTime.Now;
instance.Properties.Start = DateTime.Now;
instance.Properties.InstanceId = instanceId;
instance.Properties.PrivateInstanceId = privateInstanceId;
instance.Properties.DropletId = message.DropletId;
instance.Properties.InstanceIndex = message.Index;
instance.Properties.Name = message.Name;
instance.Properties.Uris = message.Uris;
instance.Properties.Users = message.Users;
instance.Properties.Version = message.Version;
instance.Properties.Stack = message.Stack;
instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index);
instance.Properties.Flapping = message.Flapping;
instance.Properties.CloudControllerPartition = message.CloudControllerPartition;
this.AddDropletInstance(instance);
return(instance);
}
/// <summary>
/// Creates the application variable for an instance. Is is used for the plugin configuration.
/// </summary>
/// <param name="instance">The instance for which the application variable is to be generated.</param>
/// <returns>The application variable.</returns>
private string CreateInstanceVariable(DropletInstance instance)
{
List<string> whitelist = new List<string>() { "instance_id", "instance_index", "name", "uris", "users", "version", "start", "runtime", "state_timestamp", "port" };
Dictionary<string, object> result = new Dictionary<string, object>();
Dictionary<string, object> jsonInstance = instance.Properties.ToJsonIntermediateObject();
foreach (string key in whitelist)
{
if (jsonInstance.ContainsKey(key))
{
// result[key] = JsonConvertibleObject.ObjectToValue<object>(jInstance[key]);
result[key] = jsonInstance[key];
}
}
result["host"] = Host;
result["limits"] = new Dictionary<string, object>()
{
{ "fds", instance.Properties.FDSQuota },
{ "mem", instance.Properties.MemoryQuotaBytes },
{ "disk", instance.Properties.DiskQuotaBytes }
};
return JsonConvertibleObject.SerializeToJson(result);
}
private string CreateStartScript(DropletInstance instance)
{
string startCommand = StagingInfo.getStartCommand(Path.Combine(instance.Properties.Directory, "staging_info.yml"));
var startScriptTemplate =
@"
set > {0}\logs\env.log
cd {0}\app
{1} > {0}\logs\stdout.log 2> {0}\logs\stderr.log
";
string startScript = String.Format(startScriptTemplate, instance.Properties.Directory, startCommand);
string scriptPath = Path.Combine(instance.Properties.Directory, "start.cmd");
File.WriteAllText(scriptPath, startScript, Encoding.ASCII);
return scriptPath;
}
/// <summary>
/// Prepares the app directory.
/// </summary>
/// <param name="bitsFile">The bits file.</param>
/// <param name="bitsUri">The bits URI.</param>
/// <param name="hash">The sha1.</param>
/// <param name="tarZipFile">The TGZ file.</param>
/// <param name="instance">The instance.</param>
public void PrepareAppDirectory(string bitsFile, string bitsUri, string hash, string tarZipFile, DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
// What we do here, in order of preference..
// 1. Check our own staged directory.
// 2. Check shared directory from CloudController that could be mounted (bits_file)
// 3. Pull from http if needed.
string instanceDir = instance.Properties.Directory;
lock (this.stagerLock)
{
// check before downloading
if (instance.Properties.StopProcessed)
{
return;
}
if (File.Exists(tarZipFile))
{
Logger.Debug(Strings.FoundStagedBitsInLocalCache);
}
else
{
// If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version.
DateTime start = DateTime.Now;
if (File.Exists(bitsFile))
{
Logger.Debug(Strings.SharingCloudControllerStagingDirectory);
File.Copy(bitsFile, tarZipFile);
Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start);
}
else
{
Uri downloadUri = new Uri(bitsUri);
Logger.Debug(Strings.Needtodownloadappbitsfrom, downloadUri);
this.DownloadAppBits(downloadUri, hash, tarZipFile);
Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start);
}
}
// check before extracting
if (instance.Properties.StopProcessed)
{
return;
}
DateTime startStageing = DateTime.Now;
// Explode the app into its directory and optionally bind its local runtime.
Directory.CreateDirectory(instance.Properties.Directory);
DirectoryInfo deploymentDirInfo = new DirectoryInfo(instance.Properties.Directory);
DirectorySecurity deploymentDirSecurity = deploymentDirInfo.GetAccessControl();
// Owner is important to account for disk quota
deploymentDirSecurity.SetOwner(new NTAccount(instance.Properties.WindowsUserName));
deploymentDirSecurity.SetAccessRule(
new FileSystemAccessRule(
instance.Properties.WindowsUserName,
FileSystemRights.Write | FileSystemRights.Read | FileSystemRights.Delete | FileSystemRights.Modify | FileSystemRights.FullControl,
InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit,
PropagationFlags.None | PropagationFlags.InheritOnly,
AccessControlType.Allow));
// Taking ownership of a file has to be executed with restore privilege elevated privilages
using (new ProcessPrivileges.PrivilegeEnabler(Process.GetCurrentProcess(), ProcessPrivileges.Privilege.Restore))
{
deploymentDirInfo.SetAccessControl(deploymentDirSecurity);
}
// Impersonate user to cascade the owernship to every file
// Neccessary for windows disk quota
using (new UserImpersonator(instance.Properties.WindowsUserName, ".", instance.Properties.WindowsPassword, true))
{
DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip
string tarFileName = Directory.GetFiles(instanceDir, "*.tar")[0];
DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar
File.Delete(Path.Combine(instanceDir, tarFileName));
}
Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing);
}
}
/// <summary>
/// Registers the instance with the Vcap router. Called when the application is running and ready.
/// </summary>
/// <param name="instance">The instance to be registered.</param>
private void RegisterInstanceWithRouter(DropletInstance instance)
{
RouterMessage response = new RouterMessage();
try
{
instance.Lock.EnterReadLock();
if (instance.Properties.Uris == null || instance.Properties.Uris.Length == 0)
{
return;
}
response.DeaId = UUID;
response.Host = Host;
response.Port = instance.Properties.Port;
response.Uris = new List<string>(instance.Properties.Uris).ToArray();
response.Tags = new RouterMessage.TagsObject();
response.Tags.Framework = instance.Properties.Framework;
response.Tags.Runtime = instance.Properties.Runtime;
}
finally
{
instance.Lock.ExitReadLock();
}
this.deaReactor.SendRouterRegister(response.SerializeToJson());
}
/// <summary>
/// Detects the if an app is ready and run the callback.
/// </summary>
/// <param name="instance">The instance to be checked.</param>
/// <param name="callBack">The call back.</param>
private static void DetectAppReady(DropletInstance instance, BoolStateBlockCallback callBack)
{
DetectPortReady(instance, callBack);
}
private void CheckUsage(DropletInstance instance)
{
DropletInstanceUsage curUsage = instance.Properties.LastUsage;
if (instance == null || curUsage == null)
{
return;
}
//// Check Memory
//// Memory usage also enforced by windows job object
//if (curUsage.MemoryBytes > (instance.Properties.MemoryQuotaBytes))
//{
// instance.ErrorLog.Fatal(
// "Memory size usage exceeded the limit of {0} MiB. Memory size used: {1} MiB. Stopping the app instance.",
// instance.Properties.MemoryQuotaBytes / 1024 / 1024,
// curUsage.MemoryBytes / 1024 / 1024);
// this.StopDroplet(instance);
//}
//// Check Disk
//// Disk usage also enforced by windows disk quota
//if (curUsage.DiskBytes > instance.Properties.DiskQuotaBytes * 1.05)
//{
// instance.ErrorLog.Fatal(
// "Disk size usage exceeded the limit of {0} MiB. Disk size used: {1} MiB. Stopping the app instance.",
// instance.Properties.DiskQuotaBytes / 1024 / 1024,
// curUsage.DiskBytes / 1024 / 1024);
// this.StopDroplet(instance);
//}
// Check CPU
if (instance.Usage.Count == 0)
{
return;
}
if (curUsage.Cpu > Monitoring.BeginReniceCpuThreshold)
{
int nice = instance.Properties.Nice + 1;
if (nice <= Monitoring.MaxReniceValue)
{
instance.Properties.Nice = nice;
ProcessPriorityClass priority =
nice == 0 ? ProcessPriorityClass.Normal :
nice == 1 ? ProcessPriorityClass.BelowNormal :
ProcessPriorityClass.Idle;
// TODO: instantiate ErrorLog
// instance.ErrorLog.Warning(Strings.LoggerLoweringPriority, priority.ToString());
Logger.Info(Strings.LoweringPriorityOnCpuBound, instance.Properties.Name, priority);
instance.Prison.jobObject.PriorityClass = priority;
}
}
}
private void StartDropletInstance(DropletInstance instance, string sha1, string executableFile, string executableUri)
{
try
{
try
{
instance.Lock.EnterWriteLock();
var prisonInfo = new ProcessPrisonCreateInfo();
prisonInfo.Id = instance.Properties.InstanceId;
prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes;
if (this.useDiskQuota)
{
prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes;
prisonInfo.DiskQuotaPath = instance.Properties.Directory;
}
if (this.uploadThrottleBitsps > 0)
{
prisonInfo.NetworkOutboundRateLimitBitsPerSecond = this.uploadThrottleBitsps;
}
prisonInfo.UrlPortAccess = instance.Properties.Port;
Logger.Info("Creating Process Prisson: {0}", prisonInfo.Id);
instance.Prison.Create(prisonInfo);
Logger.Info("Opening firewall port {0} for instance {1}", instance.Properties.Port, instance.Properties.LoggingId);
FirewallTools.OpenPort(instance.Properties.Port, instance.Properties.InstanceId);
instance.Properties.WindowsPassword = instance.Prison.WindowsPassword;
instance.Properties.WindowsUserName = instance.Prison.WindowsUsername;
}
finally
{
instance.Lock.ExitWriteLock();
}
string tgzFile = Path.Combine(this.fileResources.StagedDir, sha1 + ".tgz");
this.fileResources.PrepareAppDirectory(executableFile, executableUri, sha1, tgzFile, instance);
Logger.Debug(Strings.Downloadcompleate);
string starting = string.Format(CultureInfo.InvariantCulture, Strings.StartingUpInstanceOnPort, instance.Properties.LoggingId, instance.Properties.Port);
Logger.Info(starting);
Logger.Debug(Strings.Clients, this.monitoring.Clients);
Logger.Debug(Strings.ReservedMemoryUsageMb, this.monitoring.MemoryReservedMbytes, this.monitoring.MaxMemoryMbytes);
try
{
instance.Lock.EnterWriteLock();
instance.Properties.EnvironmentVariables.Add(VcapWindowsUserVariable, instance.Properties.WindowsUserName);
instance.Properties.EnvironmentVariables.Add(VcapWindowsUserPasswordVariable, instance.Properties.WindowsPassword);
instance.Prison.SetUsersEnvironmentVariables(instance.Properties.EnvironmentVariables);
}
finally
{
instance.Lock.ExitWriteLock();
}
DateTime start = DateTime.Now;
string startSciprtPath = this.CreateStartScript(instance);
var runInfo = new ProcessPrisonRunInfo();
runInfo.WorkingDirectory = Path.Combine(instance.Properties.Directory, "app");
runInfo.FileName = startSciprtPath;
instance.Prison.RunProcess(runInfo);
Logger.Debug(Strings.TookXTimeToLoadConfigureAndStartDebugMessage, (DateTime.Now - start).TotalSeconds);
try
{
instance.Lock.EnterWriteLock();
if (!instance.Properties.StopProcessed)
{
this.droplets.ScheduleSnapshotAppState();
}
}
finally
{
instance.Lock.ExitWriteLock();
}
this.DetectAppReady(instance);
}
catch (Exception ex)
{
Logger.Warning(Strings.FailedStagingAppDir, instance.Properties.Directory, instance.Properties.LoggingId, ex.ToString());
try
{
instance.Lock.EnterWriteLock();
instance.Properties.State = DropletInstanceState.Crashed;
instance.Properties.ExitReason = DropletExitReason.Crashed;
instance.Properties.StateTimestamp = DateTime.Now;
this.StopDroplet(instance);
}
finally
{
instance.Lock.ExitWriteLock();
}
}
}
/// <summary>
/// Unregisters the instance from the Vcap router. Called when the applicatoin is not in a running state any more.
/// </summary>
/// <param name="instance">The instance.</param>
private void UnregisterInstanceFromRouter(DropletInstance instance)
{
RouterMessage response = new RouterMessage();
try
{
instance.Lock.EnterReadLock();
if (instance.Properties.Uris == null || instance.Properties.Uris.Length == 0)
{
return;
}
response.DeaId = UUID;
response.Host = Host;
response.Port = instance.Properties.Port;
response.Uris = instance.Properties.Uris;
response.Tags = new RouterMessage.TagsObject();
response.Tags.Component = "dea-" + this.Index.ToString();
}
finally
{
instance.Lock.ExitReadLock();
}
this.deaReactor.SendRouterUnregister(response.SerializeToJson());
}
/// <summary>
/// Detects if an droplet instance is ready, so that it can be set to a Running state and registerd with the router.
/// </summary>
/// <param name="instance">The instance do be detected.</param>
private void DetectAppReady(DropletInstance instance)
{
ThreadPool.QueueUserWorkItem(
delegate
{
DetectAppReady(
instance,
delegate(bool detected)
{
try
{
instance.Lock.EnterWriteLock();
if (detected)
{
if (instance.Properties.State == DropletInstanceState.Starting)
{
Logger.Info(Strings.InstanceIsReadyForConnections, instance.Properties.LoggingId);
instance.Properties.State = DropletInstanceState.Running;
instance.Properties.StateTimestamp = DateTime.Now;
this.deaReactor.SendDeaHeartbeat(instance.GenerateHeartbeat().SerializeToJson());
this.RegisterInstanceWithRouter(instance);
this.droplets.ScheduleSnapshotAppState();
}
}
else
{
Logger.Warning(Strings.GivingUpOnConnectingApp);
this.StopDroplet(instance);
}
}
finally
{
instance.Lock.ExitWriteLock();
}
});
});
}
/// <summary>
/// Setups the instance environment variables to be passed when configuring the plugin of an instance.
/// </summary>
/// <param name="instance">The instance for which to generate the variables.</param>
/// <param name="appVars">The user application variables.</param>
/// <param name="services">The services to be bound to the instance.</param>
/// <returns>The application variables.</returns>
private Dictionary<string, string> SetupInstanceEnv(DropletInstance instance, string[] appVars, Dictionary<string, object>[] services)
{
Dictionary<string, string> env = new Dictionary<string, string>();
env.Add(HomeVariable, instance.Properties.Directory);
env.Add(VcapApplicationVariable, this.CreateInstanceVariable(instance));
env.Add(VcapServicesVariable, CreateServicesApplicationVariable(services));
env.Add(VcapAppHostVariable, Host);
env.Add(VcapAppPortVariable, instance.Properties.Port.ToString(CultureInfo.InvariantCulture));
env.Add(VcapAppDebugIpVariable, instance.Properties.DebugIP);
env.Add(VcapAppDebugPortVariable, instance.Properties.DebugPort != null ? instance.Properties.DebugPort.ToString() : null);
if (instance.Properties.DebugPort != null && this.stager.Runtimes[instance.Properties.Runtime].DebugEnvironmentVariables != null)
{
if (this.stager.Runtimes[instance.Properties.Runtime].DebugEnvironmentVariables.ContainsKey(instance.Properties.DebugMode))
{
foreach (KeyValuePair<string, string> debugEnv in this.stager.Runtimes[instance.Properties.Runtime].DebugEnvironmentVariables[instance.Properties.DebugMode])
{
env.Add(debugEnv.Key, debugEnv.Value);
}
}
}
// Do the runtime environment settings
foreach (KeyValuePair<string, string> runtimeEnv in this.stager.Runtimes[instance.Properties.Runtime].Environment)
{
env.Add(runtimeEnv.Key, runtimeEnv.Value);
}
// User's environment settings
if (appVars != null)
{
foreach (string appEnv in appVars)
{
string[] envVar = appEnv.Split(new char[] { '=' }, 2);
env.Add(envVar[0], envVar[1]);
}
}
return env;
}
private void StartDropletInstance(DropletInstance instance, string sha1, string executableFile, Uri executableUri)
{
try
{
string tgzFile = Path.Combine(this.stager.StagedDir, sha1 + ".tgz");
this.stager.StageAppDirectory(executableFile, executableUri, sha1, tgzFile, instance);
Logger.Debug(Strings.Downloadcompleate);
string starting = string.Format(CultureInfo.InvariantCulture, Strings.StartingUpInstanceOnPort, instance.Properties.LoggingId, instance.Properties.Port);
if (!string.IsNullOrEmpty(instance.Properties.DebugMode))
{
Logger.Info(starting + Strings.WithDebuggerPort, instance.Properties.DebugPort);
}
else
{
Logger.Info(starting);
}
Logger.Debug(Strings.Clients, this.monitoring.Clients);
Logger.Debug(Strings.ReservedMemoryUsageMb, this.monitoring.MemoryReservedMbytes, this.monitoring.MaxMemoryMbytes);
List<ApplicationVariable> appVariables = new List<ApplicationVariable>();
try
{
instance.Lock.EnterWriteLock();
instance.Properties.WindowsPassword = "******" + Credentials.GenerateCredential();
instance.Properties.WindowsUserName = WindowsVCAPUsers.CreateUser(instance.Properties.InstanceId, instance.Properties.WindowsPassword);
instance.Properties.EnvironmentVariables.Add(VcapWindowsUserVariable, instance.Properties.WindowsUserName);
instance.Properties.EnvironmentVariables.Add(VcapWindowsUserPasswordVariable, instance.Properties.WindowsPassword);
instance.Properties.EnvironmentVariables.Add(VcapPluginStagingInfoVariable, File.ReadAllText(Path.Combine(instance.Properties.Directory, "startup")));
foreach (KeyValuePair<string, string> appEnv in instance.Properties.EnvironmentVariables)
{
ApplicationVariable appVariable = new ApplicationVariable();
appVariable.Name = appEnv.Key;
appVariable.Value = appEnv.Value;
appVariables.Add(appVariable);
}
}
finally
{
instance.Lock.ExitWriteLock();
}
DateTime start = DateTime.Now;
instance.LoadPlugin();
instance.Plugin.ConfigureApplication(appVariables.ToArray());
instance.Plugin.StartApplication();
int pid = instance.Plugin.GetApplicationProcessId();
Logger.Debug(Strings.TookXTimeToLoadConfigureAndStartDebugMessage, (DateTime.Now - start).TotalSeconds);
try
{
instance.Lock.EnterWriteLock();
if (!instance.Properties.StopProcessed)
{
Logger.Info(Strings.PidAssignedToDroplet, pid, instance.Properties.LoggingId);
instance.Properties.ProcessId = pid;
this.droplets.ScheduleSnapshotAppState();
}
}
finally
{
instance.Lock.ExitWriteLock();
}
this.DetectAppReady(instance);
}
catch (Exception ex)
{
Logger.Warning(Strings.FailedStagingAppDir, instance.Properties.Directory, instance.Properties.LoggingId, ex.ToString());
try
{
instance.Lock.EnterWriteLock();
instance.Properties.State = DropletInstanceState.Crashed;
instance.Properties.ExitReason = DropletExitReason.Crashed;
instance.Properties.StateTimestamp = DateTime.Now;
this.StopDroplet(instance);
}
finally
{
instance.Lock.ExitWriteLock();
}
}
}
/// <summary>
/// Checks the usage of the instance. If it has a usage above the quota and the DEA is in secure mode, the instance will be stopped.
/// </summary>
/// <param name="instance">The instance to checks.</param>
private void CheckUsage(DropletInstance instance)
{
DropletInstanceUsage curUsage = instance.Properties.LastUsage;
if (instance == null || curUsage == null)
{
return;
}
// Check Memory
if (curUsage.MemoryKbytes > (instance.Properties.MemoryQuotaBytes / 1024))
{
instance.ErrorLog.Fatal(Strings.MemoryLimitOfExceeded, instance.Properties.MemoryQuotaBytes / 1024 / 1024);
instance.ErrorLog.Fatal(Strings.ActualUsageWasProcessTerminated, curUsage.MemoryKbytes / 1024);
this.StopDroplet(instance);
}
// Check Disk
if (curUsage.DiskBytes > instance.Properties.DiskQuotaBytes)
{
instance.ErrorLog.Fatal(Strings.DiskUsageLimitOf, instance.Properties.DiskQuotaBytes / 1024 / 1024);
instance.ErrorLog.Fatal(Strings.ActualUsageWasProcessTerminated, curUsage.DiskBytes / 1024 / 1024);
this.StopDroplet(instance);
}
// Check CPU
if (instance.Usage.Count == 0)
{
return;
}
if (curUsage.Cpu > Monitoring.BeginReniceCpuThreshold)
{
int nice = instance.Properties.Nice + 1;
if (nice <= Monitoring.MaxReniceValue)
{
instance.Properties.Nice = nice;
ProcessPriorityClass priority =
nice == 0 ? ProcessPriorityClass.Normal :
nice == 1 ? ProcessPriorityClass.BelowNormal :
ProcessPriorityClass.Idle;
instance.ErrorLog.Warning(Strings.LoggerLoweringPriority, priority.ToString());
Logger.Info(Strings.LoweringPriorityOnCpuBound, instance.Properties.Name, priority);
// Process.GetProcessById(instance.Properties.ProcessId).PriorityClass = priority;
instance.JobObject.PriorityClass = priority;
}
}
// TODO, Check for an attack, or what looks like one, and look at history?
// pegged_cpus = @num_cores * 100
// also check for opened handles
}
public void RecoverExistingDroplets()
{
if (!File.Exists(this.droplets.AppStateFile))
{
this.droplets.RecoveredDroplets = true;
return;
}
object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.droplets.AppStateFile));
foreach (object obj in instances)
{
DropletInstance instance = null;
try
{
instance = new DropletInstance();
instance.Properties.FromJsonIntermediateObject(obj);
instance.Properties.Orphaned = true;
instance.Properties.ResourcesTracked = false;
this.monitoring.AddInstanceResources(instance);
instance.Properties.StopProcessed = false;
instance.JobObject.JobMemoryLimit = instance.Properties.MemoryQuotaBytes;
try
{
instance.LoadPlugin();
instance.Properties.EnvironmentVariables[VcapAppPidVariable] = instance.Properties.ProcessId.ToString(CultureInfo.InvariantCulture);
List<ApplicationVariable> appVariables = new List<ApplicationVariable>();
foreach (KeyValuePair<string, string> appEnv in instance.Properties.EnvironmentVariables)
{
ApplicationVariable appVariable = new ApplicationVariable();
appVariable.Name = appEnv.Key;
appVariable.Value = appEnv.Value;
appVariables.Add(appVariable);
}
instance.Plugin.RecoverApplication(appVariables.ToArray());
}
catch (Exception ex)
{
instance.ErrorLog.Error(ex.ToString());
}
if (instance.Properties.State == DropletInstanceState.Starting)
{
this.DetectAppReady(instance);
}
this.droplets.AddDropletInstance(instance);
instance = null;
}
catch (Exception ex)
{
Logger.Warning(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.InstanceId, ex.ToString());
}
finally
{
if (instance != null)
{
instance.Dispose();
}
}
}
this.droplets.RecoveredDroplets = true;
if (this.monitoring.Clients > 0)
{
Logger.Info(Strings.DeaRecoveredApplications, this.monitoring.Clients);
}
this.MonitorApps();
this.droplets.ForEach(delegate(DropletInstance instance)
{
this.RegisterInstanceWithRouter(instance);
});
this.SendHeartbeat();
this.droplets.ScheduleSnapshotAppState();
}
/// <summary>
/// Setups the instance environment variables to be passed when configuring the plugin of an instance.
/// </summary>
/// <param name="instance">The instance for which to generate the variables.</param>
/// <param name="appVars">The user application variables.</param>
/// <param name="services">The services to be bound to the instance.</param>
/// <returns>The application variables.</returns>
private Dictionary<string, string> SetupInstanceEnv(DropletInstance instance, string[] appVars, Dictionary<string, object>[] services)
{
Dictionary<string, string> env = new Dictionary<string, string>();
env.Add(HomeVariable, Path.Combine(instance.Properties.Directory, "app"));
env.Add(VcapApplicationVariable, this.CreateInstanceVariable(instance));
env.Add(VcapServicesVariable, CreateServicesApplicationVariable(services));
env.Add(VcapAppHostVariable, Host);
env.Add(VcapAppPortVariable, instance.Properties.Port.ToString(CultureInfo.InvariantCulture));
env.Add("PORT", instance.Properties.Port.ToString(CultureInfo.InvariantCulture));
// User's environment settings
if (appVars != null)
{
foreach (string appEnv in appVars)
{
string[] envVar = appEnv.Split(new char[] { '=' }, 2);
env.Add(envVar[0], envVar[1]);
}
}
return env;
}
/// <summary>
/// Untracks the memory used by the instance and flags it/
/// </summary>
/// <param name="instance">The instance to be untracked.</param>
public void RemoveInstanceResources(DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterWriteLock();
if (instance.Properties.ResourcesTracked)
{
instance.Properties.ResourcesTracked = false;
this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024;
this.Clients--;
}
}
finally
{
instance.Lock.ExitWriteLock();
this.Lock.ExitWriteLock();
}
}
private void StopDroplet(DropletInstance instance)
{
try
{
instance.Lock.EnterWriteLock();
if (instance.Properties.StopProcessed)
{
return;
}
// Unplug us from the system immediately, both the routers and health managers.
if (!instance.Properties.NotifiedExited)
{
this.UnregisterInstanceFromRouter(instance);
if (instance.Properties.ExitReason == null)
{
instance.Properties.ExitReason = DropletExitReason.Crashed;
instance.Properties.State = DropletInstanceState.Crashed;
instance.Properties.StateTimestamp = DateTime.Now;
if (!instance.IsProcessIdRunning)
{
instance.Properties.ProcessId = 0;
}
}
this.deaReactor.SendDropletExited(instance.GenerateDropletExitedMessage().SerializeToJson());
instance.Properties.NotifiedExited = true;
}
Logger.Info(Strings.StoppingInstance, instance.Properties.LoggingId);
// if system thinks this process is running, make sure to execute stop script
if (instance.Properties.State == DropletInstanceState.Starting || instance.Properties.State == DropletInstanceState.Running)
{
instance.Properties.State = DropletInstanceState.Stopped;
instance.Properties.StateTimestamp = DateTime.Now;
}
// this.monitoring.RemoveInstanceResources(instance);
instance.Properties.StopProcessed = true;
}
catch (Exception ex)
{
Logger.Error(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.DropletId, instance.Properties.InstanceId, ex.ToString());
}
finally
{
instance.Lock.ExitWriteLock();
}
}
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
DropletInstance instance = null;
instance = new DropletInstance();
string instanceId = Credentials.GenerateSecureGuid().ToString("N");
string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");
instance.Properties.State = DropletInstanceState.Starting;
instance.Properties.StateTimestamp = DateTime.Now;
instance.Properties.Start = DateTime.Now;
instance.Properties.InstanceId = instanceId;
instance.Properties.PrivateInstanceId = privateInstanceId;
instance.Properties.DropletId = message.DropletId;
instance.Properties.InstanceIndex = message.Index;
instance.Properties.Name = message.Name;
instance.Properties.Uris = message.Uris;
instance.Properties.Users = message.Users;
instance.Properties.Version = message.Version;
instance.Properties.Stack = message.Stack;
instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index);
instance.Properties.Flapping = message.Flapping;
instance.Properties.CloudControllerPartition = message.CloudControllerPartition;
this.AddDropletInstance(instance);
return instance;
}
public void RecoverExistingDroplets()
{
if (!File.Exists(this.droplets.AppStateFile))
{
this.droplets.RecoveredDroplets = true;
return;
}
object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.droplets.AppStateFile));
foreach (object obj in instances)
{
DropletInstance instance = null;
try
{
instance = new DropletInstance();
instance.Properties.FromJsonIntermediateObject(obj);
instance.Properties.Orphaned = true;
instance.Properties.ResourcesTracked = false;
this.monitoring.AddInstanceResources(instance);
instance.Properties.StopProcessed = false;
var prisonInfo = new ProcessPrisonCreateInfo();
prisonInfo.Id = instance.Properties.InstanceId;
prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes;
prisonInfo.WindowsPassword = instance.Properties.WindowsPassword;
if (this.useDiskQuota)
{
prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes;
prisonInfo.DiskQuotaPath = instance.Properties.Directory;
}
Logger.Info("Recovering Process Prisson: {0}", prisonInfo.Id);
instance.Prison.Attach(prisonInfo);
if (instance.Properties.State == DropletInstanceState.Starting)
{
this.DetectAppReady(instance);
}
this.droplets.AddDropletInstance(instance);
instance = null;
}
catch (Exception ex)
{
Logger.Warning(Strings.ErrorRecoveringDropletWarningMessage, instance.Properties.InstanceId, ex.ToString());
}
finally
{
if (instance != null)
{
instance.Dispose();
}
}
}
this.droplets.RecoveredDroplets = true;
if (this.monitoring.Clients > 0)
{
Logger.Info(Strings.DeaRecoveredApplications, this.monitoring.Clients);
}
this.MonitorApps();
this.droplets.ForEach(delegate(DropletInstance instance)
{
this.RegisterInstanceWithRouter(instance);
});
this.SendHeartbeat();
this.droplets.ScheduleSnapshotAppState();
}
/// <summary>
/// Adds a droplet instance to the collection.
/// </summary>
/// <param name="instance">The droplet instance to add.</param>
public void AddDropletInstance(DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterReadLock();
if (!this.Droplets.ContainsKey(instance.Properties.DropletId))
{
this.Droplets.Add(instance.Properties.DropletId, new Droplet());
}
this.Droplets[instance.Properties.DropletId].DropletInstances[instance.Properties.InstanceId] = instance;
}
finally
{
instance.Lock.ExitReadLock();
this.Lock.ExitWriteLock();
}
this.ScheduleSnapshotAppState();
}
/// <summary>
/// Detects if an application has the port ready and then invoke the call back.
/// </summary>
/// <param name="instance">The instance to be checked.</param>
/// <param name="callBack">The call back.</param>
private static void DetectPortReady(DropletInstance instance, BoolStateBlockCallback callBack)
{
int attempts = 0;
bool keep_going = true;
while (attempts <= 1000 && instance.Properties.State == DropletInstanceState.Starting && keep_going == true)
{
if (instance.IsPortReady(150))
{
keep_going = false;
callBack(true);
}
else
{
Thread.Sleep(100);
attempts++;
}
}
if (keep_going)
{
callBack(false);
}
}
public DropletInstance CreateDropletInstance(DeaStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
DropletInstance instance = null;
instance = new DropletInstance();
string instanceId = Guid.NewGuid().ToString("N");
instance.Properties.State = DropletInstanceState.Starting;
instance.Properties.StateTimestamp = DateTime.Now;
instance.Properties.Start = DateTime.Now;
instance.Properties.InstanceId = instanceId;
instance.Properties.DropletId = message.DropletId;
instance.Properties.InstanceIndex = message.Index;
instance.Properties.Name = message.Name;
instance.Properties.Uris = message.Uris;
instance.Properties.Users = message.Users;
instance.Properties.Version = message.Version;
instance.Properties.Framework = message.Framework;
instance.Properties.Runtime = message.Runtime;
instance.Properties.LoggingId = string.Format(CultureInfo.InvariantCulture, Strings.NameAppIdInstance, message.Name, message.DropletId, instanceId, message.Index);
this.AddDropletInstance(instance);
return instance;
}
/// <summary>
/// Prepares the app directory.
/// </summary>
/// <param name="bitsFile">The bits file.</param>
/// <param name="bitsUri">The bits URI.</param>
/// <param name="hash">The sha1.</param>
/// <param name="tarZipFile">The TGZ file.</param>
/// <param name="instance">The instance.</param>
public void PrepareAppDirectory(string bitsFile, string bitsUri, string hash, string tarZipFile, DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
// What we do here, in order of preference..
// 1. Check our own staged directory.
// 2. Check shared directory from CloudController that could be mounted (bits_file)
// 3. Pull from http if needed.
string instanceDir = instance.Properties.Directory;
lock (this.stagerLock)
{
// check before downloading
if (instance.Properties.StopProcessed)
{
return;
}
if (File.Exists(tarZipFile))
{
Logger.Debug(Strings.FoundStagedBitsInLocalCache);
}
else
{
// If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version.
DateTime start = DateTime.Now;
if (File.Exists(bitsFile))
{
Logger.Debug(Strings.SharingCloudControllerStagingDirectory);
File.Copy(bitsFile, tarZipFile);
Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start);
}
else
{
Uri downloadUri = new Uri(bitsUri);
Logger.Debug(Strings.Needtodownloadappbitsfrom, downloadUri);
this.DownloadAppBits(downloadUri, hash, tarZipFile);
Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start);
}
}
// check before extracting
if (instance.Properties.StopProcessed)
{
return;
}
DateTime startStageing = DateTime.Now;
// Explode the app into its directory and optionally bind its local runtime.
Directory.CreateDirectory(instance.Properties.Directory);
DirectoryInfo deploymentDirInfo = new DirectoryInfo(instance.Properties.Directory);
DirectorySecurity deploymentDirSecurity = deploymentDirInfo.GetAccessControl();
// Owner is important to account for disk quota
deploymentDirSecurity.SetOwner(new NTAccount(instance.Properties.WindowsUserName));
deploymentDirSecurity.SetAccessRule(
new FileSystemAccessRule(
instance.Properties.WindowsUserName,
FileSystemRights.Write | FileSystemRights.Read | FileSystemRights.Delete | FileSystemRights.Modify | FileSystemRights.FullControl,
InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit,
PropagationFlags.None | PropagationFlags.InheritOnly,
AccessControlType.Allow));
// Taking ownership of a file has to be executed with restore privilege elevated privilages
using (new ProcessPrivileges.PrivilegeEnabler(Process.GetCurrentProcess(), ProcessPrivileges.Privilege.Restore))
{
deploymentDirInfo.SetAccessControl(deploymentDirSecurity);
}
// Impersonate user to cascade the owernship to every file
// Neccessary for windows disk quota
using (new UserImpersonator(instance.Properties.WindowsUserName, ".", instance.Properties.WindowsPassword, true))
{
DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip
string tarFileName = Directory.GetFiles(instanceDir, "*.tar")[0];
DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar
File.Delete(Path.Combine(instanceDir, tarFileName));
}
Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing);
}
}
/// <summary>
/// Removes a droplet instance from the collection.
/// </summary>
/// <param name="instance">The droplet instance to remove.</param>
public void RemoveDropletInstance(DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
if (this.Droplets.ContainsKey(instance.Properties.DropletId))
{
Droplet droplet = this.Droplets[instance.Properties.DropletId];
if (droplet.DropletInstances.ContainsKey(instance.Properties.InstanceId))
{
droplet.DropletInstances.Remove(instance.Properties.InstanceId);
if (droplet.DropletInstances.Count == 0)
{
this.Droplets.Remove(instance.Properties.DropletId);
}
}
}
}
finally
{
this.Lock.ExitWriteLock();
}
this.ScheduleSnapshotAppState();
}
/// <summary>
/// Stages the app directory.
/// </summary>
/// <param name="bitsFile">The bits file.</param>
/// <param name="bitsUri">The bits URI.</param>
/// <param name="hash">The sha1.</param>
/// <param name="tarZipFile">The TGZ file.</param>
/// <param name="instance">The instance.</param>
public void StageAppDirectory(string bitsFile, Uri bitsUri, string hash, string tarZipFile, DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
// What we do here, in order of preference..
// 1. Check our own staged directory.
// 2. Check shared directory from CloudController that could be mounted (bits_file)
// 3. Pull from http if needed.
string instanceDir = instance.Properties.Directory;
lock (this.stagerLock)
{
// check before downloading
if (instance.Properties.StopProcessed)
{
return;
}
if (File.Exists(tarZipFile))
{
Logger.Debug(Strings.FoundStagedBitsInLocalCache);
}
else
{
// If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version.
DateTime start = DateTime.Now;
if (!this.ForceHttpFileSharing && File.Exists(bitsFile))
{
Logger.Debug(Strings.SharingCloudControllerStagingDirectory);
File.Copy(bitsFile, tarZipFile);
Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start);
}
else
{
Logger.Debug(Strings.Needtodownloadappbitsfrom, bitsUri);
this.DownloadAppBits(bitsUri, hash, tarZipFile);
Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start);
}
}
// check before extracting
if (instance.Properties.StopProcessed)
{
return;
}
DateTime startStageing = DateTime.Now;
// Explode the app into its directory and optionally bind its local runtime.
Directory.CreateDirectory(instanceDir);
string tarFileName = Path.GetFileName(tarZipFile);
tarFileName = Path.ChangeExtension(tarFileName, ".tar");
DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip
DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar
File.Delete(Path.Combine(instanceDir, tarFileName));
Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing);
}
}
/// <summary>
/// Stages the app directory.
/// </summary>
/// <param name="bitsFile">The bits file.</param>
/// <param name="bitsUri">The bits URI.</param>
/// <param name="hash">The sha1.</param>
/// <param name="tarZipFile">The TGZ file.</param>
/// <param name="instance">The instance.</param>
public void StageAppDirectory(string bitsFile, Uri bitsUri, string hash, string tarZipFile, DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
// What we do here, in order of preference..
// 1. Check our own staged directory.
// 2. Check shared directory from CloudController that could be mounted (bits_file)
// 3. Pull from http if needed.
string instanceDir = instance.Properties.Directory;
lock (this.stagerLock)
{
// check before downloading
if (instance.Properties.StopProcessed)
{
return;
}
if (File.Exists(tarZipFile))
{
Logger.Debug(Strings.FoundStagedBitsInLocalCache);
}
else
{
// If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version.
DateTime start = DateTime.Now;
if (!this.ForceHttpFileSharing && File.Exists(bitsFile))
{
Logger.Debug(Strings.SharingCloudControllerStagingDirectory);
File.Copy(bitsFile, tarZipFile);
Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start);
}
else
{
Logger.Debug(Strings.Needtodownloadappbitsfrom, bitsUri);
this.DownloadAppBits(bitsUri, hash, tarZipFile);
Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start);
}
}
// check before extracting
if (instance.Properties.StopProcessed)
{
return;
}
DateTime startStageing = DateTime.Now;
// Explode the app into its directory and optionally bind its local runtime.
Directory.CreateDirectory(instanceDir);
string tarFileName = Path.GetFileName(tarZipFile);
tarFileName = Path.ChangeExtension(tarFileName, ".tar");
DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip
DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar
File.Delete(Path.Combine(instanceDir, tarFileName));
Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing);
}
}
