/// <summary>
/// Untracks the memory used by the instance and flags it/
/// </summary>
/// <param name="instance">The instance to be untracked.</param>
public void RemoveInstanceResources(StagingInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterWriteLock();
if (instance.Properties.ResourcesTracked)
{
instance.Properties.ResourcesTracked = false;
this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024;
this.Clients--;
}
}
finally
{
instance.Lock.ExitWriteLock();
this.Lock.ExitWriteLock();
}
}
public StagingInstance CreateStagingInstance(StagingStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
StagingInstance instance = new StagingInstance();
string instanceId = Credentials.GenerateSecureGuid().ToString("N");
string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");
instance.StartMessage = message.StartMessage;
instance.Properties.InstanceId = instanceId;
instance.Properties.TaskId = message.TaskID;
instance.Properties.AppId = message.AppID;
instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI;
instance.Properties.BuildpackCacheUploadURI = message.BuildpackCacheUploadURI;
instance.Properties.DownloadURI = message.DownloadURI;
instance.Properties.UploadURI = message.UploadURI;
if (message.Properties.Meta != null)
{
if (message.Properties.Meta.Command != null)
{
instance.Properties.MetaCommand = message.Properties.Meta.Command;
}
}
this.AddStagingInstance(instance);
return(instance);
}
public void AddStagingInstance(StagingInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterReadLock();
if (!this.Tasks.ContainsKey(instance.Properties.TaskId))
{
this.Tasks.Add(instance.Properties.TaskId, instance);
}
}
finally
{
instance.Lock.ExitReadLock();
this.Lock.ExitWriteLock();
}
this.ScheduleSnapshotStagingState();
}
private void CleanupStagingInstances()
{
if (!File.Exists(this.stagingTaskRegistry.StagingStateFile))
{
return;
}
object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.stagingTaskRegistry.StagingStateFile));
foreach (object obj in instances)
{
StagingInstance instance = null;
try
{
instance = new StagingInstance();
instance.Properties.FromJsonIntermediateObject(obj);
var prisonInfo = new ProcessPrisonCreateInfo();
prisonInfo.Id = instance.Properties.InstanceId;
prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes;
prisonInfo.WindowsPassword = instance.Properties.WindowsPassword;
if (this.useDiskQuota)
{
prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes;
prisonInfo.DiskQuotaPath = instance.Properties.Directory;
}
Logger.Info("Recovering Process Prisson: {0}", prisonInfo.Id);
instance.Prison.Attach(prisonInfo);
foreach (Process p in instance.Prison.jobObject.GetJobProcesses())
{
if (!p.HasExited)
{
p.Kill();
}
}
if (instance.Prison.Created)
{
instance.Prison.Destroy();
}
instance.Workspace = new StagingWorkspace(instance.Properties.Directory);
instance.Cleanup();
}
catch (Exception ex)
{
Logger.Warning("Error deleting staging environment for task {0}: {1}", instance.Properties.TaskId, ex.ToString());
}
finally
{
this.stagingTaskRegistry.RemoveStagingInstance(instance);
if (instance != null)
{
instance.Dispose();
}
}
}
}
private void AfterStagingSetup(StagingInstance instance)
{
StagingStartMessageResponse response = new StagingStartMessageResponse();
response.TaskId = instance.Properties.TaskId;
response.TaskStreamingLogURL = instance.Properties.StreamingLogUrl;
if (instance.StagingException != null)
{
response.Error = instance.StagingException.ToString();
}
this.deaReactor.SendReply(instance.Properties.Reply, response.SerializeToJson());
Logger.Debug("Staging task {0}: sent reply {1}", instance.Properties.TaskId, response.SerializeToJson());
}
private void AfterStagingFinished(StagingInstance instance)
{
StagingStartMessageResponse response = new StagingStartMessageResponse();
try
{
if (instance.StagingException == null)
{
try
{
Logger.Info("Staging task {0}: Saving buildpackInfo", instance.Properties.TaskId);
StagingInfo.SaveBuildpackInfo(Path.Combine(instance.Workspace.StagedDir, StagingWorkspace.StagingInfo), instance.Buildpack.Name, instance.GetStartCommand());
this.stagingTaskRegistry.ScheduleSnapshotStagingState();
Logger.Debug("Staging task {0}: Packing droplet {1}", instance.Properties.TaskId, instance.Workspace.StagedDropletPath);
Directory.CreateDirectory(instance.Workspace.StagedDropletDir);
string tempFile = Path.ChangeExtension(instance.Workspace.StagedDropletPath, "tar");
DEAUtilities.TarDirectory(instance.Workspace.StagedDir, tempFile);
DEAUtilities.GzipFile(tempFile, instance.Workspace.StagedDropletPath);
File.Delete(tempFile);
if (File.Exists(instance.Workspace.StagedDropletPath))
{
using (Stream stream = File.OpenRead(instance.Workspace.StagedDropletPath))
{
using (SHA1 sha = SHA1.Create())
{
response.DropletSHA = BitConverter.ToString(sha.ComputeHash(stream)).Replace("-", string.Empty);
}
}
}
this.StartStagedDropletInstance(instance, response.DropletSHA);
Uri uri = new Uri(instance.Properties.UploadURI);
Logger.Debug("Staging task {0}: Uploading droplet {1} to {2}", instance.Properties.TaskId, instance.Workspace.StagedDropletPath, instance.Properties.UploadURI);
DEAUtilities.HttpUploadFile(instance.Properties.UploadURI, new FileInfo(instance.Workspace.StagedDropletPath), "upload[droplet]", "application/octet-stream", uri.UserInfo);
}
catch (Exception ex)
{
instance.StagingException = ex;
}
try
{
Directory.CreateDirectory(instance.Workspace.Cache);
string tempFile = Path.ChangeExtension(instance.Workspace.StagedBuildpackCachePath, "tar");
DEAUtilities.TarDirectory(instance.Workspace.Cache, tempFile);
DEAUtilities.GzipFile(tempFile, instance.Workspace.StagedBuildpackCachePath);
Uri uri = new Uri(instance.Properties.BuildpackCacheUploadURI);
Logger.Debug("Staging task {0}: Uploading buildpack cache {1} to {2}", instance.Properties.TaskId, instance.Workspace.StagedBuildpackCachePath, instance.Properties.BuildpackCacheUploadURI);
DEAUtilities.HttpUploadFile(instance.Properties.BuildpackCacheUploadURI, new FileInfo(instance.Workspace.StagedBuildpackCachePath), "upload[droplet]", "application/octet-stream", uri.UserInfo);
}
catch
{
Logger.Debug("Staging task {0}: Cannot pack buildpack cache", instance.Properties.TaskId);
}
}
if (instance.StagingException != null)
{
response.Error = instance.StagingException.ToString();
}
response.TaskId = instance.Properties.TaskId;
// try to read log. don't throw exception if it fails
try
{
response.TaskLog = File.ReadAllText(instance.Properties.TaskLog);
}
catch { }
if (instance.Properties.DetectedBuildpack != null)
{
response.DetectedBuildpack = instance.Properties.DetectedBuildpack;
}
this.deaReactor.SendReply(instance.Properties.Reply, response.SerializeToJson());
Logger.Debug("Staging task {0}: sent reply {1}", instance.Properties.TaskId, response.SerializeToJson());
}
finally
{
Logger.Debug("Cleaning up directory {0}", instance.Workspace.BaseDir);
instance.Cleanup();
}
}
private void StopStaging(StagingInstance instance, string reply_to)
{
try
{
if (instance.Properties.Stopped)
{
return;
}
instance.Lock.EnterWriteLock();
instance.Properties.Stopped = true;
StagingStartMessageResponse response = new StagingStartMessageResponse();
response.TaskId = instance.Properties.TaskId;
this.deaReactor.SendReply(reply_to, response.SerializeToJson());
}
catch (Exception ex)
{
Logger.Error("Could not stop staging task {0}: {1}", instance.Properties.TaskId, ex.ToString());
}
finally
{
instance.Lock.ExitWriteLock();
}
}
private void StartStagingInstance(StagingInstance instance, StagingStartMessageRequest pmessage)
{
StagingWorkspace workspace = new StagingWorkspace(instance.Properties.Directory);
try
{
try
{
instance.Lock.EnterWriteLock();
instance.Properties.UseDiskQuota = this.useDiskQuota;
instance.Properties.UploadThrottleBitsps = this.uploadThrottleBitsps;
UriBuilder streamingLog = new UriBuilder();
streamingLog.Host = this.ExternalHost;
streamingLog.Scheme = "http";
streamingLog.Path = string.Format("/staging_tasks/{0}/file_path", pmessage.TaskID);
streamingLog.Query = string.Format("path={0}×tamp={1}", workspace.StagingLogSuffix, RubyCompatibility.DateTimeToEpochSeconds(DateTime.Now));
instance.Properties.StreamingLogUrl = DEAUtilities.GetHmacedUri(streamingLog.Uri.ToString(), this.directoryServerHmacKey, new string[] { "path", "timestamp" }).ToString();
instance.Workspace = workspace;
instance.Properties.TaskLog = workspace.StagingLogPath;
}
finally
{
instance.Lock.ExitWriteLock();
}
instance.AfterSetup += new StagingInstance.StagingTaskEventHandler(this.AfterStagingSetup);
Logger.Info("Started staging task {0}", instance.Properties.TaskId);
try
{
instance.SetupStagingEnvironment();
}
catch (Exception ex)
{
Logger.Error("Error setting up staging environment: ", ex.ToString());
throw ex;
}
instance.UnpackDroplet();
instance.PrepareStagingDirs();
instance.GetBuildpack(pmessage, this.gitPath, this.buildpacksDir);
this.stagingTaskRegistry.ScheduleSnapshotStagingState();
try
{
Logger.Info("Staging task {0}: Running compilation script", pmessage.TaskID);
instance.CreatePrison();
this.stagingTaskRegistry.ScheduleSnapshotStagingState();
instance.CompileProcess = instance.Buildpack.StartCompile(instance.Prison);
instance.Lock.EnterWriteLock();
instance.Properties.Start = DateTime.Now;
}
finally
{
if(instance.Lock.IsWriteLockHeld)
{
instance.Lock.ExitWriteLock();
}
}
}
catch (Exception ex)
{
instance.StagingException = ex;
instance.Properties.Stopped = true;
Logger.Error(ex.ToString());
}
}
private void StartStagedDropletInstance(StagingInstance stagingInstance, string dropletSha)
{
DropletInstance instance;
try
{
this.droplets.Lock.EnterWriteLock();
if (this.shuttingDown)
{
return;
}
string tgzFile = Path.Combine(this.fileResources.StagedDir, dropletSha + ".tgz");
Logger.Info("Copying droplet to {0}", tgzFile);
File.Copy(stagingInstance.Workspace.StagedDropletPath, tgzFile);
long memoryMbytes = stagingInstance.StartMessage.Limits != null && stagingInstance.StartMessage.Limits.MemoryMbytes != null ? stagingInstance.StartMessage.Limits.MemoryMbytes.Value : Monitoring.DefaultAppMemoryMbytes;
long diskMbytes = stagingInstance.StartMessage.Limits != null && stagingInstance.StartMessage.Limits.DiskMbytes != null ? stagingInstance.StartMessage.Limits.DiskMbytes.Value : Monitoring.DefaultAppDiskMbytes;
long fds = stagingInstance.StartMessage.Limits != null && stagingInstance.StartMessage.Limits.FileDescriptors != null ? stagingInstance.StartMessage.Limits.FileDescriptors.Value : Monitoring.DefaultAppFds;
if (this.monitoring.MemoryReservedMbytes + memoryMbytes > this.monitoring.MaxMemoryMbytes || this.monitoring.Clients >= this.monitoring.MaxClients)
{
Logger.Info(Strings.Donothaveroomforthisclient);
return;
}
instance = this.droplets.CreateDropletInstance(stagingInstance.StartMessage);
instance.Properties.MemoryQuotaBytes = memoryMbytes * 1024 * 1024;
instance.Properties.DiskQuotaBytes = diskMbytes * 1024 * 1024;
instance.Properties.FDSQuota = fds;
instance.Properties.Staged = instance.Properties.Name + "-" + instance.Properties.InstanceIndex + "-" + instance.Properties.InstanceId;
instance.Properties.Directory = Path.Combine(this.fileResources.AppsDir, instance.Properties.Staged);
if (!string.IsNullOrEmpty(instance.Properties.DebugMode))
{
instance.Properties.DebugPort = NetworkInterface.GrabEphemeralPort();
instance.Properties.DebugIP = Host;
}
instance.Properties.Port = NetworkInterface.GrabEphemeralPort();
instance.Properties.EnvironmentVariables = this.SetupInstanceEnv(instance, stagingInstance.StartMessage.Environment, stagingInstance.StartMessage.Services);
this.monitoring.AddInstanceResources(instance);
}
finally
{
this.droplets.Lock.ExitWriteLock();
}
ThreadPool.QueueUserWorkItem(delegate(object data)
{
this.StartDropletInstance(instance, dropletSha, stagingInstance.StartMessage.ExecutableFile, stagingInstance.StartMessage.ExecutableUri);
});
}
public void AddStagingInstance(StagingInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterReadLock();
if (!this.Tasks.ContainsKey(instance.Properties.TaskId))
{
this.Tasks.Add(instance.Properties.TaskId, instance);
}
}
finally
{
instance.Lock.ExitReadLock();
this.Lock.ExitWriteLock();
}
this.ScheduleSnapshotStagingState();
}
public StagingInstance CreateStagingInstance(StagingStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
StagingInstance instance = new StagingInstance();
string instanceId = Credentials.GenerateSecureGuid().ToString("N");
string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");
instance.StartMessage = message.StartMessage;
instance.Properties.InstanceId = instanceId;
instance.Properties.TaskId = message.TaskID;
instance.Properties.AppId = message.AppID;
instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI;
instance.Properties.BuildpackCacheUploadURI = message.BuildpackCacheUploadURI;
instance.Properties.DownloadURI = message.DownloadURI;
instance.Properties.UploadURI = message.UploadURI;
if (message.Properties.Meta != null)
{
if (message.Properties.Meta.Command != null)
{
instance.Properties.MetaCommand = message.Properties.Meta.Command;
}
}
this.AddStagingInstance(instance);
return instance;
}
/// <summary>
/// Untracks the memory used by the instance and flags it/
/// </summary>
/// <param name="instance">The instance to be untracked.</param>
public void RemoveInstanceResources(StagingInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
try
{
this.Lock.EnterWriteLock();
instance.Lock.EnterWriteLock();
if (instance.Properties.ResourcesTracked)
{
instance.Properties.ResourcesTracked = false;
this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024;
this.Clients--;
}
}
finally
{
instance.Lock.ExitWriteLock();
this.Lock.ExitWriteLock();
}
}